diff --git a/changelogs/unreleased/id-increase-lsif-scanner-buffer.yml b/changelogs/unreleased/id-increase-lsif-scanner-buffer.yml new file mode 100644 index 0000000000000000000000000000000000000000..5d6ecf1418663aa28d9250a893aeb36539e12bda --- /dev/null +++ b/changelogs/unreleased/id-increase-lsif-scanner-buffer.yml @@ -0,0 +1,5 @@ +--- +title: Increase LSIF scanner buffer +merge_request: 609 +author: +type: fixed diff --git a/internal/lsif_transformer/parser/docs.go b/internal/lsif_transformer/parser/docs.go index 3b293a1eecb5373603482f89b9e8710b5b4477cf..6e1138eaf29aee56e5418e8b2b55c771fd92c019 100644 --- a/internal/lsif_transformer/parser/docs.go +++ b/internal/lsif_transformer/parser/docs.go @@ -8,6 +8,8 @@ import ( "strings" ) +const maxScanTokenSize = 1024 * 1024 + type Line struct { Type string `json:"label"` } @@ -49,6 +51,8 @@ func NewDocs(config Config) (*Docs, error) { func (d *Docs) Parse(r io.Reader) error { scanner := bufio.NewScanner(r) + buf := make([]byte, 0, bufio.MaxScanTokenSize) + scanner.Buffer(buf, maxScanTokenSize) for scanner.Scan() { if err := d.process(scanner.Bytes()); err != nil { diff --git a/internal/lsif_transformer/parser/docs_test.go b/internal/lsif_transformer/parser/docs_test.go index 1d0154c40ffba8967edaf83ae958e8fa29008b30..b8b1072cabb654e11b8168d5b3bd462de77b4d90 100644 --- a/internal/lsif_transformer/parser/docs_test.go +++ b/internal/lsif_transformer/parser/docs_test.go @@ -3,6 +3,7 @@ package parser import ( "bytes" "fmt" + "strings" "testing" "github.com/stretchr/testify/require" @@ -40,3 +41,13 @@ func TestParseContainsLine(t *testing.T) { require.Equal(t, []Id{2, 3}, d.DocRanges[1]) } + +func TestParsingVeryLongLine(t *testing.T) { + d, err := NewDocs(Config{}) + require.NoError(t, err) + defer d.Close() + + line := []byte(`{"id": "` + strings.Repeat("a", 64*1024) + `"}`) + + require.NoError(t, d.Parse(bytes.NewReader(line))) +}