From 2975f34140285bb4358a29474f8d4e01860d38a0 Mon Sep 17 00:00:00 2001
From: Igor Drozdov <idrozdov@gitlab.com>
Date: Fri, 18 Sep 2020 11:21:06 +0300
Subject: [PATCH] Increase LSIF scanner buffer
Sometimes we receive bufio.Scanner: token too long is the parsed
string is too long
Let's increase the buffer size
---
.../unreleased/id-increase-lsif-scanner-buffer.yml | 5 +++++
internal/lsif_transformer/parser/docs.go | 4 ++++
internal/lsif_transformer/parser/docs_test.go | 11 +++++++++++
3 files changed, 20 insertions(+)
create mode 100644 changelogs/unreleased/id-increase-lsif-scanner-buffer.yml
diff --git a/changelogs/unreleased/id-increase-lsif-scanner-buffer.yml b/changelogs/unreleased/id-increase-lsif-scanner-buffer.yml
new file mode 100644
index 0000000000000..5d6ecf1418663
--- /dev/null
+++ b/changelogs/unreleased/id-increase-lsif-scanner-buffer.yml
@@ -0,0 +1,5 @@
+---
+title: Increase LSIF scanner buffer
+merge_request: 609
+author:
+type: fixed
diff --git a/internal/lsif_transformer/parser/docs.go b/internal/lsif_transformer/parser/docs.go
index 3b293a1eecb53..6e1138eaf29ae 100644
--- a/internal/lsif_transformer/parser/docs.go
+++ b/internal/lsif_transformer/parser/docs.go
@@ -8,6 +8,8 @@ import (
"strings"
)
+const maxScanTokenSize = 1024 * 1024
+
type Line struct {
Type string `json:"label"`
}
@@ -49,6 +51,8 @@ func NewDocs(config Config) (*Docs, error) {
func (d *Docs) Parse(r io.Reader) error {
scanner := bufio.NewScanner(r)
+ buf := make([]byte, 0, bufio.MaxScanTokenSize)
+ scanner.Buffer(buf, maxScanTokenSize)
for scanner.Scan() {
if err := d.process(scanner.Bytes()); err != nil {
diff --git a/internal/lsif_transformer/parser/docs_test.go b/internal/lsif_transformer/parser/docs_test.go
index 1d0154c40ffba..b8b1072cabb65 100644
--- a/internal/lsif_transformer/parser/docs_test.go
+++ b/internal/lsif_transformer/parser/docs_test.go
@@ -3,6 +3,7 @@ package parser
import (
"bytes"
"fmt"
+ "strings"
"testing"
"github.com/stretchr/testify/require"
@@ -40,3 +41,13 @@ func TestParseContainsLine(t *testing.T) {
require.Equal(t, []Id{2, 3}, d.DocRanges[1])
}
+
+func TestParsingVeryLongLine(t *testing.T) {
+ d, err := NewDocs(Config{})
+ require.NoError(t, err)
+ defer d.Close()
+
+ line := []byte(`{"id": "` + strings.Repeat("a", 64*1024) + `"}`)
+
+ require.NoError(t, d.Parse(bytes.NewReader(line)))
+}
--
GitLab