Commit 2975f341 authored by Igor Drozdov's avatar Igor Drozdov

Increase LSIF scanner buffer

Sometimes we receive bufio.Scanner: token too long is the parsed
string is too long

Let's increase the buffer size
parent 8a20c4af
---
title: Increase LSIF scanner buffer
merge_request: 609
author:
type: fixed
...@@ -8,6 +8,8 @@ import ( ...@@ -8,6 +8,8 @@ import (
"strings" "strings"
) )
const maxScanTokenSize = 1024 * 1024
type Line struct { type Line struct {
Type string `json:"label"` Type string `json:"label"`
} }
...@@ -49,6 +51,8 @@ func NewDocs(config Config) (*Docs, error) { ...@@ -49,6 +51,8 @@ func NewDocs(config Config) (*Docs, error) {
func (d *Docs) Parse(r io.Reader) error { func (d *Docs) Parse(r io.Reader) error {
scanner := bufio.NewScanner(r) scanner := bufio.NewScanner(r)
buf := make([]byte, 0, bufio.MaxScanTokenSize)
scanner.Buffer(buf, maxScanTokenSize)
for scanner.Scan() { for scanner.Scan() {
if err := d.process(scanner.Bytes()); err != nil { if err := d.process(scanner.Bytes()); err != nil {
......
...@@ -3,6 +3,7 @@ package parser ...@@ -3,6 +3,7 @@ package parser
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"strings"
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
...@@ -40,3 +41,13 @@ func TestParseContainsLine(t *testing.T) { ...@@ -40,3 +41,13 @@ func TestParseContainsLine(t *testing.T) {
require.Equal(t, []Id{2, 3}, d.DocRanges[1]) require.Equal(t, []Id{2, 3}, d.DocRanges[1])
} }
func TestParsingVeryLongLine(t *testing.T) {
d, err := NewDocs(Config{})
require.NoError(t, err)
defer d.Close()
line := []byte(`{"id": "` + strings.Repeat("a", 64*1024) + `"}`)
require.NoError(t, d.Parse(bytes.NewReader(line)))
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment