Truncate hover values to 250 symbols

Super long documentation is not very usable, but consumes a lot
of memory

internal/lsif_transformer/parser/code_hover.go

@@ -3,20 +3,51 @@ package parser
 import (
 	"encoding/json"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/alecthomas/chroma"
 	"github.com/alecthomas/chroma/lexers"
 )
 
+const maxValueSize = 250
+
 type token struct {
 	Class string `json:"class,omitempty"`
 	Value string `json:"value"`
 }
 
 type codeHover struct {
-	Value    string    `json:"value,omitempty"`
-	Tokens   [][]token `json:"tokens,omitempty"`
-	Language string    `json:"language,omitempty"`
+	TruncatedValue *truncatableString `json:"value,omitempty"`
+	Tokens         [][]token          `json:"tokens,omitempty"`
+	Language       string             `json:"language,omitempty"`
+	Truncated      bool               `json:"truncated,omitempty"`
+}
+
+type truncatableString struct {
+	Value     string
+	Truncated bool
+}
+
+func (ts *truncatableString) UnmarshalText(b []byte) error {
+	s := 0
+	for i := 0; s < len(b); i++ {
+		if i >= maxValueSize {
+			ts.Truncated = true
+			break
+		}
+
+		_, size := utf8.DecodeRune(b[s:])
+
+		s += size
+	}
+
+	ts.Value = string(b[0:s])
+
+	return nil
+}
+
+func (ts *truncatableString) MarshalJSON() ([]byte, error) {
+	return json.Marshal(ts.Value)
 }
 
 func newCodeHover(content json.RawMessage) (*codeHover, error) {
@@ -24,7 +55,7 @@ func newCodeHover(content json.RawMessage) (*codeHover, error) {
 	// Or a string with documentation
 	// We try to unmarshal the content into a string and if we fail, we unmarshal it into an object
 	var c codeHover
-	if err := json.Unmarshal(content, &c.Value); err != nil {
+	if err := json.Unmarshal(content, &c.TruncatedValue); err != nil {
 		if err := json.Unmarshal(content, &c); err != nil {
 			return nil, err
 		}
@@ -32,6 +63,12 @@ func newCodeHover(content json.RawMessage) (*codeHover, error) {
 		c.setTokens()
 	}
 
+	c.Truncated = c.TruncatedValue.Truncated
+
+	if len(c.Tokens) > 0 {
+		c.TruncatedValue = nil // remove value for hovers which have tokens
+	}
+
 	return &c, nil
 }
 
@@ -41,7 +78,7 @@ func (c *codeHover) setTokens() {
 		return
 	}
 
-	iterator, err := lexer.Tokenise(nil, c.Value)
+	iterator, err := lexer.Tokenise(nil, c.TruncatedValue.Value)
 	if err != nil {
 		return
 	}
@@ -76,7 +113,6 @@ func (c *codeHover) setTokens() {
 	}
 
 	c.Tokens = tokenLines
-	c.Value = ""
 }
 
 func (c *codeHover) classFor(tokenType chroma.TokenType) string {

internal/lsif_transformer/parser/code_hover_test.go

@@ -3,6 +3,7 @@ package parser
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -77,5 +78,29 @@ func TestMarkdown(t *testing.T) {
 	c, err := newCodeHover(json.RawMessage(value))
 
 	require.NoError(t, err)
-	require.Equal(t, "This method reverses a string \n\n", c.Value)
+	require.Equal(t, "This method reverses a string \n\n", c.TruncatedValue.Value)
+}
+
+func TestTruncatedValue(t *testing.T) {
+	value := strings.Repeat("a", 500)
+	rawValue, err := json.Marshal(value)
+	require.NoError(t, err)
+
+	c, err := newCodeHover(rawValue)
+	require.NoError(t, err)
+
+	require.Equal(t, value[0:maxValueSize], c.TruncatedValue.Value)
+	require.True(t, c.TruncatedValue.Truncated)
+}
+
+func TestTruncatingMultiByteChars(t *testing.T) {
+	value := strings.Repeat("ಅ", 500)
+	rawValue, err := json.Marshal(value)
+	require.NoError(t, err)
+
+	c, err := newCodeHover(rawValue)
+	require.NoError(t, err)
+
+	symbolSize := 3
+	require.Equal(t, value[0:maxValueSize*symbolSize], c.TruncatedValue.Value)
 }

internal/lsif_transformer/parser/performance_test.go

+package parser
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func BenchmarkGenerate(b *testing.B) {
+	filePath := "testdata/workhorse.lsif.zip"
+	tmpDir := filePath + ".tmp"
+	defer os.RemoveAll(tmpDir)
+
+	m := measureMemory(func() {
+		file, err := os.Open(filePath)
+		require.NoError(b, err)
+
+		p, err := NewParser(file, "")
+		require.NoError(b, err)
+
+		_, err = p.ZipReader()
+		require.NoError(b, err)
+		require.NoError(b, p.Close())
+	})
+
+	// Golang 1.13 has `func (*B) ReportMetric`
+	// It makes sense to replace fmt.Printf with
+	// b.ReportMetric(m, "MiB/op")
+
+	fmt.Printf("BenchmarkGenerate: %f MiB/op\n", m)
+}
+
+func measureMemory(f func()) float64 {
+	var m, m1 runtime.MemStats
+	runtime.ReadMemStats(&m)
+
+	f()
+
+	runtime.ReadMemStats(&m1)
+
+	return float64(m1.Alloc-m.Alloc) / 1024 / 1024
+}