Merge branch 'id-lsif-move-ranges-hash-into-tmp-file' into 'master'

Cache ranges value in a file instead of a hash

See merge request gitlab-org/gitlab-workhorse!500

internal/lsif_transformer/parser/cache.go

+package parser
+
+import (
+	"encoding/binary"
+	"io"
+	"io/ioutil"
+	"os"
+)
+
+// This cache implementation is using a temp file to provide key-value data storage
+// It allows to avoid storing intermediate calculations in RAM
+// The stored data must be a fixed-size value or a slice of fixed-size values, or a pointer to such data
+type cache struct {
+	file      *os.File
+	chunkSize int64
+}
+
+func newCache(tempDir, filename string, data interface{}) (*cache, error) {
+	f, err := ioutil.TempFile(tempDir, filename)
+	if err != nil {
+		return nil, err
+	}
+
+	return &cache{file: f, chunkSize: int64(binary.Size(data))}, nil
+}
+
+func (c *cache) SetEntry(id Id, data interface{}) error {
+	if err := c.setOffset(id); err != nil {
+		return err
+	}
+
+	return binary.Write(c.file, binary.LittleEndian, data)
+}
+
+func (c *cache) Entry(id Id, data interface{}) error {
+	if err := c.setOffset(id); err != nil {
+		return err
+	}
+
+	return binary.Read(c.file, binary.LittleEndian, data)
+}
+
+func (c *cache) Close() error {
+	if err := c.file.Close(); err != nil {
+		return err
+	}
+
+	return os.Remove(c.file.Name())
+}
+
+func (c *cache) setOffset(id Id) error {
+	offset := int64(id) * c.chunkSize
+	_, err := c.file.Seek(offset, io.SeekStart)
+
+	return err
+}

internal/lsif_transformer/parser/cache_test.go

+package parser
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type chunk struct {
+	A int16
+	B int16
+}
+
+func TestCache(t *testing.T) {
+	cache, err := newCache("", "test-chunks", chunk{})
+	require.NoError(t, err)
+	defer cache.Close()
+
+	c := chunk{A: 1, B: 2}
+	require.NoError(t, cache.SetEntry(1, &c))
+	require.NoError(t, cache.setOffset(0))
+
+	content, err := ioutil.ReadAll(cache.file)
+	require.NoError(t, err)
+
+	expected := []byte{0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x2, 0x0}
+	require.Equal(t, expected, content)
+
+	var nc chunk
+	require.NoError(t, cache.Entry(1, &nc))
+	require.Equal(t, c, nc)
+}

internal/lsif_transformer/parser/ranges.go

@@ -6,13 +6,15 @@ import (
 	"strconv"
 )
 
-const Definitions = "definitions"
-const References = "references"
+const (
+	definitions = "definitions"
+	references  = "references"
+)
 
 type Ranges struct {
-	Entries map[Id]*Range
 	DefRefs map[Id]*DefRef
 	Hovers  *Hovers
+	Cache   *cache
 }
 
 type RawRange struct {
@@ -51,10 +53,15 @@ func NewRanges(tempDir string) (*Ranges, error) {
 		return nil, err
 	}
 
+	cache, err := newCache(tempDir, "ranges", Range{})
+	if err != nil {
+		return nil, err
+	}
+
 	return &Ranges{
-		Entries: make(map[Id]*Range),
 		DefRefs: make(map[Id]*DefRef),
 		Hovers:  hovers,
+		Cache:   cache,
 	}, nil
 }
 
@@ -84,7 +91,11 @@ func (r *Ranges) Serialize(f io.Writer, rangeIds []Id, docs map[Id]string) error
 	}
 
 	for i, rangeId := range rangeIds {
-		entry := r.Entries[rangeId]
+		entry, err := r.getRange(rangeId)
+		if err != nil {
+			continue
+		}
+
 		serializedRange := SerializedRange{
 			StartLine:      entry.Line,
 			StartChar:      entry.Character,
@@ -109,6 +120,10 @@ func (r *Ranges) Serialize(f io.Writer, rangeIds []Id, docs map[Id]string) error
 }
 
 func (r *Ranges) Close() error {
+	if err := r.Cache.Close(); err != nil {
+		return err
+	}
+
 	return r.Hovers.Close()
 }
 
@@ -129,9 +144,7 @@ func (r *Ranges) addRange(line []byte) error {
 		return err
 	}
 
-	r.Entries[rg.Id] = &rg.Data
-
-	return nil
+	return r.Cache.SetEntry(rg.Id, &rg.Data)
 }
 
 func (r *Ranges) addItem(line []byte) error {
@@ -140,26 +153,49 @@ func (r *Ranges) addItem(line []byte) error {
 		return err
 	}
 
-	if defRef.Property != Definitions && defRef.Property != References {
+	if defRef.Property != definitions && defRef.Property != references {
 		return nil
 	}
 
 	for _, rangeId := range defRef.RangeIds {
-		if entry, ok := r.Entries[rangeId]; ok {
-			entry.RefId = defRef.RefId
+		var rg Range
+		if err := r.Cache.Entry(rangeId, &rg); err != nil {
+			return err
+		}
+
+		rg.RefId = defRef.RefId
+
+		if err := r.Cache.SetEntry(rangeId, &rg); err != nil {
+			return err
 		}
 	}
 
-	if defRef.Property != Definitions {
-		return nil
+	if defRef.Property == definitions {
+		return r.addDefRef(&defRef)
 	}
 
-	defRange := r.Entries[defRef.RangeIds[0]]
+	return nil
+}
+
+func (r *Ranges) addDefRef(defRef *RawDefRef) error {
+	var rg Range
+	if err := r.Cache.Entry(defRef.RangeIds[0], &rg); err != nil {
+		return err
+	}
 
 	r.DefRefs[defRef.RefId] = &DefRef{
-		Line:  strconv.Itoa(int(defRange.Line + 1)),
+		Line:  strconv.Itoa(int(rg.Line + 1)),
 		DocId: defRef.DocId,
 	}
 
 	return nil
 }
+
+func (r *Ranges) getRange(rangeId Id) (*Range, error) {
+	var rg Range
+	if err := r.Cache.Entry(rangeId, &rg); err != nil {
+		return nil, err
+	}
+
+	return &rg, nil
+}

internal/lsif_transformer/parser/ranges_test.go

@@ -12,10 +12,14 @@ func TestRangesRead(t *testing.T) {
 	defer cleanup()
 
 	firstRange := Range{Line: 1, Character: 2, RefId: 3}
-	require.Equal(t, &firstRange, r.Entries[1])
+	rg, err := r.getRange(1)
+	require.NoError(t, err)
+	require.Equal(t, &firstRange, rg)
 
 	secondRange := Range{Line: 5, Character: 4, RefId: 3}
-	require.Equal(t, &secondRange, r.Entries[2])
+	rg, err = r.getRange(2)
+	require.NoError(t, err)
+	require.Equal(t, &secondRange, rg)
 }
 
 func TestSerialize(t *testing.T) {