LSIF: Support markdown-like content format

Hover contents value can be either a list of objects:

[{ "value": "func main()", "language": "go" }]

Or a list of strings with documentation:

["String of documentation"]

Or a markdown object:

{ "value": "```go\nfunc main()\n```", "kind": "markdown" }

This commit introduces support for the markdown object.

The content in the markdown object still won't be highlighted,
but at least it won't raise an error

workhorse/internal/lsif_transformer/parser/code_hover.go

@@ -50,9 +50,29 @@ func (ts *truncatableString) MarshalJSON() ([]byte, error) {
 	return json.Marshal(ts.Value)
 }
 
+func newCodeHovers(contents json.RawMessage) ([]*codeHover, error) {
+	var rawContents []json.RawMessage
+	if err := json.Unmarshal(contents, &rawContents); err != nil {
+		rawContents = []json.RawMessage{contents}
+	}
+
+	codeHovers := []*codeHover{}
+	for _, rawContent := range rawContents {
+		c, err := newCodeHover(rawContent)
+		if err != nil {
+			return nil, err
+		}
+
+		codeHovers = append(codeHovers, c)
+	}
+
+	return codeHovers, nil
+}
+
 func newCodeHover(content json.RawMessage) (*codeHover, error) {
 	// Hover value can be either an object: { "value": "func main()", "language": "go" }
 	// Or a string with documentation
+	// Or a markdown object: { "value": "```go\nfunc main()\n```", "kind": "markdown" }
 	// We try to unmarshal the content into a string and if we fail, we unmarshal it into an object
 	var c codeHover
 	if err := json.Unmarshal(content, &c.TruncatedValue); err != nil {

workhorse/internal/lsif_transformer/parser/code_hover_test.go

@@ -64,21 +64,33 @@ func TestHighlight(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			raw := []byte(fmt.Sprintf(`{"language":"%s","value":"%s"}`, tt.language, tt.value))
-			c, err := newCodeHover(json.RawMessage(raw))
+			raw := []byte(fmt.Sprintf(`[{"language":"%s","value":"%s"}]`, tt.language, tt.value))
+			c, err := newCodeHovers(json.RawMessage(raw))
 
 			require.NoError(t, err)
-			require.Equal(t, tt.want, c.Tokens)
+			require.Len(t, c, 1)
+			require.Equal(t, tt.want, c[0].Tokens)
 		})
 	}
 }
 
 func TestMarkdown(t *testing.T) {
-	value := `"This method reverses a string \n\n"`
-	c, err := newCodeHover(json.RawMessage(value))
+	value := `["This method reverses a string \n\n"]`
+	c, err := newCodeHovers(json.RawMessage(value))
 
 	require.NoError(t, err)
-	require.Equal(t, "This method reverses a string \n\n", c.TruncatedValue.Value)
+	require.Len(t, c, 1)
+	require.Equal(t, "This method reverses a string \n\n", c[0].TruncatedValue.Value)
+}
+
+func TestMarkdownContentsFormat(t *testing.T) {
+	value := `{"kind":"markdown","value":"some _markdown_ **text**"}`
+	c, err := newCodeHovers(json.RawMessage(value))
+
+	require.NoError(t, err)
+	require.Len(t, c, 1)
+	require.Equal(t, [][]token(nil), c[0].Tokens)
+	require.Equal(t, "some _markdown_ **text**", c[0].TruncatedValue.Value)
 }
 
 func TestTruncatedValue(t *testing.T) {

workhorse/internal/lsif_transformer/parser/hovers.go

@@ -18,7 +18,7 @@ type Hovers struct {
 }
 
 type RawResult struct {
-	Contents []json.RawMessage `json:"contents"`
+	Contents json.RawMessage `json:"contents"`
 }
 
 type RawData struct {
@@ -107,14 +107,9 @@ func (h *Hovers) addData(line []byte) error {
 		return err
 	}
 
-	codeHovers := []*codeHover{}
-	for _, rawContent := range rawData.Result.Contents {
-		c, err := newCodeHover(rawContent)
-		if err != nil {
-			return err
-		}
-
-		codeHovers = append(codeHovers, c)
+	codeHovers, err := newCodeHovers(rawData.Result.Contents)
+	if err != nil {
+		return err
 	}
 
 	codeHoversData, err := json.Marshal(codeHovers)