Merge branch 'id-fetch-lsif-ranges' into 'master'

Code Nav PoC: API for fetching code-nav info

See merge request gitlab-org/gitlab!23674

app/models/ci/build.rb

@@ -23,6 +23,7 @@ module Ci
     belongs_to :trigger_request
     belongs_to :erased_by, class_name: 'User'
     belongs_to :resource_group, class_name: 'Ci::ResourceGroup', inverse_of: :builds
+    belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id
 
     RUNNER_FEATURES = {
       upload_multiple_artifacts: -> (build) { build.publishes_artifacts_reports? },

app/models/ci/job_artifact.rb

@@ -28,7 +28,7 @@ module Ci
       license_scanning: 'gl-license-scanning-report.json',
       performance: 'performance.json',
       metrics: 'metrics.txt',
-      lsif: 'lsif.sqlite3'
+      lsif: 'lsif.json'
     }.freeze
 
     INTERNAL_TYPES = {
@@ -74,6 +74,7 @@ module Ci
 
     scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) }
     scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) }
+    scope :for_sha, ->(sha) { joins(job: :pipeline).where(ci_pipelines: { sha: sha }) }
 
     scope :with_file_types, -> (file_types) do
       types = self.file_types.select { |file_type| file_types.include?(file_type) }.values
@@ -117,7 +118,7 @@ module Ci
       metrics: 12, ## EE-specific
       metrics_referee: 13, ## runner referees
       network_referee: 14, ## runner referees
-      lsif: 15 # LSIF dump for code navigation
+      lsif: 15 # LSIF data for code navigation
     }
 
     enum file_format: {

app/services/projects/lsif_data_service.rb

+# frozen_string_literal: true
+
+module Projects
+  class LsifDataService
+    attr_reader :file, :project, :path, :commit_id
+
+    CACHE_EXPIRE_IN = 1.hour
+
+    def initialize(file, project, params)
+      @file = file
+      @project = project
+      @path = params[:path]
+      @commit_id = params[:commit_id]
+    end
+
+    def execute
+      docs, doc_ranges, ranges =
+        fetch_data.values_at('docs', 'doc_ranges', 'ranges')
+
+      doc_id = doc_id_from(docs)
+
+      doc_ranges[doc_id]&.map do |range_id|
+        line_data, column_data = ranges[range_id]['loc']
+
+        {
+          start_line: line_data.first,
+          end_line: line_data.last,
+          start_char: column_data.first,
+          end_char: column_data.last
+        }
+      end
+    end
+
+    private
+
+    def fetch_data
+      Rails.cache.fetch("project:#{project.id}:lsif:#{commit_id}", expires_in: CACHE_EXPIRE_IN) do
+        data = nil
+
+        file.open do |stream|
+          Zlib::GzipReader.wrap(stream) do |gz_stream|
+            data = JSON.parse(gz_stream.read)
+          end
+        end
+
+        data
+      end
+    end
+
+    def doc_id_from(docs)
+      docs.reduce(nil) do |doc_id, (id, doc_path)|
+        next doc_id unless doc_path =~ /#{path}$/
+
+        if doc_id.nil? || docs[doc_id].size > doc_path.size
+          doc_id = id
+        end
+
+        doc_id
+      end
+    end
+  end
+end

lib/api/api.rb

@@ -142,6 +142,7 @@ module API
       mount ::API::Keys
       mount ::API::Labels
       mount ::API::Lint
+      mount ::API::LsifData
       mount ::API::Markdown
       mount ::API::Members
       mount ::API::MergeRequestDiffs

lib/api/lsif_data.rb

+# frozen_string_literal: true
+
+module API
+  class LsifData < Grape::API
+    MAX_FILE_SIZE = 10.megabytes
+
+    before do
+      not_found! if Feature.disabled?(:code_navigation, user_project)
+    end
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+      requires :commit_id, type: String, desc: 'The ID of a commit'
+    end
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      segment ':id/commits/:commit_id' do
+        params do
+          requires :path, type: String, desc: 'The path of a file'
+        end
+        get 'lsif/info' do
+          authorize! :download_code, user_project
+
+          artifact =
+            @project.job_artifacts
+              .with_file_types(['lsif'])
+              .for_sha(params[:commit_id])
+              .last
+
+          not_found! unless artifact
+          authorize! :read_pipeline, artifact.job.pipeline
+          file_too_large! if artifact.file.cached_size > MAX_FILE_SIZE
+
+          ::Projects::LsifDataService.new(artifact.file, @project, params).execute
+        end
+      end
+    end
+  end
+end

spec/factories/ci/job_artifacts.rb

@@ -139,6 +139,16 @@ FactoryBot.define do
       end
     end
 
+    trait :lsif do
+      file_type { :lsif }
+      file_format { :raw }
+
+      after(:build) do |artifact, evaluator|
+        artifact.file = fixture_file_upload(
+          Rails.root.join('spec/fixtures/lsif.json.gz'), 'application/octet-stream')
+      end
+    end
+
     trait :correct_checksum do
       after(:build) do |artifact, evaluator|
         artifact.file_sha256 = Digest::SHA256.file(artifact.file.path).hexdigest

spec/lib/gitlab/ci/config/entry/reports_spec.rb

@@ -43,7 +43,7 @@ describe Gitlab::Ci::Config::Entry::Reports do
         :license_management | 'gl-license-management-report.json'
         :license_scanning | 'gl-license-scanning-report.json'
         :performance | 'performance.json'
-        :lsif | 'lsif.sqlite3'
+        :lsif | 'lsif.json'
       end
 
       with_them do

spec/models/ci/job_artifact_spec.rb

@@ -111,6 +111,18 @@ describe Ci::JobArtifact do
     end
   end
 
+  describe '.for_sha' do
+    it 'returns job artifacts for a given pipeline sha' do
+      first_pipeline = create(:ci_pipeline)
+      second_pipeline = create(:ci_pipeline, sha: Digest::SHA1.hexdigest(SecureRandom.hex))
+      first_artifact = create(:ci_job_artifact, job: create(:ci_build, pipeline: first_pipeline))
+      second_artifact = create(:ci_job_artifact, job: create(:ci_build, pipeline: second_pipeline))
+
+      expect(described_class.for_sha(first_pipeline.sha)).to eq([first_artifact])
+      expect(described_class.for_sha(second_pipeline.sha)).to eq([second_artifact])
+    end
+  end
+
   describe 'callbacks' do
     subject { create(:ci_job_artifact, :archive) }
 

spec/requests/api/lsif_data_spec.rb

+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe API::LsifData do
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project, :repository) }
+
+  let(:commit) { project.commit }
+
+  describe 'GET lsif/info' do
+    let(:endpoint_path) { "/projects/#{project.id}/commits/#{commit.id}/lsif/info" }
+
+    context 'user does not have access to the project' do
+      before do
+        project.add_guest(user)
+      end
+
+      it 'returns 403' do
+        get api(endpoint_path, user), params: { path: 'main.go' }
+
+        expect(response).to have_gitlab_http_status(:forbidden)
+      end
+    end
+
+    context 'user has access to the project' do
+      before do
+        project.add_reporter(user)
+      end
+
+      context 'code_navigation feature is disabled' do
+        before do
+          stub_feature_flags(code_navigation: false)
+        end
+
+        it 'returns 404' do
+          get api(endpoint_path, user)
+
+          expect(response).to have_gitlab_http_status(:not_found)
+        end
+      end
+
+      context 'there is no job artifact for the passed commit' do
+        it 'returns 404' do
+          get api(endpoint_path, user), params: { path: 'main.go' }
+
+          expect(response).to have_gitlab_http_status(:not_found)
+        end
+      end
+
+      context 'lsif data is stored as a job artifact' do
+        let!(:pipeline) { create(:ci_pipeline, project: project, sha: commit.id) }
+        let!(:artifact) { create(:ci_job_artifact, :lsif, job: create(:ci_build, pipeline: pipeline)) }
+
+        it 'returns code navigation info for a given path' do
+          get api(endpoint_path, user), params: { path: 'main.go' }
+
+          expect(response).to have_gitlab_http_status(:ok)
+          expect(response.parsed_body.last).to eq({
+            'end_char' => 18,
+            'end_line' => 8,
+            'start_char' => 13,
+            'start_line' => 8
+          })
+        end
+
+        context 'the stored file is too large' do
+          it 'returns 413' do
+            allow_any_instance_of(JobArtifactUploader).to receive(:cached_size).and_return(20.megabytes)
+
+            get api(endpoint_path, user), params: { path: 'main.go' }
+
+            expect(response).to have_gitlab_http_status(:payload_too_large)
+          end
+        end
+
+        context 'the user does not have access to the pipeline' do
+          let(:project) { create(:project, :repository, builds_access_level: ProjectFeature::DISABLED) }
+
+          it 'returns 403' do
+            get api(endpoint_path, user), params: { path: 'main.go' }
+
+            expect(response).to have_gitlab_http_status(:forbidden)
+          end
+        end
+      end
+    end
+  end
+end

spec/services/projects/lsif_data_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Projects::LsifDataService do
+  let(:artifact) { create(:ci_job_artifact, :lsif) }
+  let(:project) { build_stubbed(:project) }
+  let(:path) { 'main.go' }
+  let(:commit_id) { Digest::SHA1.hexdigest(SecureRandom.hex) }
+  let(:params) { { path: path, commit_id: commit_id } }
+
+  let(:service) { described_class.new(artifact.file, project, params) }
+
+  describe '#execute' do
+    context 'fetched lsif file', :use_clean_rails_memory_store_caching do
+      it 'is cached' do
+        service.execute
+
+        cached_data = Rails.cache.fetch("project:#{project.id}:lsif:#{commit_id}")
+
+        expect(cached_data.keys).to eq(%w[def_refs doc_ranges docs hover_refs ranges])
+      end
+    end
+
+    context 'for main.go' do
+      it 'returns lsif ranges for the file' do
+        expect(service.execute).to eq([
+          {
+            end_char: 9,
+            end_line: 6,
+            start_char: 5,
+            start_line: 6
+          },
+          {
+            end_char: 36,
+            end_line: 3,
+            start_char: 1,
+            start_line: 3
+          },
+          {
+            end_char: 12,
+            end_line: 7,
+            start_char: 1,
+            start_line: 7
+          },
+          {
+            end_char: 20,
+            end_line: 7,
+            start_char: 13,
+            start_line: 7
+          },
+          {
+            end_char: 12,
+            end_line: 8,
+            start_char: 1,
+            start_line: 8
+          },
+          {
+            end_char: 18,
+            end_line: 8,
+            start_char: 13,
+            start_line: 8
+          }
+        ])
+      end
+    end
+
+    context 'for morestring/reverse.go' do
+      let(:path) { 'morestrings/reverse.go' }
+
+      it 'returns lsif ranges for the file' do
+        expect(service.execute.first).to eq({
+          end_char: 2,
+          end_line: 11,
+          start_char: 1,
+          start_line: 11
+        })
+      end
+    end
+
+    context 'for an unknown file' do
+      let(:path) { 'unknown.go' }
+
+      it 'returns nil' do
+        expect(service.execute).to eq(nil)
+      end
+    end
+  end
+
+  describe '#doc_id_from' do
+    context 'when the passed path matches multiple files' do
+      let(:path) { 'check/main.go' }
+      let(:docs) do
+        {
+          1 => 'cmd/check/main.go',
+          2 => 'cmd/command.go',
+          3 => 'check/main.go',
+          4 => 'cmd/nested/check/main.go'
+        }
+      end
+
+      it 'fetches the document with the shortest absolute path' do
+        expect(service.__send__(:doc_id_from, docs)).to eq(3)
+      end
+    end
+  end
+end