Merge branch '338483-npm-presenter-eager-load' into 'master'

Remove a n+1 situation in the NPM presenter

See merge request gitlab-org/gitlab!68275

app/finders/packages/npm/package_finder.rb

@@ -12,11 +12,16 @@ module Packages
       end
 
       def execute
-        base.npm
-            .with_name(@package_name)
-            .installable
-            .last_of_each_version
-            .preload_files
+        results = base.npm
+                      .with_name(@package_name)
+                      .installable
+                      .last_of_each_version
+
+        unless Feature.enabled?(:npm_presenter_queries_tuning)
+          results = results.preload_files
+        end
+
+        results
       end
 
       private

app/models/packages/package.rb

 # frozen_string_literal: true
 class Packages::Package < ApplicationRecord
+  include EachBatch
   include Sortable
   include Gitlab::SQL::Pattern
   include UsageStatistics
@@ -104,6 +105,7 @@ class Packages::Package < ApplicationRecord
   scope :including_build_info, -> { includes(pipelines: :user) }
   scope :including_project_route, -> { includes(project: { namespace: :route }) }
   scope :including_tags, -> { includes(:tags) }
+  scope :including_dependency_links, -> { includes(dependency_links: :dependency) }
 
   scope :with_conan_channel, ->(package_channel) do
     joins(:conan_metadatum).where(packages_conan_metadata: { package_channel: package_channel })

app/presenters/packages/npm/package_presenter.rb

@@ -7,14 +7,26 @@ module Packages
 
       attr_reader :name, :packages
 
-      NPM_VALID_DEPENDENCY_TYPES = %i[dependencies devDependencies bundleDependencies peerDependencies].freeze
-
       def initialize(name, packages)
         @name = name
         @packages = packages
       end
 
       def versions
+        if queries_tuning?
+          new_versions
+        else
+          legacy_versions
+        end
+      end
+
+      def dist_tags
+        build_package_tags.tap { |t| t["latest"] ||= sorted_versions.last }
+      end
+
+      private
+
+      def legacy_versions
         package_versions = {}
 
         packages.each do |package|
@@ -28,11 +40,23 @@ module Packages
         package_versions
       end
 
-      def dist_tags
-        build_package_tags.tap { |t| t["latest"] ||= sorted_versions.last }
-      end
+      def new_versions
+        package_versions = {}
 
-      private
+        packages.each_batch do |relation|
+          relation.including_dependency_links
+                  .preload_files
+                  .each do |package|
+                    package_file = package.package_files.last
+
+                    next unless package_file
+
+                    package_versions[package.version] = build_package_version(package, package_file)
+                  end
+        end
+
+        package_versions
+      end
 
       def build_package_tags
         package_tags.to_h { |tag| [tag.name, tag.package.version] }
@@ -59,26 +83,38 @@ module Packages
 
       def build_package_dependencies(package)
         dependencies = Hash.new { |h, key| h[key] = {} }
-        dependency_links = package.dependency_links
-                                  .with_dependency_type(NPM_VALID_DEPENDENCY_TYPES)
-                                  .includes_dependency
 
-        dependency_links.find_each do |dependency_link|
-          dependency = dependency_link.dependency
-          dependencies[dependency_link.dependency_type][dependency.name] = dependency.version_pattern
+        if queries_tuning?
+          package.dependency_links.each do |dependency_link|
+            dependency = dependency_link.dependency
+            dependencies[dependency_link.dependency_type][dependency.name] = dependency.version_pattern
+          end
+        else
+          dependency_links = package.dependency_links
+                                    .with_dependency_type(%i[dependencies devDependencies bundleDependencies peerDependencies])
+                                    .includes_dependency
+
+          dependency_links.find_each do |dependency_link|
+            dependency = dependency_link.dependency
+            dependencies[dependency_link.dependency_type][dependency.name] = dependency.version_pattern
+          end
         end
 
         dependencies
       end
 
       def sorted_versions
-        versions = packages.map(&:version).compact
+        versions = packages.pluck_versions.compact
         VersionSorter.sort(versions)
       end
 
       def package_tags
         Packages::Tag.for_packages(packages)
-                    .preload_package
+                     .preload_package
+      end
+
+      def queries_tuning?
+        Feature.enabled?(:npm_presenter_queries_tuning)
       end
     end
   end

config/feature_flags/development/npm_presenter_queries_tuning.yml

+---
+name: npm_presenter_queries_tuning
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68275
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/338603
+milestone: '14.2'
+type: development
+group: group::package
+default_enabled: false

db/migrate/20210817130415_add_project_id_name_version_id_to_npm_packages.rb

+# frozen_string_literal: true
+
+class AddProjectIdNameVersionIdToNpmPackages < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'idx_installable_npm_pkgs_on_project_id_name_version_id'
+
+  def up
+    add_concurrent_index :packages_packages, [:project_id, :name, :version, :id], where: 'package_type = 2 AND status = 0', name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index :packages_packages, [:project_id, :name, :version, :id], where: 'package_type = 2 AND status = 0', name: INDEX_NAME
+  end
+end

db/schema_migrations/20210817130415

+8c1ec0dfc043861377786bd7731a1a1f994d6f03833f4dcc2ba94ab1ddc83acf
\ No newline at end of file

db/structure.sql

@@ -23024,6 +23024,8 @@ CREATE UNIQUE INDEX idx_environment_merge_requests_unique_index ON deployment_me
 
 CREATE INDEX idx_geo_con_rep_updated_events_on_container_repository_id ON geo_container_repository_updated_events USING btree (container_repository_id);
 
+CREATE INDEX idx_installable_npm_pkgs_on_project_id_name_version_id ON packages_packages USING btree (project_id, name, version, id) WHERE ((package_type = 2) AND (status = 0));
+
 CREATE INDEX idx_issues_on_health_status_not_null ON issues USING btree (health_status) WHERE (health_status IS NOT NULL);
 
 CREATE INDEX idx_issues_on_project_id_and_created_at_and_id_and_state_id ON issues USING btree (project_id, created_at, id, state_id);

spec/factories/packages.rb

@@ -112,7 +112,7 @@ FactoryBot.define do
 
     factory :npm_package do
       sequence(:name) { |n| "@#{project.root_namespace.path}/package-#{n}"}
-      version { '1.0.0' }
+      sequence(:version) { |n| "1.0.#{n}" }
       package_type { :npm }
 
       after :create do |package|

spec/finders/packages/npm/package_finder_spec.rb

@@ -68,6 +68,20 @@ RSpec.describe ::Packages::Npm::PackageFinder do
 
         it { is_expected.to be_empty }
       end
+
+      context 'with npm_presenter_queries_tuning disabled' do
+        before do
+          stub_feature_flags(npm_presenter_queries_tuning: false)
+        end
+
+        it_behaves_like 'finding packages by name'
+
+        context 'set to nil' do
+          let(:project) { nil }
+
+          it { is_expected.to be_empty }
+        end
+      end
     end
 
     context 'with a namespace' do
@@ -80,6 +94,20 @@ RSpec.describe ::Packages::Npm::PackageFinder do
 
         it { is_expected.to be_empty }
       end
+
+      context 'with npm_presenter_queries_tuning disabled' do
+        before do
+          stub_feature_flags(npm_presenter_queries_tuning: false)
+        end
+
+        it_behaves_like 'accepting a namespace for', 'finding packages by name'
+
+        context 'set to nil' do
+          let_it_be(:namespace) { nil }
+
+          it { is_expected.to be_empty }
+        end
+      end
     end
   end
 
@@ -109,6 +137,24 @@ RSpec.describe ::Packages::Npm::PackageFinder do
 
       it_behaves_like 'accepting a namespace for', 'finding packages by version'
     end
+
+    context 'with npm_presenter_queries_tuning disabled' do
+      before do
+        stub_feature_flags(npm_presenter_queries_tuning: false)
+      end
+
+      context 'with a project' do
+        let(:finder) { described_class.new(package_name, project: project) }
+
+        it_behaves_like 'finding packages by version'
+      end
+
+      context 'with a namespace' do
+        let(:finder) { described_class.new(package_name, namespace: namespace) }
+
+        it_behaves_like 'accepting a namespace for', 'finding packages by version'
+      end
+    end
   end
 
   describe '#last' do
@@ -118,31 +164,43 @@ RSpec.describe ::Packages::Npm::PackageFinder do
       it { is_expected.to eq(package) }
     end
 
-    context 'with a project' do
-      let(:finder) { described_class.new(package_name, project: project) }
+    shared_examples 'handling project or namespace parameter' do
+      context 'with a project' do
+        let(:finder) { described_class.new(package_name, project: project) }
 
-      it_behaves_like 'finding package by last'
-    end
+        it_behaves_like 'finding package by last'
+      end
 
-    context 'with a namespace' do
-      let(:finder) { described_class.new(package_name, namespace: namespace) }
+      context 'with a namespace' do
+        let(:finder) { described_class.new(package_name, namespace: namespace) }
 
-      it_behaves_like 'accepting a namespace for', 'finding package by last'
+        it_behaves_like 'accepting a namespace for', 'finding package by last'
 
-      context 'with duplicate packages' do
-        let_it_be(:namespace) { create(:group) }
-        let_it_be(:subgroup1) { create(:group, parent: namespace) }
-        let_it_be(:subgroup2) { create(:group, parent: namespace) }
-        let_it_be(:project2) { create(:project, namespace: subgroup2) }
-        let_it_be(:package2) { create(:npm_package, name: package.name, project: project2) }
+        context 'with duplicate packages' do
+          let_it_be(:namespace) { create(:group) }
+          let_it_be(:subgroup1) { create(:group, parent: namespace) }
+          let_it_be(:subgroup2) { create(:group, parent: namespace) }
+          let_it_be(:project2) { create(:project, namespace: subgroup2) }
+          let_it_be(:package2) { create(:npm_package, name: package.name, project: project2) }
 
-        before do
-          project.update!(namespace: subgroup1)
+          before do
+            project.update!(namespace: subgroup1)
+          end
+
+          # the most recent one is returned
+          it { is_expected.to eq(package2) }
         end
+      end
+    end
 
-        # the most recent one is returned
-        it { is_expected.to eq(package2) }
+    it_behaves_like 'handling project or namespace parameter'
+
+    context 'with npm_presenter_queries_tuning disabled' do
+      before do
+        stub_feature_flags(npm_presenter_queries_tuning: false)
       end
+
+      it_behaves_like 'handling project or namespace parameter'
     end
   end
 end

spec/presenters/packages/npm/package_presenter_spec.rb

@@ -5,62 +5,131 @@ require 'spec_helper'
 RSpec.describe ::Packages::Npm::PackagePresenter do
   let_it_be(:project) { create(:project) }
   let_it_be(:package_name) { "@#{project.root_namespace.path}/test" }
+  let_it_be(:package1) { create(:npm_package, version: '2.0.4', project: project, name: package_name) }
+  let_it_be(:package2) { create(:npm_package, version: '2.0.6', project: project, name: package_name) }
+  let_it_be(:latest_package) { create(:npm_package, version: '2.0.11', project: project, name: package_name) }
 
-  let!(:package1) { create(:npm_package, version: '1.0.4', project: project, name: package_name) }
-  let!(:package2) { create(:npm_package, version: '1.0.6', project: project, name: package_name) }
-  let!(:latest_package) { create(:npm_package, version: '1.0.11', project: project, name: package_name) }
   let(:packages) { project.packages.npm.with_name(package_name).last_of_each_version }
   let(:presenter) { described_class.new(package_name, packages) }
 
   describe '#versions' do
     subject { presenter.versions }
 
-    context 'for packages without dependencies' do
-      it { is_expected.to be_a(Hash) }
-      it { expect(subject[package1.version].with_indifferent_access).to match_schema('public_api/v4/packages/npm_package_version') }
-      it { expect(subject[package2.version].with_indifferent_access).to match_schema('public_api/v4/packages/npm_package_version') }
+    shared_examples 'returning packages versions' do |expect_n_plus_one: false|
+      context 'for packages without dependencies' do
+        it { is_expected.to be_a(Hash) }
+        it { expect(subject[package1.version].with_indifferent_access).to match_schema('public_api/v4/packages/npm_package_version') }
+        it { expect(subject[package2.version].with_indifferent_access).to match_schema('public_api/v4/packages/npm_package_version') }
 
-      described_class::NPM_VALID_DEPENDENCY_TYPES.each do |dependency_type|
-        it { expect(subject.dig(package1.version, dependency_type)).to be nil }
-        it { expect(subject.dig(package2.version, dependency_type)).to be nil }
+        ::Packages::DependencyLink.dependency_types.keys.each do |dependency_type|
+          it { expect(subject.dig(package1.version, dependency_type)).to be nil }
+          it { expect(subject.dig(package2.version, dependency_type)).to be nil }
+        end
+
+        it 'avoids N+1 database queries' do
+          check_n_plus_one(:versions, expect_it: expect_n_plus_one) do
+            create_list(:npm_package, 5, project: project, name: package_name)
+          end
+        end
       end
-    end
 
-    context 'for packages with dependencies' do
-      described_class::NPM_VALID_DEPENDENCY_TYPES.each do |dependency_type|
-        let!("package_dependency_link_for_#{dependency_type}") { create(:packages_dependency_link, package: package1, dependency_type: dependency_type) }
+      context 'for packages with dependencies' do
+        ::Packages::DependencyLink.dependency_types.keys.each do |dependency_type|
+          let_it_be("package_dependency_link_for_#{dependency_type}") { create(:packages_dependency_link, package: package1, dependency_type: dependency_type) }
+        end
+
+        it { is_expected.to be_a(Hash) }
+        it { expect(subject[package1.version].with_indifferent_access).to match_schema('public_api/v4/packages/npm_package_version') }
+        it { expect(subject[package2.version].with_indifferent_access).to match_schema('public_api/v4/packages/npm_package_version') }
+        ::Packages::DependencyLink.dependency_types.keys.each do |dependency_type|
+          it { expect(subject.dig(package1.version, dependency_type.to_s)).to be_any }
+        end
+
+        it 'avoids N+1 database queries' do
+          check_n_plus_one(:versions, expect_it: expect_n_plus_one) do
+            create_list(:npm_package, 5, project: project, name: package_name).each do |npm_package|
+              ::Packages::DependencyLink.dependency_types.keys.each do |dependency_type|
+                create(:packages_dependency_link, package: npm_package, dependency_type: dependency_type)
+              end
+            end
+          end
+        end
       end
+    end
+
+    it_behaves_like 'returning packages versions'
 
-      it { is_expected.to be_a(Hash) }
-      it { expect(subject[package1.version].with_indifferent_access).to match_schema('public_api/v4/packages/npm_package_version') }
-      it { expect(subject[package2.version].with_indifferent_access).to match_schema('public_api/v4/packages/npm_package_version') }
-      described_class::NPM_VALID_DEPENDENCY_TYPES.each do |dependency_type|
-        it { expect(subject.dig(package1.version, dependency_type.to_s)).to be_any }
+    context 'with npm_presenter_queries_tuning disabled' do
+      before do
+        stub_feature_flags(npm_presenter_queries_tuning: false)
       end
+
+      it_behaves_like 'returning packages versions', expect_n_plus_one: true
     end
   end
 
   describe '#dist_tags' do
     subject { presenter.dist_tags }
 
-    context 'for packages without tags' do
-      it { is_expected.to be_a(Hash) }
-      it { expect(subject["latest"]).to eq(latest_package.version) }
+    shared_examples 'returning packages tags' do
+      context 'for packages without tags' do
+        it { is_expected.to be_a(Hash) }
+        it { expect(subject["latest"]).to eq(latest_package.version) }
+
+        it 'avoids N+1 database queries' do
+          check_n_plus_one(:dist_tags) do
+            create_list(:npm_package, 5, project: project, name: package_name)
+          end
+        end
+      end
+
+      context 'for packages with tags' do
+        let_it_be(:package_tag1) { create(:packages_tag, package: package1, name: 'release_a') }
+        let_it_be(:package_tag2) { create(:packages_tag, package: package1, name: 'test_release') }
+        let_it_be(:package_tag3) { create(:packages_tag, package: package2, name: 'release_b') }
+        let_it_be(:package_tag4) { create(:packages_tag, package: latest_package, name: 'release_c') }
+        let_it_be(:package_tag5) { create(:packages_tag, package: latest_package, name: 'latest') }
+
+        it { is_expected.to be_a(Hash) }
+        it { expect(subject[package_tag1.name]).to eq(package1.version) }
+        it { expect(subject[package_tag2.name]).to eq(package1.version) }
+        it { expect(subject[package_tag3.name]).to eq(package2.version) }
+        it { expect(subject[package_tag4.name]).to eq(latest_package.version) }
+        it { expect(subject[package_tag5.name]).to eq(latest_package.version) }
+
+        it 'avoids N+1 database queries' do
+          check_n_plus_one(:dist_tags) do
+            create_list(:npm_package, 5, project: project, name: package_name).each_with_index do |npm_package, index|
+              create(:packages_tag, package: npm_package, name: "tag_#{index}")
+            end
+          end
+        end
+      end
     end
 
-    context 'for packages with tags' do
-      let!(:package_tag1) { create(:packages_tag, package: package1, name: 'release_a') }
-      let!(:package_tag2) { create(:packages_tag, package: package1, name: 'test_release') }
-      let!(:package_tag3) { create(:packages_tag, package: package2, name: 'release_b') }
-      let!(:package_tag4) { create(:packages_tag, package: latest_package, name: 'release_c') }
-      let!(:package_tag5) { create(:packages_tag, package: latest_package, name: 'latest') }
-
-      it { is_expected.to be_a(Hash) }
-      it { expect(subject[package_tag1.name]).to eq(package1.version) }
-      it { expect(subject[package_tag2.name]).to eq(package1.version) }
-      it { expect(subject[package_tag3.name]).to eq(package2.version) }
-      it { expect(subject[package_tag4.name]).to eq(latest_package.version) }
-      it { expect(subject[package_tag5.name]).to eq(latest_package.version) }
+    it_behaves_like 'returning packages tags'
+
+    context 'with npm_presenter_queries_tuning disabled' do
+      before do
+        stub_feature_flags(npm_presenter_queries_tuning: false)
+      end
+
+      it_behaves_like 'returning packages tags'
+    end
+  end
+
+  def check_n_plus_one(field, expect_it: false)
+    pkgs = project.packages.npm.with_name(package_name).last_of_each_version.preload_files
+    control = ActiveRecord::QueryRecorder.new { described_class.new(package_name, pkgs).public_send(field) }
+
+    yield
+
+    pkgs = project.packages.npm.with_name(package_name).last_of_each_version.preload_files
+
+    if expect_it
+      expect { described_class.new(package_name, pkgs).public_send(field) }.to exceed_query_limit(control)
+    else
+      expect { described_class.new(package_name, pkgs).public_send(field) }.not_to exceed_query_limit(control)
     end
   end
 end

spec/support/shared_examples/requests/api/npm_packages_shared_examples.rb

@@ -21,11 +21,24 @@ RSpec.shared_examples 'handling get metadata requests' do |scope: :project|
       expect(response).to match_response_schema('public_api/v4/packages/npm_package')
       expect(json_response['name']).to eq(package.name)
       expect(json_response['versions'][package.version]).to match_schema('public_api/v4/packages/npm_package_version')
-      ::Packages::Npm::PackagePresenter::NPM_VALID_DEPENDENCY_TYPES.each do |dependency_type|
+      ::Packages::DependencyLink.dependency_types.keys.each do |dependency_type|
         expect(json_response.dig('versions', package.version, dependency_type.to_s)).to be_any
       end
       expect(json_response['dist-tags']).to match_schema('public_api/v4/packages/npm_package_tags')
     end
+
+    it 'avoids N+1 database queries' do
+      control = ActiveRecord::QueryRecorder.new { get(url, headers: headers) }
+
+      create_list(:npm_package, 5, project: project, name: package_name).each do |npm_package|
+        ::Packages::DependencyLink.dependency_types.keys.each do |dependency_type|
+          create(:packages_dependency_link, package: package, dependency_type: dependency_type)
+        end
+      end
+
+      # query count can slightly change between the examples so we're using a custom threshold
+      expect { get(url, headers: headers) }.not_to exceed_query_limit(control).with_threshold(4)
+    end
   end
 
   shared_examples 'reject metadata request' do |status:|