Merge branch 'eb-merge-rake-task-for-artifacts-size-recalculation' into 'master'

Make refresh task fetch IDs from CSV URL

See merge request gitlab-org/gitlab!83194

lib/tasks/ci/build_artifacts.rake

-# frozen_string_literal: true
-
-require 'httparty'
-require 'csv'
-
-namespace :ci do
-  namespace :build_artifacts do
-    desc "GitLab | CI | Fetch projects with incorrect artifact size on GitLab.com"
-    task :project_with_incorrect_artifact_size do
-      csv_url = ENV['SISENSE_PROJECT_IDS_WITH_INCORRECT_ARTIFACTS_URL']
-
-      # rubocop: disable Gitlab/HTTParty
-      body = HTTParty.get(csv_url)
-      # rubocop: enable Gitlab/HTTParty
-
-      table = CSV.parse(body.parsed_response, headers: true)
-      puts table['PROJECT_ID'].join(' ')
-    end
-  end
-end

lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake

 # frozen_string_literal: true
 
+require 'httparty'
+require 'csv'
+
 namespace :gitlab do
-  desc "GitLab | Refresh build artifacts size project statistics for given project IDs"
+  desc "GitLab | Refresh build artifacts size project statistics for given list of Project IDs from remote CSV"
 
   BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE = 500
 
-  task :refresh_project_statistics_build_artifacts_size, [:project_ids] => :environment do |_t, args|
-    project_ids = []
-    project_ids = $stdin.read.split unless $stdin.tty?
-    project_ids = args.project_ids.to_s.split unless project_ids.any?
+  task :refresh_project_statistics_build_artifacts_size, [:csv_url] => :environment do |_t, args|
+    csv_url = args.csv_url
+
+    # rubocop: disable Gitlab/HTTParty
+    body = HTTParty.get(csv_url)
+    # rubocop: enable Gitlab/HTTParty
+
+    table = CSV.parse(body.to_s, headers: true)
+    project_ids = table['PROJECT_ID']
+
+    puts "Loaded #{project_ids.size} project ids to import"
+
+    imported = 0
+    missing = 0
 
     if project_ids.any?
-      project_ids.in_groups_of(BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE) do |ids|
+      project_ids.in_groups_of(BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE, false) do |ids|
         projects = Project.where(id: ids)
         Projects::BuildArtifactsSizeRefresh.enqueue_refresh(projects)
+
+        imported += projects.size
+        missing += ids.size - projects.size
+        puts "#{imported}/#{project_ids.size} (missing projects: #{missing})"
       end
       puts 'Done.'.green
     else
-      puts 'Please provide a string of space-separated project IDs as the argument or through the STDIN'.red
+      puts 'Project IDs must be listed in the CSV under the header PROJECT_ID'.red
     end
   end
 end

spec/tasks/gitlab/refresh_project_statistics_build_artifacts_size_rake_spec.rb

@@ -11,37 +11,44 @@ RSpec.describe 'gitlab:refresh_project_statistics_build_artifacts_size rake task
     let_it_be(:project_3) { create(:project) }
 
     let(:string_of_ids) { "#{project_1.id} #{project_2.id} #{project_3.id} 999999" }
+    let(:csv_url) { 'https://www.example.com/foo.csv' }
+    let(:csv_body) do
+      <<~BODY
+        PROJECT_ID
+        #{project_1.id}
+        #{project_2.id}
+        #{project_3.id}
+      BODY
+    end
 
     before do
       Rake.application.rake_require('tasks/gitlab/refresh_project_statistics_build_artifacts_size')
 
       stub_const("BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE", 2)
-    end
-
-    context 'when given a list of space-separated IDs through STDIN' do
-      before do
-        allow($stdin).to receive(:tty?).and_return(false)
-        allow($stdin).to receive(:read).and_return(string_of_ids)
-      end
-
-      it 'enqueues the projects for refresh' do
-        expect { run_rake_task(rake_task) }.to output(/Done/).to_stdout
 
-        expect(Projects::BuildArtifactsSizeRefresh.all.map(&:project)).to match_array([project_1, project_2, project_3])
-      end
+      stub_request(:get, csv_url).to_return(status: 200, body: csv_body)
     end
 
     context 'when given a list of space-separated IDs through rake argument' do
       it 'enqueues the projects for refresh' do
-        expect { run_rake_task(rake_task, string_of_ids) }.to output(/Done/).to_stdout
+        expect { run_rake_task(rake_task, csv_url) }.to output(/Done/).to_stdout
 
         expect(Projects::BuildArtifactsSizeRefresh.all.map(&:project)).to match_array([project_1, project_2, project_3])
       end
     end
 
-    context 'when not given any IDs' do
+    context 'when CSV has invalid header' do
+      let(:csv_body) do
+        <<~BODY
+          projectid
+          #{project_1.id}
+          #{project_2.id}
+          #{project_3.id}
+        BODY
+      end
+
       it 'returns an error message' do
-        expect { run_rake_task(rake_task) }.to output(/Please provide a string of space-separated project IDs/).to_stdout
+        expect { run_rake_task(rake_task, csv_url) }.to output(/Project IDs must be listed in the CSV under the header PROJECT_ID/).to_stdout
       end
     end
   end