Merge branch 'qa-perf-testdata' into 'master'

Script to generate Perf Testdata

Closes #58140

See merge request gitlab-org/gitlab-ce!25548

qa/.gitignore

 tmp/
 .ruby-version
+urls.txt

qa/Gemfile

@@ -9,3 +9,4 @@ gem 'selenium-webdriver', '~> 3.12'
 gem 'airborne', '~> 0.2.13'
 gem 'nokogiri', '~> 1.10.1'
 gem 'rspec-retry', '~> 0.6.1'
+gem 'faker', '~> 1.6', '>= 1.6.6'

qa/Gemfile.lock

@@ -32,6 +32,8 @@ GEM
     diff-lcs (1.3)
     domain_name (0.5.20170404)
       unf (>= 0.0.5, < 1.0.0)
+    faker (1.9.3)
+      i18n (>= 0.7)
     ffi (1.9.25)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
@@ -99,6 +101,7 @@ DEPENDENCIES
   airborne (~> 0.2.13)
   capybara (~> 2.16.1)
   capybara-screenshot (~> 1.0.18)
+  faker (~> 1.6, >= 1.6.6)
   nokogiri (~> 1.10.1)
   pry-byebug (~> 3.5.1)
   rake (~> 12.3.0)

qa/Rakefile

 require_relative 'qa/tools/revoke_all_personal_access_tokens'
 require_relative 'qa/tools/delete_subgroups'
+require_relative 'qa/tools/generate_perf_testdata'
 
 desc "Revokes all personal access tokens"
 task :revoke_personal_access_tokens do
@@ -10,3 +11,8 @@ desc "Deletes subgroups within a provided group"
 task :delete_subgroups do
   QA::Tools::DeleteSubgroups.new.run
 end
+
+desc "Generate Performance Testdata"
+task :generate_perf_testdata do
+  QA::Tools::GeneratePerfTestdata.new.run
+end

qa/qa/support/api.rb

@@ -20,6 +20,16 @@ module QA
         e.response
       end
 
+      def put(url, payload)
+        RestClient::Request.execute(
+          method: :put,
+          url: url,
+          payload: payload,
+          verify_ssl: false)
+      rescue RestClient::ExceptionWithResponse => e
+        e.response
+      end
+
       def delete(url)
         RestClient::Request.execute(
           method: :delete,

qa/qa/tools/generate_perf_testdata.rb

+# frozen_string_literal: true
+
+require 'securerandom'
+require 'faker'
+require_relative '../../qa'
+# This script generates testdata for Performance Testing.
+# Required environment variables: PERSONAL_ACCESS_TOKEN and GITLAB_ADDRESS
+# This job creates a urls.txt which contains a hash of all the URLs needed for Performance Testing
+# Run `rake generate_perf_testdata`
+
+module QA
+  module Tools
+    class GeneratePerfTestdata
+      include Support::Api
+
+      def initialize
+        raise ArgumentError, "Please provide GITLAB_ADDRESS" unless ENV['GITLAB_ADDRESS']
+        raise ArgumentError, "Please provide PERSONAL_ACCESS_TOKEN" unless ENV['PERSONAL_ACCESS_TOKEN']
+
+        @api_client = Runtime::API::Client.new(ENV['GITLAB_ADDRESS'], personal_access_token: ENV['PERSONAL_ACCESS_TOKEN'])
+        @group_name = "gitlab-qa-perf-sandbox-#{SecureRandom.hex(8)}"
+        @project_name = "my-test-project-#{SecureRandom.hex(8)}"
+        @urls = {}
+      end
+
+      def run
+        STDOUT.puts 'Running...'
+        group_id = create_group
+        create_project(group_id)
+        create_branch
+        add_new_file
+        methods_arr = [
+          method(:create_issues),
+          method(:create_todos),
+          method(:create_merge_requests),
+          method(:create_issue_with_500_discussions),
+          method(:create_mr_with_large_files)
+        ]
+        threads_arr = []
+
+        methods_arr.each do |m|
+          threads_arr << Thread.new {m.call}
+        end
+
+        threads_arr.each(&:join)
+        STDOUT.puts "\nURLs: #{@urls}"
+        File.open("urls.txt", "w") { |file| file.puts @urls.to_s}
+        STDOUT.puts "\nDone"
+      end
+
+      private
+
+      def create_group
+        group_search_response = post Runtime::API::Request.new(@api_client, "/groups").url, "name=#{@group_name}&path=#{@group_name}"
+        group = JSON.parse(group_search_response.body)
+        @urls[:group_page] = group["web_url"]
+        group["id"]
+      end
+
+      def create_project(group_id)
+        create_project_response = post Runtime::API::Request.new(@api_client, "/projects").url, "name=#{@project_name}&namespace_id=#{group_id}"
+        @urls[:project_page] = JSON.parse(create_project_response.body)["web_url"]
+      end
+
+      def create_issues
+        30.times do |i|
+          post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/issues").url, "title=issue#{i}&description=desc#{i}"
+        end
+        @urls[:issues_list_page] = @urls[:project_page] + "/issues"
+        STDOUT.puts "Created Issues"
+      end
+
+      def create_todos
+        30.times do |i|
+          post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/issues/#{i + 1}/todo").url, nil
+        end
+        @urls[:todos_page] = ENV['GITLAB_ADDRESS'] + "/dashboard/todos"
+        STDOUT.puts "Created todos"
+      end
+
+      def create_merge_requests
+        30.times do |i|
+          post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/merge_requests").url, "source_branch=branch#{i}&target_branch=master&title=MR#{i}"
+        end
+        @urls[:mr_list_page] = @urls[:project_page] + "/merge_requests"
+        STDOUT.puts "Created MRs"
+      end
+
+      def add_new_file
+        post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/repository/files/hello.txt").url, "branch=master&commit_message=\"hello\"&content=\"my new content\""
+        30.times do |i|
+          post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/repository/files/hello#{i}.txt").url, "branch=branch#{i}&commit_message=\"hello\"&content=\"my new content\""
+        end
+        STDOUT.puts "Added Files"
+      end
+
+      def create_branch
+        30.times do |i|
+          post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/repository/branches").url, "branch=branch#{i}&ref=master"
+        end
+        STDOUT.puts "Created branches"
+      end
+
+      def create_issue_with_500_discussions
+        issue_id = 1
+        500.times do
+          post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/issues/#{issue_id}/discussions").url, "body=\"Let us discuss\""
+        end
+        @urls[:large_issue] = @urls[:project_page] + "/issues/#{issue_id}"
+        STDOUT.puts "Created Issue with 500 Discussions"
+      end
+
+      def create_mr_with_large_files
+        content_arr = []
+        20.times do |i|
+          faker_line_arr = Faker::Lorem.sentences(1500)
+          content = faker_line_arr.join("\n\r")
+          post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/repository/files/hello#{i}.txt").url, "branch=master&commit_message=\"Add hello#{i}.txt\"&content=#{content}"
+          content_arr[i] = faker_line_arr
+        end
+
+        post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/repository/branches").url, "branch=performance&ref=master"
+
+        20.times do |i|
+          missed_line_array = content_arr[i].each_slice(2).map(&:first)
+          content = missed_line_array.join("\n\rIm new!:D \n\r ")
+          put Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/repository/files/hello#{i}.txt").url, "branch=performance&commit_message=\"Update hello#{i}.txt\"&content=#{content}"
+        end
+
+        create_mr_response = post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/merge_requests").url, "source_branch=performance&target_branch=master&title=Large_MR"
+
+        iid = JSON.parse(create_mr_response.body)["iid"]
+        500.times do
+          post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/merge_requests/#{iid}/discussions").url, "body=\"Let us discuss\""
+        end
+        @urls[:large_mr] = JSON.parse(create_mr_response.body)["web_url"]
+        STDOUT.puts "Created MR with 500 Discussions and 20 Very Large Files"
+      end
+    end
+  end
+end