Automatic merge of gitlab-org/gitlab-ce master

.gitlab-ci.yml

@@ -71,6 +71,7 @@ stages:
     paths:
       - knapsack/
       - rspec_flaky/
+      - rspec_profiling/
 
 .use-pg: &use-pg
   services:
@@ -190,6 +191,7 @@ stages:
       - coverage/
       - knapsack/
       - rspec_flaky/
+      - rspec_profiling/
       - tmp/capybara/
     reports:
       junit: junit_rspec.xml
@@ -427,6 +429,7 @@ retrieve-tests-metadata:
     - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
     - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
     - mkdir -p rspec_flaky/
+    - mkdir -p rspec_profiling/
     - wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH
     - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}'
     - wget -O $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH
@@ -471,7 +474,7 @@ update-tests-metadata:
       - rspec_flaky/
     policy: push
   script:
-    - retry gem install fog-aws mime-types activesupport --no-document
+    - retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document
     - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json
     - scripts/merge-reports ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg-ee_node_*.json
     - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json
@@ -481,6 +484,7 @@ update-tests-metadata:
     - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH'
     - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
     - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
+    - scripts/insert-rspec-profiling-data
 
 flaky-examples-check:
   <<: *dedicated-runner

config/initializers/rspec_profiling.rb

+# frozen_string_literal: true
+
+return unless Rails.env.test?
+
 module RspecProfilingExt
-  module PSQL
-    def establish_connection
-      ::RspecProfiling::Collectors::PSQL::Result.establish_connection(ENV['RSPEC_PROFILING_POSTGRES_URL'])
+  module Collectors
+    class CSVWithTimestamps < ::RspecProfiling::Collectors::CSV
+      TIMESTAMP_FIELDS = %w(created_at updated_at).freeze
+      HEADERS = (::RspecProfiling::Collectors::CSV::HEADERS + TIMESTAMP_FIELDS).freeze
+
+      def insert(attributes)
+        output << HEADERS.map do |field|
+          if TIMESTAMP_FIELDS.include?(field)
+            Time.now
+          else
+            attributes.fetch(field.to_sym)
+          end
+        end
+      end
+
+      private
+
+      def output
+        @output ||= ::CSV.open(path, "w").tap { |csv| csv << HEADERS }
+      end
     end
   end
 
@@ -10,9 +31,13 @@ module RspecProfilingExt
       if ENV['CI_COMMIT_REF_NAME']
         "#{defined?(Gitlab::License) ? 'ee' : 'ce'}:#{ENV['CI_COMMIT_REF_NAME']}"
       else
-        super
+        super&.chomp
       end
     end
+
+    def sha
+      super&.chomp
+    end
   end
 
   module Run
@@ -30,16 +55,11 @@ module RspecProfilingExt
   end
 end
 
-if Rails.env.test?
-  RspecProfiling.configure do |config|
-    if ENV['RSPEC_PROFILING_POSTGRES_URL'].present?
-      RspecProfiling::Collectors::PSQL.prepend(RspecProfilingExt::PSQL)
-      config.collector = RspecProfiling::Collectors::PSQL
-    end
-
-    if ENV.key?('CI')
-      RspecProfiling::VCS::Git.prepend(RspecProfilingExt::Git)
-      RspecProfiling::Run.prepend(RspecProfilingExt::Run)
-    end
+RspecProfiling.configure do |config|
+  if ENV.key?('CI') || ENV.key?('RSPEC_PROFILING')
+    RspecProfiling::VCS::Git.prepend(RspecProfilingExt::Git)
+    RspecProfiling::Run.prepend(RspecProfilingExt::Run)
+    config.collector = RspecProfilingExt::Collectors::CSVWithTimestamps
+    config.csv_path = -> { "rspec_profiling/#{Time.now.to_i}-#{SecureRandom.hex(8)}-rspec-data.csv" }
   end
 end

package.json

@@ -30,7 +30,7 @@
     "@babel/preset-env": "^7.3.1",
     "@gitlab/csslab": "^1.8.0",
     "@gitlab/svgs": "^1.54.0",
-    "@gitlab/ui": "^2.2.2",
+    "@gitlab/ui": "^2.2.3",
     "apollo-boost": "^0.3.1",
     "apollo-client": "^2.5.1",
     "autosize": "^4.0.0",

scripts/insert-rspec-profiling-data

+#!/usr/bin/env ruby
+
+require 'csv'
+require 'rspec_profiling'
+require 'postgres-copy'
+
+module RspecProfiling
+  module Collectors
+    class PSQL
+      def establish_connection
+        # This disables the automatic creation of the database and
+        # table. In the future, we may want a way to specify the host of
+        # the database to connect so that we can call #install.
+        Result.establish_connection(results_url)
+      end
+
+      def prepared?
+        connection.data_source_exists?(table)
+      end
+
+      def results_url
+        ENV['RSPEC_PROFILING_POSTGRES_URL']
+      end
+
+      class Result < ActiveRecord::Base
+        acts_as_copy_target
+      end
+    end
+  end
+end
+
+def insert_data(path)
+  puts "#{Time.now} Inserting CI stats..."
+
+  collector = RspecProfiling::Collectors::PSQL.new
+  collector.install
+
+  files = Dir[File.join(path, "*.csv")]
+
+  files.each do |filename|
+    puts "#{Time.now} Inserting #{filename}..."
+    result = RspecProfiling::Collectors::PSQL::Result.copy_from(filename)
+    puts "#{Time.now} Inserted #{result.cmd_tuples} lines in #{filename}, DB response: #{result.cmd_status}"
+  end
+end
+
+insert_data('rspec_profiling') if ENV['RSPEC_PROFILING_POSTGRES_URL'].present?

yarn.lock

@@ -663,10 +663,10 @@
   resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-1.54.0.tgz#00320e845efd46716042cde0c348b990d4908daf"
   integrity sha512-DR17iy8TM5IbXEacqiDP0p8SuC/J8EL+98xbfVz5BKvRsPHpeZJQNlBF/petIV5d+KWM5A9v3GZTY7uMU7z/JQ==
 
-"@gitlab/ui@^2.2.2":
-  version "2.2.2"
-  resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-2.2.2.tgz#de8f436d6b52c168c4752b221a88cf7665fe0134"
-  integrity sha512-xVY8lGfDA3D2EtyfZVpJVeNUXLbpn/xJqNF4fleqHJfqrwt+IcVlsQ7yEs/LBukmIw6tXPliD4Mm9uefnQhkXA==
+"@gitlab/ui@^2.2.3":
+  version "2.2.3"
+  resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-2.2.3.tgz#b3b4d1d785662dfba44ad2a7354fcbe4cee22fc9"
+  integrity sha512-N1Q1O6zpS4zZhmvWsUCtuGXTzQeHOzRWQZctbFTEJonidIWk6juqIBduYgR0MadG3DZxiovUN12jDGVtCfZKzw==
   dependencies:
     "@babel/standalone" "^7.0.0"
     bootstrap-vue "^2.0.0-rc.11"