Merge branch 'rc/speedup-rake-jobs-ee' into 'master'

[EE] Speedup rake jobs

See merge request gitlab-org/gitlab-ee!4857

.gitlab-ci.yml

@@ -40,8 +40,14 @@ variables:
 
 before_script:
   - bundle --version
+  - date
   - source scripts/utils.sh
+  - date
   - source scripts/prepare_build.sh
+  - date
+
+after_script:
+  - date
 
 stages:
   - build
@@ -101,6 +107,26 @@ stages:
     - /(^docs[\/-].*|.*-docs$)/
     - /(^qa[\/-].*|.*-qa$)/
 
+# Jobs that only need to pull cache
+.dedicated-no-docs-pull-cache-job: &dedicated-no-docs-pull-cache-job
+  <<: *dedicated-runner
+  <<: *except-docs-and-qa
+  <<: *pull-cache
+  dependencies:
+    - setup-test-env
+  stage: test
+
+# Jobs that do not need a DB
+.dedicated-no-docs-no-db-pull-cache-job: &dedicated-no-docs-no-db-pull-cache-job
+  <<: *dedicated-no-docs-pull-cache-job
+  variables:
+    SETUP_DB: "false"
+
+.rake-exec: &rake-exec
+  <<: *dedicated-no-docs-no-db-pull-cache-job
+  script:
+    - bundle exec rake $CI_JOB_NAME
+
 .rspec-metadata: &rspec-metadata
   <<: *dedicated-runner
   <<: *except-docs-and-qa
@@ -208,21 +234,24 @@ stages:
     - master@gitlab/gitlabhq
     - master@gitlab/gitlab-ee
 
-##
-# Trigger a package build in omnibus-gitlab repository
-#
-package-qa:
-  <<: *dedicated-runner
-  image: ruby:2.4-alpine
-  before_script: []
-  stage: build
-  cache: {}
-  when: manual
+.gitlab-setup: &gitlab-setup
+  <<: *dedicated-no-docs-pull-cache-job
+  <<: *use-pg
+  variables:
+    SETUP_DB: "false"
+    CREATE_DB_USER: "true"
   script:
-    - scripts/trigger-build-omnibus
-  only:
-    - //@gitlab-org/gitlab-ce
-    - //@gitlab-org/gitlab-ee
+    # Manually clone gitlab-test and only seed this project in
+    # db/fixtures/development/04_project.rb thanks to SIZE=1 below
+    - git clone https://gitlab.com/gitlab-org/gitlab-test.git
+       /home/git/repositories/gitlab-org/gitlab-test.git
+    - scripts/gitaly-test-spawn
+    - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup
+  artifacts:
+    when: on_failure
+    expire_in: 1d
+    paths:
+      - log/development.log
 
 # Review docs base
 .review-docs: &review-docs
@@ -245,6 +274,47 @@ package-qa:
   only:
     - branches
 
+# DB migration, rollback, and seed jobs
+.db-migrate-reset: &db-migrate-reset
+  <<: *dedicated-no-docs-pull-cache-job
+  script:
+    - bundle exec rake db:migrate:reset
+
+.migration-paths: &migration-paths
+  <<: *dedicated-no-docs-pull-cache-job
+  variables:
+    CREATE_DB_USER: "true"
+  script:
+    - git fetch https://gitlab.com/gitlab-org/gitlab-ee.git v9.3.0-ee
+    - git checkout -f FETCH_HEAD
+    - bundle install $BUNDLE_INSTALL_FLAGS
+    - date
+    - cp config/gitlab.yml.example config/gitlab.yml
+    - bundle exec rake db:drop db:create db:schema:load db:seed_fu
+    - date
+    - git checkout $CI_COMMIT_SHA
+    - bundle install $BUNDLE_INSTALL_FLAGS
+    - date
+    - . scripts/prepare_build.sh
+    - date
+    - bundle exec rake db:migrate
+
+##
+# Trigger a package build in omnibus-gitlab repository
+#
+package-qa:
+  <<: *dedicated-runner
+  image: ruby:2.4-alpine
+  before_script: []
+  stage: build
+  cache: {}
+  when: manual
+  script:
+    - scripts/trigger-build-omnibus
+  only:
+    - //@gitlab-org/gitlab-ce
+    - //@gitlab-org/gitlab-ee
+
 # Trigger a docs build in gitlab-docs
 # Useful to preview the docs changes live
 review-docs-deploy:
@@ -324,7 +394,7 @@ update-tests-metadata:
 
 flaky-examples-check:
   <<: *dedicated-runner
-  image: ruby:2.3-alpine
+  image: ruby:2.4-alpine
   services: []
   before_script: []
   variables:
@@ -358,7 +428,9 @@ compile-assets:
     <<: *default-cache
   script:
     - node --version
+    - date
     - yarn install --frozen-lockfile --cache-folder .yarn-cache
+    - date
     - bundle exec rake gitlab:assets:compile
   artifacts:
     expire_in: 7d
@@ -454,26 +526,11 @@ spinach-pg 1 2: *spinach-metadata-pg
 spinach-mysql 0 2: *spinach-metadata-mysql
 spinach-mysql 1 2: *spinach-metadata-mysql
 
-# Static analysis jobs
-.ruby-static-analysis: &ruby-static-analysis
-  variables:
-    SIMPLECOV: "false"
-    SETUP_DB: "false"
-
-.rake-exec: &rake-exec
-  <<: *dedicated-runner
-  <<: *except-docs-and-qa
-  <<: *pull-cache
-  <<: *ruby-static-analysis
-  stage: test
-  script:
-    - bundle exec rake $CI_JOB_NAME
-
 static-analysis:
-  <<: *dedicated-runner
-  <<: *except-docs
-  <<: *ruby-static-analysis
-  stage: test
+  <<: *dedicated-no-docs-no-db-pull-cache-job
+  dependencies:
+    - compile-assets
+    - setup-test-env
   script:
     - scripts/static-analysis
   cache:
@@ -530,15 +587,6 @@ ee_compat_check:
     paths:
       - ee_compat_check/patches/*.patch
 
-# DB migration, rollback, and seed jobs
-.db-migrate-reset: &db-migrate-reset
-  <<: *dedicated-runner
-  <<: *except-docs-and-qa
-  <<: *pull-cache
-  stage: test
-  script:
-    - bundle exec rake db:migrate:reset
-
 db:migrate:reset-pg:
   <<: *db-migrate-reset
   <<: *use-pg
@@ -553,25 +601,6 @@ db:check-schema-pg:
   script:
     - source scripts/schema_changed.sh
 
-.migration-paths: &migration-paths
-  <<: *dedicated-runner
-  <<: *except-docs-and-qa
-  <<: *pull-cache
-  stage: test
-  variables:
-    SETUP_DB: "false"
-    CREATE_DB_USER: "true"
-  script:
-    - git fetch https://gitlab.com/gitlab-org/gitlab-ee.git v9.3.0-ee
-    - git checkout -f FETCH_HEAD
-    - bundle install $BUNDLE_INSTALL_FLAGS
-    - cp config/gitlab.yml.example config/gitlab.yml
-    - bundle exec rake db:drop db:create db:schema:load db:seed_fu
-    - git checkout $CI_COMMIT_SHA
-    - bundle install $BUNDLE_INSTALL_FLAGS
-    - . scripts/prepare_build.sh
-    - bundle exec rake db:migrate
-
 migration:path-pg:
   <<: *migration-paths
   <<: *use-pg
@@ -581,10 +610,7 @@ migration:path-mysql:
   <<: *use-mysql
 
 .db-rollback: &db-rollback
-  <<: *dedicated-runner
-  <<: *except-docs-and-qa
-  <<: *pull-cache
-  stage: test
+  <<: *dedicated-no-docs-pull-cache-job
   script:
     - bundle exec rake db:rollback STEP=119
     - bundle exec rake db:migrate
@@ -597,27 +623,6 @@ db:rollback-mysql:
   <<: *db-rollback
   <<: *use-mysql
 
-.gitlab-setup: &gitlab-setup
-  <<: *dedicated-runner
-  <<: *except-docs-and-qa
-  <<: *pull-cache
-  stage: test
-  variables:
-    SIZE: "1"
-    SETUP_DB: "false"
-    CREATE_DB_USER: "true"
-    FIXTURE_PATH: db/fixtures/development
-  script:
-    - git clone https://gitlab.com/gitlab-org/gitlab-test.git
-       /home/git/repositories/gitlab-org/gitlab-test.git
-    - scripts/gitaly-test-spawn
-    - force=yes bundle exec rake gitlab:setup
-  artifacts:
-    when: on_failure
-    expire_in: 1d
-    paths:
-      - log/development.log
-
 gitlab:setup-pg:
   <<: *gitlab-setup
   <<: *use-pg
@@ -628,10 +633,7 @@ gitlab:setup-mysql:
 
 # Frontend-related jobs
 gitlab:assets:compile:
-  <<: *dedicated-runner
-  <<: *except-docs-and-qa
-  <<: *pull-cache
-  stage: test
+  <<: *dedicated-no-docs-no-db-pull-cache-job
   dependencies: []
   variables:
     NODE_ENV: "production"
@@ -641,7 +643,9 @@ gitlab:assets:compile:
     WEBPACK_REPORT: "true"
     NO_COMPRESSION: "true"
   script:
+    - date
     - yarn install --frozen-lockfile --production --cache-folder .yarn-cache
+    - date
     - bundle exec rake gitlab:assets:compile
   artifacts:
     name: webpack-report
@@ -650,17 +654,16 @@ gitlab:assets:compile:
       - webpack-report/
 
 karma:
-  <<: *dedicated-runner
-  <<: *except-docs-and-qa
-  <<: *pull-cache
+  <<: *dedicated-no-docs-pull-cache-job
   <<: *use-pg
-  stage: test
-  variables:
-    BABEL_ENV: "coverage"
-    CHROME_LOG_FILE: "chrome_debug.log"
+  dependencies:
+    - compile-assets
+    - setup-test-env
   script:
+    - export BABEL_ENV=coverage CHROME_LOG_FILE=chrome_debug.log
+    - date
     - scripts/gitaly-test-spawn
-    - bundle exec rake gettext:po_to_json
+    - date
     - bundle exec rake karma
   coverage: '/^Statements *: (\d+\.\d+%)/'
   artifacts:
@@ -672,9 +675,7 @@ karma:
       - coverage-javascript/
 
 codequality:
-  <<: *except-docs
-  <<: *pull-cache
-  stage: test
+  <<: *dedicated-no-docs-no-db-pull-cache-job
   image: docker:latest
   before_script: []
   services:
@@ -706,11 +707,7 @@ sast:
     paths: [gl-sast-report.json]
 
 qa:internal:
-  <<: *dedicated-runner
-  <<: *except-docs
-  stage: test
-  variables:
-    SETUP_DB: "false"
+  <<: *dedicated-no-docs-no-db-pull-cache-job
   services: []
   script:
     - cd qa/
@@ -718,11 +715,7 @@ qa:internal:
     - bundle exec rspec
 
 qa:selectors:
-  <<: *dedicated-runner
-  <<: *except-docs
-  stage: test
-  variables:
-    SETUP_DB: "false"
+  <<: *dedicated-no-docs-no-db-pull-cache-job
   services: []
   script:
     - cd qa/
@@ -730,14 +723,8 @@ qa:selectors:
     - bundle exec bin/qa Test::Sanity::Selectors
 
 coverage:
-  <<: *dedicated-runner
-  <<: *except-docs-and-qa
-  <<: *pull-cache
+  <<: *dedicated-no-docs-no-db-pull-cache-job
   stage: post-test
-  services: []
-  variables:
-    SETUP_DB: "false"
-    USE_BUNDLE_INSTALL: "true"
   script:
     - bundle exec scripts/merge-simplecov
   coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
@@ -749,16 +736,16 @@ coverage:
     - coverage/assets/
 
 lint:javascript:report:
-  <<: *dedicated-runner
-  <<: *except-docs-and-qa
-  <<: *pull-cache
+  <<: *dedicated-no-docs-no-db-pull-cache-job
   stage: post-test
   dependencies:
     - compile-assets
     - setup-test-env
   before_script: []
   script:
+    - date
     - find app/ spec/ -name '*.js' -exec sed --in-place 's|/\* eslint-disable .*\*/||' {} \; # run report over all files
+    - date
     - yarn run eslint-report || true # ignore exit code
   artifacts:
     name: eslint-report
@@ -767,8 +754,7 @@ lint:javascript:report:
       - eslint-report.html
 
 pages:
-  <<: *dedicated-runner
-  <<: *pull-cache
+  <<: *dedicated-no-docs-no-db-pull-cache-job
   before_script: []
   stage: pages
   dependencies:
@@ -793,10 +779,7 @@ pages:
 # Insurance in case a gem needed by one of our releases gets yanked from
 # rubygems.org in the future.
 cache gems:
-  <<: *dedicated-runner
-  <<: *pull-cache
-  variables:
-    SETUP_DB: "false"
+  <<: *dedicated-no-docs-no-db-pull-cache-job
   script:
     - bundle package --all --all-platforms
   artifacts: