Commit 5dc15b2a authored by Sean McGivern's avatar Sean McGivern

Merge branch 'speedup-gitlab-assets-compile' into 'master'

Cache assets in gitlab:assets:compile

Closes #56691

See merge request gitlab-org/gitlab-ce!24542
parents f04910f2 6b39f55d
...@@ -386,20 +386,25 @@ flaky-examples-check: ...@@ -386,20 +386,25 @@ flaky-examples-check:
- scripts/merge-reports ${NEW_FLAKY_SPECS_REPORT} rspec_flaky/new_*_*.json - scripts/merge-reports ${NEW_FLAKY_SPECS_REPORT} rspec_flaky/new_*_*.json
- scripts/detect-new-flaky-examples $NEW_FLAKY_SPECS_REPORT - scripts/detect-new-flaky-examples $NEW_FLAKY_SPECS_REPORT
.assets-compile-cache: &assets-compile-cache
cache:
key: "assets-compile:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v3"
paths:
- vendor/ruby/
- .yarn-cache/
- tmp/cache/assets/sprockets
compile-assets: compile-assets:
<<: *dedicated-runner <<: *dedicated-runner
<<: *except-docs <<: *except-docs
<<: *use-pg <<: *use-pg
stage: prepare stage: prepare
cache:
<<: *default-cache
script: script:
- node --version - node --version
- date
- yarn install --frozen-lockfile --cache-folder .yarn-cache - yarn install --frozen-lockfile --cache-folder .yarn-cache
- date
- free -m - free -m
- bundle exec rake gitlab:assets:compile - bundle exec rake gitlab:assets:compile
- scripts/clean-old-cached-assets
variables: variables:
# we override the max_old_space_size to prevent OOM errors # we override the max_old_space_size to prevent OOM errors
NODE_OPTIONS: --max_old_space_size=3584 NODE_OPTIONS: --max_old_space_size=3584
...@@ -408,6 +413,7 @@ compile-assets: ...@@ -408,6 +413,7 @@ compile-assets:
paths: paths:
- node_modules - node_modules
- public/assets - public/assets
<<: *assets-compile-cache
setup-test-env: setup-test-env:
<<: *dedicated-runner <<: *dedicated-runner
...@@ -628,7 +634,9 @@ gitlab:setup-mysql: ...@@ -628,7 +634,9 @@ gitlab:setup-mysql:
gitlab:assets:compile: gitlab:assets:compile:
<<: *dedicated-no-docs-pull-cache-job <<: *dedicated-no-docs-pull-cache-job
image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.5.3-git-2.18-chrome-71.0-node-8.x-yarn-1.12-graphicsmagick-1.3.29-docker-18.06.1 image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.5.3-git-2.18-chrome-71.0-node-8.x-yarn-1.12-graphicsmagick-1.3.29-docker-18.06.1
dependencies: [] dependencies:
- setup-test-env
- compile-assets
services: services:
- docker:stable-dind - docker:stable-dind
variables: variables:
...@@ -642,18 +650,19 @@ gitlab:assets:compile: ...@@ -642,18 +650,19 @@ gitlab:assets:compile:
DOCKER_DRIVER: overlay2 DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375 DOCKER_HOST: tcp://docker:2375
script: script:
- date - node --version
- yarn install --frozen-lockfile --production --cache-folder .yarn-cache - yarn install --frozen-lockfile --production --cache-folder .yarn-cache
- date
- free -m - free -m
- bundle exec rake gitlab:assets:compile - bundle exec rake gitlab:assets:compile
- scripts/build_assets_image - time scripts/build_assets_image
- scripts/clean-old-cached-assets
artifacts: artifacts:
name: webpack-report name: webpack-report
expire_in: 31d expire_in: 31d
paths: paths:
- webpack-report/ - webpack-report/
- public/assets/ - public/assets/
<<: *assets-compile-cache
only: only:
- //@gitlab-org/gitlab-ce - //@gitlab-org/gitlab-ce
- //@gitlab-org/gitlab-ee - //@gitlab-org/gitlab-ee
......
# frozen_string_literal: true
Sprockets::Base.prepend(Gitlab::Patch::SprocketsBaseFileDigestKey)
# frozen_string_literal: true
# This monkey patch prevent cache ballooning when caching tmp/cache/assets/sprockets
# on the CI. See https://github.com/rails/sprockets/issues/563 and
# https://github.com/rails/sprockets/compare/3.x...jmreid:no-mtime-for-digest-key.
module Gitlab
module Patch
module SprocketsBaseFileDigestKey
def file_digest(path)
if stat = self.stat(path)
digest = self.stat_digest(path, stat)
integrity_uri = self.hexdigest_integrity_uri(digest)
key = Sprockets::UnloadedAsset.new(path, self).file_digest_key(integrity_uri)
cache.fetch(key) do
digest
end
end
end
end
end
end
# frozen_string_literal: true # frozen_string_literal: true
require 'rainbow/ext/string' require 'rainbow/ext/string'
require 'gitlab/utils/strong_memoize' require_dependency 'gitlab/utils/strong_memoize'
# rubocop:disable Rails/Output # rubocop:disable Rails/Output
module Gitlab module Gitlab
...@@ -13,6 +13,12 @@ module Gitlab ...@@ -13,6 +13,12 @@ module Gitlab
extend self extend self
def invoke_and_time_task(task)
start = Time.now
Rake::Task[task].invoke
puts "`#{task}` finished in #{Time.now - start} seconds"
end
# Ask if the user wants to continue # Ask if the user wants to continue
# #
# Returns "yes" the user chose to continue # Returns "yes" the user chose to continue
......
# frozen_string_literal: true # frozen_string_literal: true
require_dependency 'gitlab/utils'
module Gitlab module Gitlab
module Utils module Utils
module MergeHash module MergeHash
......
# frozen_string_literal: true # frozen_string_literal: true
require_dependency 'gitlab/utils'
module Gitlab module Gitlab
module Utils module Utils
module Override module Override
......
# frozen_string_literal: true # frozen_string_literal: true
require_dependency 'gitlab/utils'
module Gitlab module Gitlab
module Utils module Utils
module StrongMemoize module StrongMemoize
......
namespace :gitlab do namespace :gitlab do
namespace :assets do namespace :assets do
desc 'GitLab | Assets | Compile all frontend assets' desc 'GitLab | Assets | Compile all frontend assets'
task compile: [ task :compile do
'yarn:check', require_dependency 'gitlab/task_helpers'
'gettext:po_to_json',
'rake:assets:precompile', %w[
'webpack:compile', yarn:check
'fix_urls' gettext:po_to_json
] rake:assets:precompile
webpack:compile
gitlab:assets:fix_urls
].each(&Gitlab::TaskHelpers.method(:invoke_and_time_task))
end
desc 'GitLab | Assets | Clean up old compiled frontend assets' desc 'GitLab | Assets | Clean up old compiled frontend assets'
task clean: ['rake:assets:clean'] task clean: ['rake:assets:clean']
......
#!/bin/bash
# Clean up cached files that are older than 1 week
find tmp/cache/assets/sprockets/ -type f -mtime +7 -execdir rm -- "{}" \;
du -d 0 -h tmp/cache/assets/sprockets | cut -f1 | xargs -I % echo "tmp/cache/assets/sprockets/ is currently %"
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment