Commit fb5c3c70 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch 'artifacts' into 'master'

Implement Build Artifacts

This implements #3028 

1. It stores artifacts in shared/artifacts,
1. It adds `artifacts` to `.gitlab-ci.yml`,
1. We use GitLab Workhorse to offload artifacts uploading,
1. To download artifacts it uses GitLab Workhorse X-Sendfile extension,
1. There's one "artifact" per-build. The new upload removes previous one and creates a new one,
1. Default max artifact size is set to 100MB - this can be changed in settings.

Missing things:
1. Support for `.gitlab-ci.yml`: `artifacts: true or git-ls-files` which will upload all non tracked files,
1. Artifacts passing between builds.

GitLab Workhorse changes: https://gitlab.com/gitlab-org/gitlab-workhorse/merge_requests/5
GitLab Runner changes: https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/merge_requests/46

Syntax:
```
artifacts:
  untracked: true # default: false
  paths: # default: empty
  - bin/files
```


See merge request !1584
parents dfa09789 d70f1f35
...@@ -37,6 +37,7 @@ nohup.out ...@@ -37,6 +37,7 @@ nohup.out
public/assets/ public/assets/
public/uploads.* public/uploads.*
public/uploads/ public/uploads/
shared/artifacts/
rails_best_practices_output.html rails_best_practices_output.html
/tags /tags
tmp/ tmp/
......
...@@ -102,6 +102,7 @@ v 8.1.0 ...@@ -102,6 +102,7 @@ v 8.1.0
- Show CI status on Your projects page and Starred projects page - Show CI status on Your projects page and Starred projects page
- Remove "Continuous Integration" page from dashboard - Remove "Continuous Integration" page from dashboard
- Add notes and SSL verification entries to hook APIs (Ben Boeckel) - Add notes and SSL verification entries to hook APIs (Ben Boeckel)
- Added build artifacts
- Fix grammar in admin area "labels" .nothing-here-block when no labels exist. - Fix grammar in admin area "labels" .nothing-here-block when no labels exist.
- Move CI runners page to project settings area - Move CI runners page to project settings area
- Move CI variables page to project settings area - Move CI variables page to project settings area
......
...@@ -54,7 +54,7 @@ gem 'gollum-lib', '~> 4.0.2' ...@@ -54,7 +54,7 @@ gem 'gollum-lib', '~> 4.0.2'
gem "github-linguist", "~> 4.7.0", require: "linguist" gem "github-linguist", "~> 4.7.0", require: "linguist"
# API # API
gem 'grape', '~> 0.6.1' gem 'grape', '~> 0.13.0'
gem 'grape-entity', '~> 0.4.2' gem 'grape-entity', '~> 0.4.2'
gem 'rack-cors', '~> 0.4.0', require: 'rack/cors' gem 'rack-cors', '~> 0.4.0', require: 'rack/cors'
......
...@@ -306,10 +306,10 @@ GEM ...@@ -306,10 +306,10 @@ GEM
gon (5.0.4) gon (5.0.4)
actionpack (>= 2.3.0) actionpack (>= 2.3.0)
json json
grape (0.6.1) grape (0.13.0)
activesupport activesupport
builder builder
hashie (>= 1.2.0) hashie (>= 2.1.0)
multi_json (>= 1.3.2) multi_json (>= 1.3.2)
multi_xml (>= 0.5.2) multi_xml (>= 0.5.2)
rack (>= 1.3.0) rack (>= 1.3.0)
...@@ -835,7 +835,7 @@ DEPENDENCIES ...@@ -835,7 +835,7 @@ DEPENDENCIES
gitlab_omniauth-ldap (~> 1.2.1) gitlab_omniauth-ldap (~> 1.2.1)
gollum-lib (~> 4.0.2) gollum-lib (~> 4.0.2)
gon (~> 5.0.0) gon (~> 5.0.0)
grape (~> 0.6.1) grape (~> 0.13.0)
grape-entity (~> 0.4.2) grape-entity (~> 0.4.2)
haml-rails (~> 0.9.0) haml-rails (~> 0.9.0)
hipchat (~> 1.5.0) hipchat (~> 1.5.0)
......
...@@ -58,6 +58,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController ...@@ -58,6 +58,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
:admin_notification_email, :admin_notification_email,
:user_oauth_applications, :user_oauth_applications,
:shared_runners_enabled, :shared_runners_enabled,
:max_artifacts_size,
restricted_visibility_levels: [], restricted_visibility_levels: [],
import_sources: [] import_sources: []
) )
......
...@@ -3,6 +3,7 @@ class Projects::BuildsController < Projects::ApplicationController ...@@ -3,6 +3,7 @@ class Projects::BuildsController < Projects::ApplicationController
before_action :build, except: [:index, :cancel_all] before_action :build, except: [:index, :cancel_all]
before_action :authorize_manage_builds!, except: [:index, :show, :status] before_action :authorize_manage_builds!, except: [:index, :show, :status]
before_action :authorize_download_build_artifacts!, only: [:download]
layout "project" layout "project"
...@@ -51,6 +52,18 @@ class Projects::BuildsController < Projects::ApplicationController ...@@ -51,6 +52,18 @@ class Projects::BuildsController < Projects::ApplicationController
redirect_to build_path(build) redirect_to build_path(build)
end end
def download
unless artifacts_file.file_storage?
return redirect_to artifacts_file.url
end
unless artifacts_file.exists?
return not_found!
end
send_file artifacts_file.path, disposition: 'attachment'
end
def status def status
render json: @build.to_json(only: [:status, :id, :sha, :coverage], methods: :sha) render json: @build.to_json(only: [:status, :id, :sha, :coverage], methods: :sha)
end end
...@@ -67,6 +80,10 @@ class Projects::BuildsController < Projects::ApplicationController ...@@ -67,6 +80,10 @@ class Projects::BuildsController < Projects::ApplicationController
@build ||= ci_project.builds.unscoped.find_by!(id: params[:id]) @build ||= ci_project.builds.unscoped.find_by!(id: params[:id])
end end
def artifacts_file
build.artifacts_file
end
def build_path(build) def build_path(build)
namespace_project_build_path(build.gl_project.namespace, build.gl_project, build) namespace_project_build_path(build.gl_project.namespace, build.gl_project, build)
end end
...@@ -76,4 +93,14 @@ class Projects::BuildsController < Projects::ApplicationController ...@@ -76,4 +93,14 @@ class Projects::BuildsController < Projects::ApplicationController
return page_404 return page_404
end end
end end
def authorize_download_build_artifacts!
unless can?(current_user, :download_build_artifacts, @project)
if current_user.nil?
return authenticate_user!
else
return render_404
end
end
end
end end
...@@ -154,6 +154,7 @@ class Ability ...@@ -154,6 +154,7 @@ class Ability
:create_merge_request, :create_merge_request,
:create_wiki, :create_wiki,
:manage_builds, :manage_builds,
:download_build_artifacts,
:push_code :push_code
] ]
end end
......
...@@ -89,6 +89,7 @@ class ApplicationSetting < ActiveRecord::Base ...@@ -89,6 +89,7 @@ class ApplicationSetting < ActiveRecord::Base
restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'], import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'],
shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
max_artifacts_size: Settings.gitlab_ci['max_artifacts_size'],
) )
end end
......
...@@ -39,6 +39,8 @@ module Ci ...@@ -39,6 +39,8 @@ module Ci
scope :ignore_failures, ->() { where(allow_failure: false) } scope :ignore_failures, ->() { where(allow_failure: false) }
scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) } scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) }
mount_uploader :artifacts_file, ArtifactUploader
acts_as_taggable acts_as_taggable
# To prevent db load megabytes of data from trace # To prevent db load megabytes of data from trace
...@@ -217,6 +219,14 @@ module Ci ...@@ -217,6 +219,14 @@ module Ci
"#{dir_to_trace}/#{id}.log" "#{dir_to_trace}/#{id}.log"
end end
def token
project.token
end
def valid_token? token
project.valid_token? token
end
def target_url def target_url
Gitlab::Application.routes.url_helpers. Gitlab::Application.routes.url_helpers.
namespace_project_build_url(gl_project.namespace, gl_project, self) namespace_project_build_url(gl_project.namespace, gl_project, self)
...@@ -248,6 +258,13 @@ module Ci ...@@ -248,6 +258,13 @@ module Ci
pending? && !any_runners_online? pending? && !any_runners_online?
end end
def download_url
if artifacts_file.exists?
Gitlab::Application.routes.url_helpers.
download_namespace_project_build_path(gl_project.namespace, gl_project, self)
end
end
private private
def yaml_variables def yaml_variables
......
...@@ -92,4 +92,8 @@ class CommitStatus < ActiveRecord::Base ...@@ -92,4 +92,8 @@ class CommitStatus < ActiveRecord::Base
def show_warning? def show_warning?
false false
end end
def download_url
nil
end
end end
# encoding: utf-8
class ArtifactUploader < CarrierWave::Uploader::Base
storage :file
attr_accessor :build, :field
def self.artifacts_path
File.expand_path('shared/artifacts/', Rails.root)
end
def self.artifacts_upload_path
File.expand_path('shared/artifacts/tmp/uploads/', Rails.root)
end
def self.artifacts_cache_path
File.expand_path('shared/artifacts/tmp/cache/', Rails.root)
end
def initialize(build, field)
@build, @field = build, field
end
def artifacts_path
File.join(build.created_at.utc.strftime('%Y_%m'), build.project.id.to_s, build.id.to_s)
end
def store_dir
File.join(ArtifactUploader.artifacts_path, artifacts_path)
end
def cache_dir
File.join(ArtifactUploader.artifacts_cache_path, artifacts_path)
end
def file_storage?
self.class.storage == CarrierWave::Storage::File
end
def exists?
file.try(:exists?)
end
def move_to_cache
true
end
def move_to_store
true
end
end
...@@ -139,5 +139,10 @@ ...@@ -139,5 +139,10 @@
= f.check_box :shared_runners_enabled = f.check_box :shared_runners_enabled
Enable shared runners for a new projects Enable shared runners for a new projects
.form-group
= f.label :max_artifacts_size, 'Maximum artifacts size (MB)', class: 'control-label col-sm-2'
.col-sm-10
= f.number_field :max_artifacts_size, class: 'form-control'
.form-actions .form-actions
= f.submit 'Save', class: 'btn btn-primary' = f.submit 'Save', class: 'btn btn-primary'
...@@ -87,6 +87,9 @@ ...@@ -87,6 +87,9 @@
Test coverage Test coverage
%h1 #{@build.coverage}% %h1 #{@build.coverage}%
- if current_user && can?(current_user, :download_build_artifacts, @project) && @build.download_url
.build-widget.center
= link_to "Download artifacts", @build.download_url, class: 'btn btn-sm btn-primary'
.build-widget .build-widget
%h4.title %h4.title
......
...@@ -61,6 +61,9 @@ ...@@ -61,6 +61,9 @@
%td %td
.pull-right .pull-right
- if current_user && can?(current_user, :download_build_artifacts, @project) && commit_status.download_url
= link_to commit_status.download_url, title: 'Download artifacts' do
%i.fa.fa-download
- if current_user && can?(current_user, :manage_builds, commit_status.gl_project) - if current_user && can?(current_user, :manage_builds, commit_status.gl_project)
- if commit_status.active? - if commit_status.active?
- if commit_status.cancel_url - if commit_status.cancel_url
......
...@@ -186,6 +186,7 @@ Settings.gitlab_ci['all_broken_builds'] = true if Settings.gitlab_ci['all_br ...@@ -186,6 +186,7 @@ Settings.gitlab_ci['all_broken_builds'] = true if Settings.gitlab_ci['all_br
Settings.gitlab_ci['add_pusher'] = false if Settings.gitlab_ci['add_pusher'].nil? Settings.gitlab_ci['add_pusher'] = false if Settings.gitlab_ci['add_pusher'].nil?
Settings.gitlab_ci['url'] ||= Settings.send(:build_gitlab_ci_url) Settings.gitlab_ci['url'] ||= Settings.send(:build_gitlab_ci_url)
Settings.gitlab_ci['builds_path'] = File.expand_path(Settings.gitlab_ci['builds_path'] || "builds/", Rails.root) Settings.gitlab_ci['builds_path'] = File.expand_path(Settings.gitlab_ci['builds_path'] || "builds/", Rails.root)
Settings.gitlab_ci['max_artifacts_size'] ||= 100 # in megabytes
# #
# Reply by email # Reply by email
......
...@@ -611,6 +611,7 @@ Gitlab::Application.routes.draw do ...@@ -611,6 +611,7 @@ Gitlab::Application.routes.draw do
member do member do
get :status get :status
post :cancel post :cancel
get :download
post :retry post :retry
end end
end end
......
class AddArtifactsFileToBuilds < ActiveRecord::Migration
def change
add_column :ci_builds, :artifacts_file, :text
end
end
class AddMaxArtifactsSizeToApplicationSettings < ActiveRecord::Migration
def change
add_column :application_settings, :max_artifacts_size, :integer, default: 100, null: false
end
end
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20151105094515) do ActiveRecord::Schema.define(version: 20151109100728) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
...@@ -48,6 +48,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do ...@@ -48,6 +48,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do
t.text "help_page_text" t.text "help_page_text"
t.string "admin_notification_email" t.string "admin_notification_email"
t.boolean "shared_runners_enabled", default: true, null: false t.boolean "shared_runners_enabled", default: true, null: false
t.integer "max_artifacts_size", default: 100, null: false
end end
create_table "audit_events", force: true do |t| create_table "audit_events", force: true do |t|
...@@ -108,6 +109,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do ...@@ -108,6 +109,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do
t.string "type" t.string "type"
t.string "target_url" t.string "target_url"
t.string "description" t.string "description"
t.text "artifacts_file"
end end
add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree
......
...@@ -141,6 +141,7 @@ job_name: ...@@ -141,6 +141,7 @@ job_name:
| tags | optional | Defines a list of tags which are used to select runner | | tags | optional | Defines a list of tags which are used to select runner |
| allow_failure | optional | Allow build to fail. Failed build doesn't contribute to commit status | | allow_failure | optional | Allow build to fail. Failed build doesn't contribute to commit status |
| when | optional | Define when to run build. Can be `on_success`, `on_failure` or `always` | | when | optional | Define when to run build. Can be `on_success`, `on_failure` or `always` |
| artifacts | optional | Define list build artifacts |
### script ### script
`script` is a shell script which is executed by runner. The shell script is prepended with `before_script`. `script` is a shell script which is executed by runner. The shell script is prepended with `before_script`.
...@@ -258,6 +259,35 @@ The above script will: ...@@ -258,6 +259,35 @@ The above script will:
1. Execute `cleanup_build` only when the `build` failed, 1. Execute `cleanup_build` only when the `build` failed,
2. Always execute `cleanup` as the last step in pipeline. 2. Always execute `cleanup` as the last step in pipeline.
### artifacts
`artifacts` is used to specify list of files and directories which should be attached to build after success.
1. Send all files in `binaries` and `.config`:
```
artifacts:
paths:
- binaries/
- .config
```
2. Send all git untracked files:
```
artifacts:
untracked: true
```
3. Send all git untracked files and files in `binaries`:
```
artifacts:
untracked: true
paths:
- binaries/
```
The artifacts will be send after the build success to GitLab and will be accessible in GitLab interface to download.
This feature requires GitLab Runner v0.7.0 or higher.
## Validate the .gitlab-ci.yml ## Validate the .gitlab-ci.yml
Each instance of GitLab CI has an embedded debug tool called Lint. Each instance of GitLab CI has an embedded debug tool called Lint.
You can find the link to the Lint in the project's settings page or use short url `/lint`. You can find the link to the Lint in the project's settings page or use short url `/lint`.
......
...@@ -246,6 +246,9 @@ We recommend using a PostgreSQL database. For MySQL check [MySQL setup guide](da ...@@ -246,6 +246,9 @@ We recommend using a PostgreSQL database. For MySQL check [MySQL setup guide](da
# Change the permissions of the directory where CI build traces are stored # Change the permissions of the directory where CI build traces are stored
sudo chmod -R u+rwX builds/ sudo chmod -R u+rwX builds/
# Change the permissions of the directory where CI artifacts are stored
sudo chmod -R u+rwX shared/artifacts/
# Copy the example Unicorn config # Copy the example Unicorn config
sudo -u git -H cp config/unicorn.rb.example config/unicorn.rb sudo -u git -H cp config/unicorn.rb.example config/unicorn.rb
......
...@@ -29,7 +29,8 @@ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production ...@@ -29,7 +29,8 @@ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production
``` ```
Also you can choose what should be backed up by adding environment variable SKIP. Available options: db, Also you can choose what should be backed up by adding environment variable SKIP. Available options: db,
uploads (attachments), repositories, builds(CI build output logs). Use a comma to specify several options at the same time. uploads (attachments), repositories, builds(CI build output logs), artifacts (CI build artifacts).
Use a comma to specify several options at the same time.
``` ```
sudo gitlab-rake gitlab:backup:create SKIP=db,uploads sudo gitlab-rake gitlab:backup:create SKIP=db,uploads
......
...@@ -133,6 +133,12 @@ module API ...@@ -133,6 +133,12 @@ module API
authorize! :admin_project, user_project authorize! :admin_project, user_project
end end
def require_gitlab_workhorse!
unless env['HTTP_GITLAB_WORKHORSE'].present?
forbidden!('Request should be executed via GitLab Workhorse')
end
end
def can?(object, action, subject) def can?(object, action, subject)
abilities.allowed?(object, action, subject) abilities.allowed?(object, action, subject)
end end
...@@ -234,6 +240,10 @@ module API ...@@ -234,6 +240,10 @@ module API
render_api_error!(message || '409 Conflict', 409) render_api_error!(message || '409 Conflict', 409)
end end
def file_to_large!
render_api_error!('413 Request Entity Too Large', 413)
end
def render_validation_error!(model) def render_validation_error!(model)
if model.errors.any? if model.errors.any?
render_api_error!(model.errors.messages || '400 Bad Request', 400) render_api_error!(model.errors.messages || '400 Bad Request', 400)
...@@ -282,6 +292,44 @@ module API ...@@ -282,6 +292,44 @@ module API
end end
end end
# file helpers
def uploaded_file!(field, uploads_path)
if params[field]
bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename)
return params[field]
end
# sanitize file paths
# this requires all paths to exist
required_attributes! %W(#{field}.path)
uploads_path = File.realpath(uploads_path)
file_path = File.realpath(params["#{field}.path"])
bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
UploadedFile.new(
file_path,
params["#{field}.name"],
params["#{field}.type"] || 'application/octet-stream',
)
end
def present_file!(path, filename, content_type = 'application/octet-stream')
filename ||= File.basename(path)
header['Content-Disposition'] = "attachment; filename=#{filename}"
header['Content-Transfer-Encoding'] = 'binary'
content_type content_type
# Support download acceleration
case headers['X-Sendfile-Type']
when 'X-Sendfile'
header['X-Sendfile'] = path
body
else
file FileStreamer.new(path)
end
end
private private
def add_pagination_headers(paginated, per_page) def add_pagination_headers(paginated, per_page)
......
require 'backup/files'
module Backup
class Artifacts < Files
def initialize
super('artifacts', ArtifactUploader.artifacts_path)
end
def create_files_dir
Dir.mkdir(app_files_dir, 0700)
end
end
end
...@@ -150,7 +150,7 @@ module Backup ...@@ -150,7 +150,7 @@ module Backup
private private
def backup_contents def backup_contents
folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "artifacts.tar.gz", "backup_information.yml"]
end end
def folders_to_backup def folders_to_backup
......
...@@ -27,6 +27,7 @@ module Ci ...@@ -27,6 +27,7 @@ module Ci
helpers Helpers helpers Helpers
helpers ::API::Helpers helpers ::API::Helpers
helpers Gitlab::CurrentSettings
mount Builds mount Builds
mount Commits mount Commits
......
...@@ -47,6 +47,106 @@ module Ci ...@@ -47,6 +47,106 @@ module Ci
build.drop build.drop
end end
end end
# Authorize artifacts uploading for build - Runners only
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# filesize (optional) - the size of uploaded file
# Example Request:
# POST /builds/:id/artifacts/authorize
post ":id/artifacts/authorize" do
require_gitlab_workhorse!
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
forbidden!('build is not running') unless build.running?
if params[:filesize]
file_size = params[:filesize].to_i
file_to_large! unless file_size < max_artifacts_size
end
status 200
{ TempPath: ArtifactUploader.artifacts_upload_path }
end
# Upload artifacts to build - Runners only
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# file (required) - The uploaded file
# Parameters (accelerated by GitLab Workhorse):
# file.path - path to locally stored body (generated by Workhorse)
# file.name - real filename as send in Content-Disposition
# file.type - real content type as send in Content-Type
# Headers:
# BUILD-TOKEN (required) - The build authorization token, the same as token
# Body:
# The file content
#
# Example Request:
# POST /builds/:id/artifacts
post ":id/artifacts" do
require_gitlab_workhorse!
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
forbidden!('build is not running') unless build.running?
file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path)
file_to_large! unless file.size < max_artifacts_size
if build.update_attributes(artifacts_file: file)
present build, with: Entities::Build
else
render_validation_error!(build)
end
end
# Download the artifacts file from build - Runners only
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# Headers:
# BUILD-TOKEN (required) - The build authorization token, the same as token
# Example Request:
# GET /builds/:id/artifacts
get ":id/artifacts" do
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
artifacts_file = build.artifacts_file
unless artifacts_file.file_storage?
return redirect_to build.artifacts_file.url
end
unless artifacts_file.exists?
not_found!
end
present_file!(artifacts_file.path, artifacts_file.filename)
end
# Remove the artifacts file from build
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# Headers:
# BUILD-TOKEN (required) - The build authorization token, the same as token
# Example Request:
# DELETE /builds/:id/artifacts
delete ":id/artifacts" do
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
build.remove_artifacts_file!
end
end end
end end
end end
......
...@@ -11,10 +11,16 @@ module Ci ...@@ -11,10 +11,16 @@ module Ci
expose :builds expose :builds
end end
class ArtifactFile < Grape::Entity
expose :filename, :size
end
class Build < Grape::Entity class Build < Grape::Entity
expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url, expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url,
:before_sha, :allow_git_fetch, :project_name :before_sha, :allow_git_fetch, :project_name
expose :name, :token, :stage
expose :options do |model| expose :options do |model|
model.options model.options
end end
...@@ -24,6 +30,7 @@ module Ci ...@@ -24,6 +30,7 @@ module Ci
end end
expose :variables expose :variables
expose :artifacts_file, using: ArtifactFile
end end
class Runner < Grape::Entity class Runner < Grape::Entity
......
module Ci module Ci
module API module API
module Helpers module Helpers
BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN"
BUILD_TOKEN_PARAM = :token
UPDATE_RUNNER_EVERY = 60 UPDATE_RUNNER_EVERY = 60
def authenticate_runners! def authenticate_runners!
...@@ -15,6 +17,11 @@ module Ci ...@@ -15,6 +17,11 @@ module Ci
forbidden! unless project.valid_token?(params[:project_token]) forbidden! unless project.valid_token?(params[:project_token])
end end
def authenticate_build_token!(build)
token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
forbidden! unless token && build.valid_token?(token)
end
def update_runner_last_contact def update_runner_last_contact
# Use a random threshold to prevent beating DB updates # Use a random threshold to prevent beating DB updates
contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
...@@ -32,6 +39,10 @@ module Ci ...@@ -32,6 +39,10 @@ module Ci
info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"])
current_runner.update(info) current_runner.update(info)
end end
def max_artifacts_size
current_application_settings.max_artifacts_size.megabytes.to_i
end
end end
end end
end end
...@@ -5,7 +5,7 @@ module Ci ...@@ -5,7 +5,7 @@ module Ci
DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGES = %w(build test deploy)
DEFAULT_STAGE = 'test' DEFAULT_STAGE = 'test'
ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables] ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when] ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts]
attr_reader :before_script, :image, :services, :variables, :path attr_reader :before_script, :image, :services, :variables, :path
...@@ -77,7 +77,8 @@ module Ci ...@@ -77,7 +77,8 @@ module Ci
when: job[:when] || 'on_success', when: job[:when] || 'on_success',
options: { options: {
image: job[:image] || @image, image: job[:image] || @image,
services: job[:services] || @services services: job[:services] || @services,
artifacts: job[:artifacts]
}.compact }.compact
} }
end end
...@@ -159,7 +160,17 @@ module Ci ...@@ -159,7 +160,17 @@ module Ci
raise ValidationError, "#{name} job: except parameter should be an array of strings" raise ValidationError, "#{name} job: except parameter should be an array of strings"
end end
if job[:allow_failure] && !job[:allow_failure].in?([true, false]) if job[:artifacts]
if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked])
raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean"
end
if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
end
end
if job[:allow_failure] && !validate_boolean(job[:allow_failure])
raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
end end
...@@ -182,6 +193,10 @@ module Ci ...@@ -182,6 +193,10 @@ module Ci
value.is_a?(String) || value.is_a?(Symbol) value.is_a?(String) || value.is_a?(Symbol)
end end
def validate_boolean(value)
value.in?([true, false])
end
def process?(only_params, except_params, ref, tag) def process?(only_params, except_params, ref, tag)
if only_params.present? if only_params.present?
return false unless matching?(only_params, ref, tag) return false unless matching?(only_params, ref, tag)
......
class FileStreamer #:nodoc:
attr_reader :to_path
def initialize(path)
@to_path = path
end
# Stream the file's contents if Rack::Sendfile isn't present.
def each
File.open(to_path, 'rb') do |file|
while chunk = file.read(16384)
yield chunk
end
end
end
end
...@@ -25,6 +25,7 @@ module Gitlab ...@@ -25,6 +25,7 @@ module Gitlab
session_expire_delay: Settings.gitlab['session_expire_delay'], session_expire_delay: Settings.gitlab['session_expire_delay'],
import_sources: Settings.gitlab['import_sources'], import_sources: Settings.gitlab['import_sources'],
shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'],
) )
end end
......
...@@ -131,6 +131,22 @@ server { ...@@ -131,6 +131,22 @@ server {
return 418; return 418;
} }
# Build artifacts should be submitted to this location
location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
client_max_body_size 0;
# 'Error' 418 is a hack to re-use the @gitlab-workhorse block
error_page 418 = @gitlab-workhorse;
return 418;
}
# Build artifacts should be submitted to this location
location ~ /ci/api/v1/builds/[0-9]+/artifacts {
client_max_body_size 0;
# 'Error' 418 is a hack to re-use the @gitlab-workhorse block
error_page 418 = @gitlab-workhorse;
return 418;
}
location @gitlab-workhorse { location @gitlab-workhorse {
## If you use HTTPS make sure you disable gzip compression ## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack. ## to be safe against BREACH attack.
......
...@@ -178,6 +178,22 @@ server { ...@@ -178,6 +178,22 @@ server {
return 418; return 418;
} }
# Build artifacts should be submitted to this location
location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
client_max_body_size 0;
# 'Error' 418 is a hack to re-use the @gitlab-workhorse block
error_page 418 = @gitlab-workhorse;
return 418;
}
# Build artifacts should be submitted to this location
location ~ /ci/api/v1/builds/[0-9]+/artifacts {
client_max_body_size 0;
# 'Error' 418 is a hack to re-use the @gitlab-workhorse block
error_page 418 = @gitlab-workhorse;
return 418;
}
location @gitlab-workhorse { location @gitlab-workhorse {
## If you use HTTPS make sure you disable gzip compression ## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack. ## to be safe against BREACH attack.
......
...@@ -12,6 +12,7 @@ namespace :gitlab do ...@@ -12,6 +12,7 @@ namespace :gitlab do
Rake::Task["gitlab:backup:repo:create"].invoke Rake::Task["gitlab:backup:repo:create"].invoke
Rake::Task["gitlab:backup:uploads:create"].invoke Rake::Task["gitlab:backup:uploads:create"].invoke
Rake::Task["gitlab:backup:builds:create"].invoke Rake::Task["gitlab:backup:builds:create"].invoke
Rake::Task["gitlab:backup:artifacts:create"].invoke
backup = Backup::Manager.new backup = Backup::Manager.new
backup.pack backup.pack
...@@ -32,6 +33,7 @@ namespace :gitlab do ...@@ -32,6 +33,7 @@ namespace :gitlab do
Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories") Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds") Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts")
Rake::Task["gitlab:shell:setup"].invoke Rake::Task["gitlab:shell:setup"].invoke
backup.cleanup backup.cleanup
...@@ -113,6 +115,25 @@ namespace :gitlab do ...@@ -113,6 +115,25 @@ namespace :gitlab do
end end
end end
namespace :artifacts do
task create: :environment do
$progress.puts "Dumping artifacts ... ".blue
if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
$progress.puts "[SKIPPED]".cyan
else
Backup::Artifacts.new.dump
$progress.puts "done".green
end
end
task restore: :environment do
$progress.puts "Restoring artifacts ... ".blue
Backup::Artifacts.new.restore
$progress.puts "done".green
end
end
def configure_cron_mode def configure_cron_mode
if ENV['CRON'] if ENV['CRON']
# We need an object we can say 'puts' and 'print' to; let's use a # We need an object we can say 'puts' and 'print' to; let's use a
......
require "tempfile"
require "fileutils"
# Taken from: Rack::Test::UploadedFile
class UploadedFile
# The filename, *not* including the path, of the "uploaded" file
attr_reader :original_filename
# The tempfile
attr_reader :tempfile
# The content type of the "uploaded" file
attr_accessor :content_type
def initialize(path, filename, content_type = "text/plain")
raise "#{path} file does not exist" unless ::File.exist?(path)
@content_type = content_type
@original_filename = filename || ::File.basename(path)
@tempfile = File.new(path, 'rb')
end
def path
@tempfile.path
end
alias_method :local_path, :path
def method_missing(method_name, *args, &block) #:nodoc:
@tempfile.__send__(method_name, *args, &block)
end
def respond_to?(method_name, include_private = false) #:nodoc:
@tempfile.respond_to?(method_name, include_private) || super
end
end
require 'spec_helper' require 'spec_helper'
describe "Builds" do describe "Builds" do
let(:artifacts_file) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') }
before do before do
login_as(:user) login_as(:user)
@commit = FactoryGirl.create :ci_commit @commit = FactoryGirl.create :ci_commit
...@@ -66,6 +68,15 @@ describe "Builds" do ...@@ -66,6 +68,15 @@ describe "Builds" do
it { expect(page).to have_content @commit.sha[0..7] } it { expect(page).to have_content @commit.sha[0..7] }
it { expect(page).to have_content @commit.git_commit_message } it { expect(page).to have_content @commit.git_commit_message }
it { expect(page).to have_content @commit.git_author_name } it { expect(page).to have_content @commit.git_author_name }
context "Download artifacts" do
before do
@build.update_attributes(artifacts_file: artifacts_file)
visit namespace_project_build_path(@gl_project.namespace, @gl_project, @build)
end
it { expect(page).to have_content 'Download artifacts' }
end
end end
describe "POST /:project/builds/:id/cancel" do describe "POST /:project/builds/:id/cancel" do
...@@ -90,4 +101,14 @@ describe "Builds" do ...@@ -90,4 +101,14 @@ describe "Builds" do
it { expect(page).to have_content 'pending' } it { expect(page).to have_content 'pending' }
it { expect(page).to have_content 'Cancel' } it { expect(page).to have_content 'Cancel' }
end end
describe "GET /:project/builds/:id/download" do
before do
@build.update_attributes(artifacts_file: artifacts_file)
visit namespace_project_build_path(@gl_project.namespace, @gl_project, @build)
click_link 'Download artifacts'
end
it { expect(page.response_headers['Content-Type']).to eq(artifacts_file.content_type) }
end
end end
...@@ -19,7 +19,7 @@ describe "Commits" do ...@@ -19,7 +19,7 @@ describe "Commits" do
stub_ci_commit_to_return_yaml_file stub_ci_commit_to_return_yaml_file
end end
describe "GET /:project/commits/:sha" do describe "GET /:project/commits/:sha/ci" do
before do before do
visit ci_status_path(@commit) visit ci_status_path(@commit)
end end
...@@ -29,6 +29,20 @@ describe "Commits" do ...@@ -29,6 +29,20 @@ describe "Commits" do
it { expect(page).to have_content @commit.git_author_name } it { expect(page).to have_content @commit.git_author_name }
end end
context "Download artifacts" do
let(:artifacts_file) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') }
before do
@build.update_attributes(artifacts_file: artifacts_file)
end
it do
visit ci_status_path(@commit)
click_on "Download artifacts"
expect(page.response_headers['Content-Type']).to eq(artifacts_file.content_type)
end
end
describe "Cancel all builds" do describe "Cancel all builds" do
it "cancels commit" do it "cancels commit" do
visit ci_status_path(@commit) visit ci_status_path(@commit)
......
...@@ -333,6 +333,43 @@ module Ci ...@@ -333,6 +333,43 @@ module Ci
end end
end end
describe "Artifacts" do
it "returns artifacts when defined" do
config = YAML.dump({
image: "ruby:2.1",
services: ["mysql"],
before_script: ["pwd"],
rspec: {
artifacts: { paths: ["logs/", "binaries/"], untracked: true },
script: "rspec"
}
})
config_processor = GitlabCiYamlProcessor.new(config)
expect(config_processor.builds_for_stage_and_ref("test", "master").size).to eq(1)
expect(config_processor.builds_for_stage_and_ref("test", "master").first).to eq({
except: nil,
stage: "test",
stage_idx: 1,
name: :rspec,
only: nil,
commands: "pwd\nrspec",
tag_list: [],
options: {
image: "ruby:2.1",
services: ["mysql"],
artifacts: {
paths: ["logs/", "binaries/"],
untracked: true
}
},
when: "on_success",
allow_failure: false
})
end
end
describe "Error handling" do describe "Error handling" do
it "indicates that object is invalid" do it "indicates that object is invalid" do
expect{GitlabCiYamlProcessor.new("invalid_yaml\n!ccdvlf%612334@@@@")}.to raise_error(GitlabCiYamlProcessor::ValidationError) expect{GitlabCiYamlProcessor.new("invalid_yaml\n!ccdvlf%612334@@@@")}.to raise_error(GitlabCiYamlProcessor::ValidationError)
...@@ -491,6 +528,20 @@ module Ci ...@@ -491,6 +528,20 @@ module Ci
GitlabCiYamlProcessor.new(config, path) GitlabCiYamlProcessor.new(config, path)
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: when parameter should be on_success, on_failure or always") end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: when parameter should be on_success, on_failure or always")
end end
it "returns errors if job artifacts:untracked is not an array of strings" do
config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: { untracked: "string" } } })
expect do
GitlabCiYamlProcessor.new(config)
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts:untracked parameter should be an boolean")
end
it "returns errors if job artifacts:paths is not an array of strings" do
config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: { paths: "string" } } })
expect do
GitlabCiYamlProcessor.new(config)
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts:paths parameter should be an array of strings")
end
end end
end end
end end
...@@ -400,4 +400,19 @@ describe Ci::Build do ...@@ -400,4 +400,19 @@ describe Ci::Build do
end end
end end
end end
describe :download_url do
subject { build.download_url }
it "should be nil if artifact doesn't exist" do
build.update_attributes(artifacts_file: nil)
is_expected.to be_nil
end
it 'should be nil if artifact exist' do
gif = fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif')
build.update_attributes(artifacts_file: gif)
is_expected.to_not be_nil
end
end
end end
...@@ -343,8 +343,9 @@ describe API::API, api: true do ...@@ -343,8 +343,9 @@ describe API::API, api: true do
end.to change{ user.keys.count }.by(1) end.to change{ user.keys.count }.by(1)
end end
it "should raise error for invalid ID" do it "should return 405 for invalid ID" do
expect{post api("/users/ASDF/keys", admin) }.to raise_error(ActionController::RoutingError) post api("/users/ASDF/keys", admin)
expect(response.status).to eq(405)
end end
end end
...@@ -374,9 +375,9 @@ describe API::API, api: true do ...@@ -374,9 +375,9 @@ describe API::API, api: true do
expect(json_response.first['title']).to eq(key.title) expect(json_response.first['title']).to eq(key.title)
end end
it "should return 404 for invalid ID" do it "should return 405 for invalid ID" do
get api("/users/ASDF/keys", admin) get api("/users/ASDF/keys", admin)
expect(response.status).to eq(404) expect(response.status).to eq(405)
end end
end end
end end
...@@ -434,7 +435,8 @@ describe API::API, api: true do ...@@ -434,7 +435,8 @@ describe API::API, api: true do
end end
it "should raise error for invalid ID" do it "should raise error for invalid ID" do
expect{post api("/users/ASDF/emails", admin) }.to raise_error(ActionController::RoutingError) post api("/users/ASDF/emails", admin)
expect(response.status).to eq(405)
end end
end end
...@@ -465,7 +467,8 @@ describe API::API, api: true do ...@@ -465,7 +467,8 @@ describe API::API, api: true do
end end
it "should raise error for invalid ID" do it "should raise error for invalid ID" do
expect{put api("/users/ASDF/emails", admin) }.to raise_error(ActionController::RoutingError) put api("/users/ASDF/emails", admin)
expect(response.status).to eq(405)
end end
end end
end end
......
...@@ -41,7 +41,7 @@ describe Ci::API::API do ...@@ -41,7 +41,7 @@ describe Ci::API::API do
it "should return 404 error if no builds for specific runner" do it "should return 404 error if no builds for specific runner" do
commit = FactoryGirl.create(:ci_commit, gl_project: shared_gl_project) commit = FactoryGirl.create(:ci_commit, gl_project: shared_gl_project)
FactoryGirl.create(:ci_build, commit: commit, status: 'pending' ) FactoryGirl.create(:ci_build, commit: commit, status: 'pending')
post ci_api("/builds/register"), token: runner.token post ci_api("/builds/register"), token: runner.token
...@@ -50,7 +50,7 @@ describe Ci::API::API do ...@@ -50,7 +50,7 @@ describe Ci::API::API do
it "should return 404 error if no builds for shared runner" do it "should return 404 error if no builds for shared runner" do
commit = FactoryGirl.create(:ci_commit, gl_project: gl_project) commit = FactoryGirl.create(:ci_commit, gl_project: gl_project)
FactoryGirl.create(:ci_build, commit: commit, status: 'pending' ) FactoryGirl.create(:ci_build, commit: commit, status: 'pending')
post ci_api("/builds/register"), token: shared_runner.token post ci_api("/builds/register"), token: shared_runner.token
...@@ -79,7 +79,7 @@ describe Ci::API::API do ...@@ -79,7 +79,7 @@ describe Ci::API::API do
{ "key" => "CI_BUILD_NAME", "value" => "spinach", "public" => true }, { "key" => "CI_BUILD_NAME", "value" => "spinach", "public" => true },
{ "key" => "CI_BUILD_STAGE", "value" => "test", "public" => true }, { "key" => "CI_BUILD_STAGE", "value" => "test", "public" => true },
{ "key" => "DB_NAME", "value" => "postgres", "public" => true }, { "key" => "DB_NAME", "value" => "postgres", "public" => true },
{ "key" => "SECRET_KEY", "value" => "secret_value", "public" => false }, { "key" => "SECRET_KEY", "value" => "secret_value", "public" => false }
]) ])
end end
...@@ -122,5 +122,194 @@ describe Ci::API::API do ...@@ -122,5 +122,194 @@ describe Ci::API::API do
expect(build.reload.trace).to eq 'hello_world' expect(build.reload.trace).to eq 'hello_world'
end end
end end
context "Artifacts" do
let(:file_upload) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') }
let(:file_upload2) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/gif') }
let(:commit) { FactoryGirl.create(:ci_commit, gl_project: gl_project) }
let(:build) { FactoryGirl.create(:ci_build, commit: commit, runner_id: runner.id) }
let(:authorize_url) { ci_api("/builds/#{build.id}/artifacts/authorize") }
let(:post_url) { ci_api("/builds/#{build.id}/artifacts") }
let(:delete_url) { ci_api("/builds/#{build.id}/artifacts") }
let(:get_url) { ci_api("/builds/#{build.id}/artifacts") }
let(:headers) { { "GitLab-Workhorse" => "1.0" } }
let(:headers_with_token) { headers.merge(Ci::API::Helpers::BUILD_TOKEN_HEADER => build.project.token) }
describe "POST /builds/:id/artifacts/authorize" do
context "should authorize posting artifact to running build" do
before do
build.run!
end
it "using token as parameter" do
post authorize_url, { token: build.project.token }, headers
expect(response.status).to eq(200)
expect(json_response["TempPath"]).to_not be_nil
end
it "using token as header" do
post authorize_url, {}, headers_with_token
expect(response.status).to eq(200)
expect(json_response["TempPath"]).to_not be_nil
end
end
context "should fail to post too large artifact" do
before do
build.run!
end
it "using token as parameter" do
settings = Gitlab::CurrentSettings::current_application_settings
settings.update_attributes(max_artifacts_size: 0)
post authorize_url, { token: build.project.token, filesize: 100 }, headers
expect(response.status).to eq(413)
end
it "using token as header" do
settings = Gitlab::CurrentSettings::current_application_settings
settings.update_attributes(max_artifacts_size: 0)
post authorize_url, { filesize: 100 }, headers_with_token
expect(response.status).to eq(413)
end
end
context "should get denied" do
it do
post authorize_url, { token: 'invalid', filesize: 100 }
expect(response.status).to eq(403)
end
end
end
describe "POST /builds/:id/artifacts" do
context "Disable sanitizer" do
before do
# by configuring this path we allow to pass temp file from any path
allow(ArtifactUploader).to receive(:artifacts_upload_path).and_return('/')
end
context "should post artifact to running build" do
before do
build.run!
end
it "uses regual file post" do
upload_artifacts(file_upload, headers_with_token, false)
expect(response.status).to eq(201)
expect(json_response["artifacts_file"]["filename"]).to eq(file_upload.original_filename)
end
it "uses accelerated file post" do
upload_artifacts(file_upload, headers_with_token, true)
expect(response.status).to eq(201)
expect(json_response["artifacts_file"]["filename"]).to eq(file_upload.original_filename)
end
it "updates artifact" do
upload_artifacts(file_upload, headers_with_token)
upload_artifacts(file_upload2, headers_with_token)
expect(response.status).to eq(201)
expect(json_response["artifacts_file"]["filename"]).to eq(file_upload2.original_filename)
end
end
context "should fail to post too large artifact" do
before do
build.run!
end
it do
settings = Gitlab::CurrentSettings::current_application_settings
settings.update_attributes(max_artifacts_size: 0)
upload_artifacts(file_upload, headers_with_token)
expect(response.status).to eq(413)
end
end
context "should fail to post artifacts without file" do
before do
build.run!
end
it do
post post_url, {}, headers_with_token
expect(response.status).to eq(400)
end
end
context "should fail to post artifacts without GitLab-Workhorse" do
before do
build.run!
end
it do
post post_url, { token: build.project.token }, {}
expect(response.status).to eq(403)
end
end
end
context "should fail to post artifacts for outside of tmp path" do
before do
# by configuring this path we allow to pass file from @tmpdir only
# but all temporary files are stored in system tmp directory
@tmpdir = Dir.mktmpdir
allow(ArtifactUploader).to receive(:artifacts_upload_path).and_return(@tmpdir)
build.run!
end
after do
FileUtils.remove_entry @tmpdir
end
it do
upload_artifacts(file_upload, headers_with_token)
expect(response.status).to eq(400)
end
end
def upload_artifacts(file, headers = {}, accelerated = true)
if accelerated
post post_url, {
'file.path' => file.path,
'file.name' => file.original_filename
}, headers
else
post post_url, { file: file }, headers
end
end
end
describe "DELETE /builds/:id/artifacts" do
before do
build.run!
post delete_url, token: build.project.token, file: file_upload
end
it "should delete artifact build" do
build.success
delete delete_url, token: build.project.token
expect(response.status).to eq(200)
end
end
describe "GET /builds/:id/artifacts" do
before do
build.run!
end
it "should download artifact" do
build.update_attributes(artifacts_file: file_upload)
get get_url, token: build.project.token
expect(response.status).to eq(200)
end
it "should fail to download if no artifact uploaded" do
get get_url, token: build.project.token
expect(response.status).to eq(404)
end
end
end
end end
end end
...@@ -16,7 +16,7 @@ describe 'gitlab:app namespace rake task' do ...@@ -16,7 +16,7 @@ describe 'gitlab:app namespace rake task' do
end end
def reenable_backup_sub_tasks def reenable_backup_sub_tasks
%w{db repo uploads builds}.each do |subtask| %w{db repo uploads builds artifacts}.each do |subtask|
Rake::Task["gitlab:backup:#{subtask}:create"].reenable Rake::Task["gitlab:backup:#{subtask}:create"].reenable
end end
end end
...@@ -56,6 +56,7 @@ describe 'gitlab:app namespace rake task' do ...@@ -56,6 +56,7 @@ describe 'gitlab:app namespace rake task' do
expect(Rake::Task["gitlab:backup:repo:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:backup:repo:restore"]).to receive(:invoke)
expect(Rake::Task["gitlab:backup:builds:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:backup:builds:restore"]).to receive(:invoke)
expect(Rake::Task["gitlab:backup:uploads:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:backup:uploads:restore"]).to receive(:invoke)
expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive(:invoke)
expect(Rake::Task["gitlab:shell:setup"]).to receive(:invoke) expect(Rake::Task["gitlab:shell:setup"]).to receive(:invoke)
expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
end end
...@@ -113,19 +114,20 @@ describe 'gitlab:app namespace rake task' do ...@@ -113,19 +114,20 @@ describe 'gitlab:app namespace rake task' do
it 'should set correct permissions on the tar contents' do it 'should set correct permissions on the tar contents' do
tar_contents, exit_status = Gitlab::Popen.popen( tar_contents, exit_status = Gitlab::Popen.popen(
%W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz} %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz}
) )
expect(exit_status).to eq(0) expect(exit_status).to eq(0)
expect(tar_contents).to match('db/') expect(tar_contents).to match('db/')
expect(tar_contents).to match('uploads.tar.gz') expect(tar_contents).to match('uploads.tar.gz')
expect(tar_contents).to match('repositories/') expect(tar_contents).to match('repositories/')
expect(tar_contents).to match('builds.tar.gz') expect(tar_contents).to match('builds.tar.gz')
expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz)\/$/) expect(tar_contents).to match('artifacts.tar.gz')
expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz)\/$/)
end end
it 'should delete temp directories' do it 'should delete temp directories' do
temp_dirs = Dir.glob( temp_dirs = Dir.glob(
File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds}') File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts}')
) )
expect(temp_dirs).to be_empty expect(temp_dirs).to be_empty
...@@ -161,12 +163,13 @@ describe 'gitlab:app namespace rake task' do ...@@ -161,12 +163,13 @@ describe 'gitlab:app namespace rake task' do
it "does not contain skipped item" do it "does not contain skipped item" do
tar_contents, _exit_status = Gitlab::Popen.popen( tar_contents, _exit_status = Gitlab::Popen.popen(
%W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz} %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz}
) )
expect(tar_contents).to match('db/') expect(tar_contents).to match('db/')
expect(tar_contents).to match('uploads.tar.gz') expect(tar_contents).to match('uploads.tar.gz')
expect(tar_contents).to match('builds.tar.gz') expect(tar_contents).to match('builds.tar.gz')
expect(tar_contents).to match('artifacts.tar.gz')
expect(tar_contents).not_to match('repositories/') expect(tar_contents).not_to match('repositories/')
end end
...@@ -178,6 +181,7 @@ describe 'gitlab:app namespace rake task' do ...@@ -178,6 +181,7 @@ describe 'gitlab:app namespace rake task' do
expect(Rake::Task["gitlab:backup:db:restore"]).to receive :invoke expect(Rake::Task["gitlab:backup:db:restore"]).to receive :invoke
expect(Rake::Task["gitlab:backup:repo:restore"]).not_to receive :invoke expect(Rake::Task["gitlab:backup:repo:restore"]).not_to receive :invoke
expect(Rake::Task["gitlab:backup:builds:restore"]).to receive :invoke expect(Rake::Task["gitlab:backup:builds:restore"]).to receive :invoke
expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive :invoke
expect(Rake::Task["gitlab:shell:setup"]).to receive :invoke expect(Rake::Task["gitlab:shell:setup"]).to receive :invoke
expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment