Commit fb5c3c70 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch 'artifacts' into 'master'

Implement Build Artifacts

This implements #3028 

1. It stores artifacts in shared/artifacts,
1. It adds `artifacts` to `.gitlab-ci.yml`,
1. We use GitLab Workhorse to offload artifacts uploading,
1. To download artifacts it uses GitLab Workhorse X-Sendfile extension,
1. There's one "artifact" per-build. The new upload removes previous one and creates a new one,
1. Default max artifact size is set to 100MB - this can be changed in settings.

Missing things:
1. Support for `.gitlab-ci.yml`: `artifacts: true or git-ls-files` which will upload all non tracked files,
1. Artifacts passing between builds.

GitLab Workhorse changes: https://gitlab.com/gitlab-org/gitlab-workhorse/merge_requests/5
GitLab Runner changes: https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/merge_requests/46

Syntax:
```
artifacts:
  untracked: true # default: false
  paths: # default: empty
  - bin/files
```


See merge request !1584
parents dfa09789 d70f1f35
......@@ -37,6 +37,7 @@ nohup.out
public/assets/
public/uploads.*
public/uploads/
shared/artifacts/
rails_best_practices_output.html
/tags
tmp/
......
......@@ -102,6 +102,7 @@ v 8.1.0
- Show CI status on Your projects page and Starred projects page
- Remove "Continuous Integration" page from dashboard
- Add notes and SSL verification entries to hook APIs (Ben Boeckel)
- Added build artifacts
- Fix grammar in admin area "labels" .nothing-here-block when no labels exist.
- Move CI runners page to project settings area
- Move CI variables page to project settings area
......
......@@ -54,7 +54,7 @@ gem 'gollum-lib', '~> 4.0.2'
gem "github-linguist", "~> 4.7.0", require: "linguist"
# API
gem 'grape', '~> 0.6.1'
gem 'grape', '~> 0.13.0'
gem 'grape-entity', '~> 0.4.2'
gem 'rack-cors', '~> 0.4.0', require: 'rack/cors'
......
......@@ -306,10 +306,10 @@ GEM
gon (5.0.4)
actionpack (>= 2.3.0)
json
grape (0.6.1)
grape (0.13.0)
activesupport
builder
hashie (>= 1.2.0)
hashie (>= 2.1.0)
multi_json (>= 1.3.2)
multi_xml (>= 0.5.2)
rack (>= 1.3.0)
......@@ -835,7 +835,7 @@ DEPENDENCIES
gitlab_omniauth-ldap (~> 1.2.1)
gollum-lib (~> 4.0.2)
gon (~> 5.0.0)
grape (~> 0.6.1)
grape (~> 0.13.0)
grape-entity (~> 0.4.2)
haml-rails (~> 0.9.0)
hipchat (~> 1.5.0)
......
......@@ -58,6 +58,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
:admin_notification_email,
:user_oauth_applications,
:shared_runners_enabled,
:max_artifacts_size,
restricted_visibility_levels: [],
import_sources: []
)
......
......@@ -3,6 +3,7 @@ class Projects::BuildsController < Projects::ApplicationController
before_action :build, except: [:index, :cancel_all]
before_action :authorize_manage_builds!, except: [:index, :show, :status]
before_action :authorize_download_build_artifacts!, only: [:download]
layout "project"
......@@ -51,6 +52,18 @@ class Projects::BuildsController < Projects::ApplicationController
redirect_to build_path(build)
end
def download
unless artifacts_file.file_storage?
return redirect_to artifacts_file.url
end
unless artifacts_file.exists?
return not_found!
end
send_file artifacts_file.path, disposition: 'attachment'
end
def status
render json: @build.to_json(only: [:status, :id, :sha, :coverage], methods: :sha)
end
......@@ -67,6 +80,10 @@ class Projects::BuildsController < Projects::ApplicationController
@build ||= ci_project.builds.unscoped.find_by!(id: params[:id])
end
def artifacts_file
build.artifacts_file
end
def build_path(build)
namespace_project_build_path(build.gl_project.namespace, build.gl_project, build)
end
......@@ -76,4 +93,14 @@ class Projects::BuildsController < Projects::ApplicationController
return page_404
end
end
def authorize_download_build_artifacts!
unless can?(current_user, :download_build_artifacts, @project)
if current_user.nil?
return authenticate_user!
else
return render_404
end
end
end
end
......@@ -154,6 +154,7 @@ class Ability
:create_merge_request,
:create_wiki,
:manage_builds,
:download_build_artifacts,
:push_code
]
end
......
......@@ -89,6 +89,7 @@ class ApplicationSetting < ActiveRecord::Base
restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'],
shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
max_artifacts_size: Settings.gitlab_ci['max_artifacts_size'],
)
end
......
......@@ -39,6 +39,8 @@ module Ci
scope :ignore_failures, ->() { where(allow_failure: false) }
scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) }
mount_uploader :artifacts_file, ArtifactUploader
acts_as_taggable
# To prevent db load megabytes of data from trace
......@@ -217,6 +219,14 @@ module Ci
"#{dir_to_trace}/#{id}.log"
end
def token
project.token
end
def valid_token? token
project.valid_token? token
end
def target_url
Gitlab::Application.routes.url_helpers.
namespace_project_build_url(gl_project.namespace, gl_project, self)
......@@ -248,6 +258,13 @@ module Ci
pending? && !any_runners_online?
end
def download_url
if artifacts_file.exists?
Gitlab::Application.routes.url_helpers.
download_namespace_project_build_path(gl_project.namespace, gl_project, self)
end
end
private
def yaml_variables
......
......@@ -92,4 +92,8 @@ class CommitStatus < ActiveRecord::Base
def show_warning?
false
end
def download_url
nil
end
end
# encoding: utf-8
class ArtifactUploader < CarrierWave::Uploader::Base
storage :file
attr_accessor :build, :field
def self.artifacts_path
File.expand_path('shared/artifacts/', Rails.root)
end
def self.artifacts_upload_path
File.expand_path('shared/artifacts/tmp/uploads/', Rails.root)
end
def self.artifacts_cache_path
File.expand_path('shared/artifacts/tmp/cache/', Rails.root)
end
def initialize(build, field)
@build, @field = build, field
end
def artifacts_path
File.join(build.created_at.utc.strftime('%Y_%m'), build.project.id.to_s, build.id.to_s)
end
def store_dir
File.join(ArtifactUploader.artifacts_path, artifacts_path)
end
def cache_dir
File.join(ArtifactUploader.artifacts_cache_path, artifacts_path)
end
def file_storage?
self.class.storage == CarrierWave::Storage::File
end
def exists?
file.try(:exists?)
end
def move_to_cache
true
end
def move_to_store
true
end
end
......@@ -139,5 +139,10 @@
= f.check_box :shared_runners_enabled
Enable shared runners for a new projects
.form-group
= f.label :max_artifacts_size, 'Maximum artifacts size (MB)', class: 'control-label col-sm-2'
.col-sm-10
= f.number_field :max_artifacts_size, class: 'form-control'
.form-actions
= f.submit 'Save', class: 'btn btn-primary'
......@@ -87,6 +87,9 @@
Test coverage
%h1 #{@build.coverage}%
- if current_user && can?(current_user, :download_build_artifacts, @project) && @build.download_url
.build-widget.center
= link_to "Download artifacts", @build.download_url, class: 'btn btn-sm btn-primary'
.build-widget
%h4.title
......
......@@ -61,6 +61,9 @@
%td
.pull-right
- if current_user && can?(current_user, :download_build_artifacts, @project) && commit_status.download_url
= link_to commit_status.download_url, title: 'Download artifacts' do
%i.fa.fa-download
- if current_user && can?(current_user, :manage_builds, commit_status.gl_project)
- if commit_status.active?
- if commit_status.cancel_url
......
......@@ -186,6 +186,7 @@ Settings.gitlab_ci['all_broken_builds'] = true if Settings.gitlab_ci['all_br
Settings.gitlab_ci['add_pusher'] = false if Settings.gitlab_ci['add_pusher'].nil?
Settings.gitlab_ci['url'] ||= Settings.send(:build_gitlab_ci_url)
Settings.gitlab_ci['builds_path'] = File.expand_path(Settings.gitlab_ci['builds_path'] || "builds/", Rails.root)
Settings.gitlab_ci['max_artifacts_size'] ||= 100 # in megabytes
#
# Reply by email
......
......@@ -611,6 +611,7 @@ Gitlab::Application.routes.draw do
member do
get :status
post :cancel
get :download
post :retry
end
end
......
class AddArtifactsFileToBuilds < ActiveRecord::Migration
def change
add_column :ci_builds, :artifacts_file, :text
end
end
class AddMaxArtifactsSizeToApplicationSettings < ActiveRecord::Migration
def change
add_column :application_settings, :max_artifacts_size, :integer, default: 100, null: false
end
end
......@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20151105094515) do
ActiveRecord::Schema.define(version: 20151109100728) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
......@@ -48,6 +48,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do
t.text "help_page_text"
t.string "admin_notification_email"
t.boolean "shared_runners_enabled", default: true, null: false
t.integer "max_artifacts_size", default: 100, null: false
end
create_table "audit_events", force: true do |t|
......@@ -108,6 +109,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do
t.string "type"
t.string "target_url"
t.string "description"
t.text "artifacts_file"
end
add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree
......
......@@ -141,6 +141,7 @@ job_name:
| tags | optional | Defines a list of tags which are used to select runner |
| allow_failure | optional | Allow build to fail. Failed build doesn't contribute to commit status |
| when | optional | Define when to run build. Can be `on_success`, `on_failure` or `always` |
| artifacts | optional | Define list build artifacts |
### script
`script` is a shell script which is executed by runner. The shell script is prepended with `before_script`.
......@@ -258,6 +259,35 @@ The above script will:
1. Execute `cleanup_build` only when the `build` failed,
2. Always execute `cleanup` as the last step in pipeline.
### artifacts
`artifacts` is used to specify list of files and directories which should be attached to build after success.
1. Send all files in `binaries` and `.config`:
```
artifacts:
paths:
- binaries/
- .config
```
2. Send all git untracked files:
```
artifacts:
untracked: true
```
3. Send all git untracked files and files in `binaries`:
```
artifacts:
untracked: true
paths:
- binaries/
```
The artifacts will be send after the build success to GitLab and will be accessible in GitLab interface to download.
This feature requires GitLab Runner v0.7.0 or higher.
## Validate the .gitlab-ci.yml
Each instance of GitLab CI has an embedded debug tool called Lint.
You can find the link to the Lint in the project's settings page or use short url `/lint`.
......
......@@ -246,6 +246,9 @@ We recommend using a PostgreSQL database. For MySQL check [MySQL setup guide](da
# Change the permissions of the directory where CI build traces are stored
sudo chmod -R u+rwX builds/
# Change the permissions of the directory where CI artifacts are stored
sudo chmod -R u+rwX shared/artifacts/
# Copy the example Unicorn config
sudo -u git -H cp config/unicorn.rb.example config/unicorn.rb
......
......@@ -29,7 +29,8 @@ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production
```
Also you can choose what should be backed up by adding environment variable SKIP. Available options: db,
uploads (attachments), repositories, builds(CI build output logs). Use a comma to specify several options at the same time.
uploads (attachments), repositories, builds(CI build output logs), artifacts (CI build artifacts).
Use a comma to specify several options at the same time.
```
sudo gitlab-rake gitlab:backup:create SKIP=db,uploads
......
......@@ -133,6 +133,12 @@ module API
authorize! :admin_project, user_project
end
def require_gitlab_workhorse!
unless env['HTTP_GITLAB_WORKHORSE'].present?
forbidden!('Request should be executed via GitLab Workhorse')
end
end
def can?(object, action, subject)
abilities.allowed?(object, action, subject)
end
......@@ -234,6 +240,10 @@ module API
render_api_error!(message || '409 Conflict', 409)
end
def file_to_large!
render_api_error!('413 Request Entity Too Large', 413)
end
def render_validation_error!(model)
if model.errors.any?
render_api_error!(model.errors.messages || '400 Bad Request', 400)
......@@ -282,6 +292,44 @@ module API
end
end
# file helpers
def uploaded_file!(field, uploads_path)
if params[field]
bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename)
return params[field]
end
# sanitize file paths
# this requires all paths to exist
required_attributes! %W(#{field}.path)
uploads_path = File.realpath(uploads_path)
file_path = File.realpath(params["#{field}.path"])
bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
UploadedFile.new(
file_path,
params["#{field}.name"],
params["#{field}.type"] || 'application/octet-stream',
)
end
def present_file!(path, filename, content_type = 'application/octet-stream')
filename ||= File.basename(path)
header['Content-Disposition'] = "attachment; filename=#{filename}"
header['Content-Transfer-Encoding'] = 'binary'
content_type content_type
# Support download acceleration
case headers['X-Sendfile-Type']
when 'X-Sendfile'
header['X-Sendfile'] = path
body
else
file FileStreamer.new(path)
end
end
private
def add_pagination_headers(paginated, per_page)
......
require 'backup/files'
module Backup
class Artifacts < Files
def initialize
super('artifacts', ArtifactUploader.artifacts_path)
end
def create_files_dir
Dir.mkdir(app_files_dir, 0700)
end
end
end
......@@ -150,7 +150,7 @@ module Backup
private
def backup_contents
folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"]
folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "artifacts.tar.gz", "backup_information.yml"]
end
def folders_to_backup
......
......@@ -27,6 +27,7 @@ module Ci
helpers Helpers
helpers ::API::Helpers
helpers Gitlab::CurrentSettings
mount Builds
mount Commits
......
......@@ -47,6 +47,106 @@ module Ci
build.drop
end
end
# Authorize artifacts uploading for build - Runners only
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# filesize (optional) - the size of uploaded file
# Example Request:
# POST /builds/:id/artifacts/authorize
post ":id/artifacts/authorize" do
require_gitlab_workhorse!
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
forbidden!('build is not running') unless build.running?
if params[:filesize]
file_size = params[:filesize].to_i
file_to_large! unless file_size < max_artifacts_size
end
status 200
{ TempPath: ArtifactUploader.artifacts_upload_path }
end
# Upload artifacts to build - Runners only
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# file (required) - The uploaded file
# Parameters (accelerated by GitLab Workhorse):
# file.path - path to locally stored body (generated by Workhorse)
# file.name - real filename as send in Content-Disposition
# file.type - real content type as send in Content-Type
# Headers:
# BUILD-TOKEN (required) - The build authorization token, the same as token
# Body:
# The file content
#
# Example Request:
# POST /builds/:id/artifacts
post ":id/artifacts" do
require_gitlab_workhorse!
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
forbidden!('build is not running') unless build.running?
file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path)
file_to_large! unless file.size < max_artifacts_size
if build.update_attributes(artifacts_file: file)
present build, with: Entities::Build
else
render_validation_error!(build)
end
end
# Download the artifacts file from build - Runners only
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# Headers:
# BUILD-TOKEN (required) - The build authorization token, the same as token
# Example Request:
# GET /builds/:id/artifacts
get ":id/artifacts" do
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
artifacts_file = build.artifacts_file
unless artifacts_file.file_storage?
return redirect_to build.artifacts_file.url
end
unless artifacts_file.exists?
not_found!
end
present_file!(artifacts_file.path, artifacts_file.filename)
end
# Remove the artifacts file from build
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# Headers:
# BUILD-TOKEN (required) - The build authorization token, the same as token
# Example Request:
# DELETE /builds/:id/artifacts
delete ":id/artifacts" do
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
build.remove_artifacts_file!
end
end
end
end
......
......@@ -11,10 +11,16 @@ module Ci
expose :builds
end
class ArtifactFile < Grape::Entity
expose :filename, :size
end
class Build < Grape::Entity
expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url,
:before_sha, :allow_git_fetch, :project_name
expose :name, :token, :stage
expose :options do |model|
model.options
end
......@@ -24,6 +30,7 @@ module Ci
end
expose :variables
expose :artifacts_file, using: ArtifactFile
end
class Runner < Grape::Entity
......
module Ci
module API
module Helpers
BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN"
BUILD_TOKEN_PARAM = :token
UPDATE_RUNNER_EVERY = 60
def authenticate_runners!
......@@ -15,6 +17,11 @@ module Ci
forbidden! unless project.valid_token?(params[:project_token])
end
def authenticate_build_token!(build)
token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
forbidden! unless token && build.valid_token?(token)
end
def update_runner_last_contact
# Use a random threshold to prevent beating DB updates
contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
......@@ -32,6 +39,10 @@ module Ci
info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"])
current_runner.update(info)
end
def max_artifacts_size
current_application_settings.max_artifacts_size.megabytes.to_i
end
end
end
end
......@@ -5,7 +5,7 @@ module Ci
DEFAULT_STAGES = %w(build test deploy)
DEFAULT_STAGE = 'test'
ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when]
ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts]
attr_reader :before_script, :image, :services, :variables, :path
......@@ -77,7 +77,8 @@ module Ci
when: job[:when] || 'on_success',
options: {
image: job[:image] || @image,
services: job[:services] || @services
services: job[:services] || @services,
artifacts: job[:artifacts]
}.compact
}
end
......@@ -159,7 +160,17 @@ module Ci
raise ValidationError, "#{name} job: except parameter should be an array of strings"
end
if job[:allow_failure] && !job[:allow_failure].in?([true, false])
if job[:artifacts]
if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked])
raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean"
end
if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
end
end
if job[:allow_failure] && !validate_boolean(job[:allow_failure])
raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
end
......@@ -182,6 +193,10 @@ module Ci
value.is_a?(String) || value.is_a?(Symbol)
end
def validate_boolean(value)
value.in?([true, false])
end
def process?(only_params, except_params, ref, tag)
if only_params.present?
return false unless matching?(only_params, ref, tag)
......
class FileStreamer #:nodoc:
attr_reader :to_path
def initialize(path)
@to_path = path
end
# Stream the file's contents if Rack::Sendfile isn't present.
def each
File.open(to_path, 'rb') do |file|
while chunk = file.read(16384)
yield chunk
end
end
end
end
......@@ -25,6 +25,7 @@ module Gitlab
session_expire_delay: Settings.gitlab['session_expire_delay'],
import_sources: Settings.gitlab['import_sources'],
shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'],
)
end
......
......@@ -131,6 +131,22 @@ server {
return 418;
}
# Build artifacts should be submitted to this location
location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
client_max_body_size 0;
# 'Error' 418 is a hack to re-use the @gitlab-workhorse block
error_page 418 = @gitlab-workhorse;
return 418;
}
# Build artifacts should be submitted to this location
location ~ /ci/api/v1/builds/[0-9]+/artifacts {
client_max_body_size 0;
# 'Error' 418 is a hack to re-use the @gitlab-workhorse block
error_page 418 = @gitlab-workhorse;
return 418;
}
location @gitlab-workhorse {
## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack.
......
......@@ -178,6 +178,22 @@ server {
return 418;
}
# Build artifacts should be submitted to this location
location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
client_max_body_size 0;
# 'Error' 418 is a hack to re-use the @gitlab-workhorse block
error_page 418 = @gitlab-workhorse;
return 418;
}
# Build artifacts should be submitted to this location
location ~ /ci/api/v1/builds/[0-9]+/artifacts {
client_max_body_size 0;
# 'Error' 418 is a hack to re-use the @gitlab-workhorse block
error_page 418 = @gitlab-workhorse;
return 418;
}
location @gitlab-workhorse {
## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack.
......
......@@ -12,6 +12,7 @@ namespace :gitlab do
Rake::Task["gitlab:backup:repo:create"].invoke
Rake::Task["gitlab:backup:uploads:create"].invoke
Rake::Task["gitlab:backup:builds:create"].invoke
Rake::Task["gitlab:backup:artifacts:create"].invoke
backup = Backup::Manager.new
backup.pack
......@@ -32,6 +33,7 @@ namespace :gitlab do
Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts")
Rake::Task["gitlab:shell:setup"].invoke
backup.cleanup
......@@ -113,6 +115,25 @@ namespace :gitlab do
end
end
namespace :artifacts do
task create: :environment do
$progress.puts "Dumping artifacts ... ".blue
if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
$progress.puts "[SKIPPED]".cyan
else
Backup::Artifacts.new.dump
$progress.puts "done".green
end
end
task restore: :environment do
$progress.puts "Restoring artifacts ... ".blue
Backup::Artifacts.new.restore
$progress.puts "done".green
end
end
def configure_cron_mode
if ENV['CRON']
# We need an object we can say 'puts' and 'print' to; let's use a
......
require "tempfile"
require "fileutils"
# Taken from: Rack::Test::UploadedFile
class UploadedFile
# The filename, *not* including the path, of the "uploaded" file
attr_reader :original_filename
# The tempfile
attr_reader :tempfile
# The content type of the "uploaded" file
attr_accessor :content_type
def initialize(path, filename, content_type = "text/plain")
raise "#{path} file does not exist" unless ::File.exist?(path)
@content_type = content_type
@original_filename = filename || ::File.basename(path)
@tempfile = File.new(path, 'rb')
end
def path
@tempfile.path
end
alias_method :local_path, :path
def method_missing(method_name, *args, &block) #:nodoc:
@tempfile.__send__(method_name, *args, &block)
end
def respond_to?(method_name, include_private = false) #:nodoc:
@tempfile.respond_to?(method_name, include_private) || super
end
end
require 'spec_helper'
describe "Builds" do
let(:artifacts_file) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') }
before do
login_as(:user)
@commit = FactoryGirl.create :ci_commit
......@@ -66,6 +68,15 @@ describe "Builds" do
it { expect(page).to have_content @commit.sha[0..7] }
it { expect(page).to have_content @commit.git_commit_message }
it { expect(page).to have_content @commit.git_author_name }
context "Download artifacts" do
before do
@build.update_attributes(artifacts_file: artifacts_file)
visit namespace_project_build_path(@gl_project.namespace, @gl_project, @build)
end
it { expect(page).to have_content 'Download artifacts' }
end
end
describe "POST /:project/builds/:id/cancel" do
......@@ -90,4 +101,14 @@ describe "Builds" do
it { expect(page).to have_content 'pending' }
it { expect(page).to have_content 'Cancel' }
end
describe "GET /:project/builds/:id/download" do
before do
@build.update_attributes(artifacts_file: artifacts_file)
visit namespace_project_build_path(@gl_project.namespace, @gl_project, @build)
click_link 'Download artifacts'
end
it { expect(page.response_headers['Content-Type']).to eq(artifacts_file.content_type) }
end
end
......@@ -19,7 +19,7 @@ describe "Commits" do
stub_ci_commit_to_return_yaml_file
end
describe "GET /:project/commits/:sha" do
describe "GET /:project/commits/:sha/ci" do
before do
visit ci_status_path(@commit)
end
......@@ -29,6 +29,20 @@ describe "Commits" do
it { expect(page).to have_content @commit.git_author_name }
end
context "Download artifacts" do
let(:artifacts_file) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') }
before do
@build.update_attributes(artifacts_file: artifacts_file)
end
it do
visit ci_status_path(@commit)
click_on "Download artifacts"
expect(page.response_headers['Content-Type']).to eq(artifacts_file.content_type)
end
end
describe "Cancel all builds" do
it "cancels commit" do
visit ci_status_path(@commit)
......
......@@ -333,6 +333,43 @@ module Ci
end
end
describe "Artifacts" do
it "returns artifacts when defined" do
config = YAML.dump({
image: "ruby:2.1",
services: ["mysql"],
before_script: ["pwd"],
rspec: {
artifacts: { paths: ["logs/", "binaries/"], untracked: true },
script: "rspec"
}
})
config_processor = GitlabCiYamlProcessor.new(config)
expect(config_processor.builds_for_stage_and_ref("test", "master").size).to eq(1)
expect(config_processor.builds_for_stage_and_ref("test", "master").first).to eq({
except: nil,
stage: "test",
stage_idx: 1,
name: :rspec,
only: nil,
commands: "pwd\nrspec",
tag_list: [],
options: {
image: "ruby:2.1",
services: ["mysql"],
artifacts: {
paths: ["logs/", "binaries/"],
untracked: true
}
},
when: "on_success",
allow_failure: false
})
end
end
describe "Error handling" do
it "indicates that object is invalid" do
expect{GitlabCiYamlProcessor.new("invalid_yaml\n!ccdvlf%612334@@@@")}.to raise_error(GitlabCiYamlProcessor::ValidationError)
......@@ -491,6 +528,20 @@ module Ci
GitlabCiYamlProcessor.new(config, path)
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: when parameter should be on_success, on_failure or always")
end
it "returns errors if job artifacts:untracked is not an array of strings" do
config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: { untracked: "string" } } })
expect do
GitlabCiYamlProcessor.new(config)
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts:untracked parameter should be an boolean")
end
it "returns errors if job artifacts:paths is not an array of strings" do
config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: { paths: "string" } } })
expect do
GitlabCiYamlProcessor.new(config)
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts:paths parameter should be an array of strings")
end
end
end
end
......@@ -400,4 +400,19 @@ describe Ci::Build do
end
end
end
describe :download_url do
subject { build.download_url }
it "should be nil if artifact doesn't exist" do
build.update_attributes(artifacts_file: nil)
is_expected.to be_nil
end
it 'should be nil if artifact exist' do
gif = fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif')
build.update_attributes(artifacts_file: gif)
is_expected.to_not be_nil
end
end
end
......@@ -343,8 +343,9 @@ describe API::API, api: true do
end.to change{ user.keys.count }.by(1)
end
it "should raise error for invalid ID" do
expect{post api("/users/ASDF/keys", admin) }.to raise_error(ActionController::RoutingError)
it "should return 405 for invalid ID" do
post api("/users/ASDF/keys", admin)
expect(response.status).to eq(405)
end
end
......@@ -374,9 +375,9 @@ describe API::API, api: true do
expect(json_response.first['title']).to eq(key.title)
end
it "should return 404 for invalid ID" do
it "should return 405 for invalid ID" do
get api("/users/ASDF/keys", admin)
expect(response.status).to eq(404)
expect(response.status).to eq(405)
end
end
end
......@@ -434,7 +435,8 @@ describe API::API, api: true do
end
it "should raise error for invalid ID" do
expect{post api("/users/ASDF/emails", admin) }.to raise_error(ActionController::RoutingError)
post api("/users/ASDF/emails", admin)
expect(response.status).to eq(405)
end
end
......@@ -465,7 +467,8 @@ describe API::API, api: true do
end
it "should raise error for invalid ID" do
expect{put api("/users/ASDF/emails", admin) }.to raise_error(ActionController::RoutingError)
put api("/users/ASDF/emails", admin)
expect(response.status).to eq(405)
end
end
end
......
......@@ -41,7 +41,7 @@ describe Ci::API::API do
it "should return 404 error if no builds for specific runner" do
commit = FactoryGirl.create(:ci_commit, gl_project: shared_gl_project)
FactoryGirl.create(:ci_build, commit: commit, status: 'pending' )
FactoryGirl.create(:ci_build, commit: commit, status: 'pending')
post ci_api("/builds/register"), token: runner.token
......@@ -50,7 +50,7 @@ describe Ci::API::API do
it "should return 404 error if no builds for shared runner" do
commit = FactoryGirl.create(:ci_commit, gl_project: gl_project)
FactoryGirl.create(:ci_build, commit: commit, status: 'pending' )
FactoryGirl.create(:ci_build, commit: commit, status: 'pending')
post ci_api("/builds/register"), token: shared_runner.token
......@@ -79,7 +79,7 @@ describe Ci::API::API do
{ "key" => "CI_BUILD_NAME", "value" => "spinach", "public" => true },
{ "key" => "CI_BUILD_STAGE", "value" => "test", "public" => true },
{ "key" => "DB_NAME", "value" => "postgres", "public" => true },
{ "key" => "SECRET_KEY", "value" => "secret_value", "public" => false },
{ "key" => "SECRET_KEY", "value" => "secret_value", "public" => false }
])
end
......@@ -122,5 +122,194 @@ describe Ci::API::API do
expect(build.reload.trace).to eq 'hello_world'
end
end
context "Artifacts" do
let(:file_upload) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') }
let(:file_upload2) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/gif') }
let(:commit) { FactoryGirl.create(:ci_commit, gl_project: gl_project) }
let(:build) { FactoryGirl.create(:ci_build, commit: commit, runner_id: runner.id) }
let(:authorize_url) { ci_api("/builds/#{build.id}/artifacts/authorize") }
let(:post_url) { ci_api("/builds/#{build.id}/artifacts") }
let(:delete_url) { ci_api("/builds/#{build.id}/artifacts") }
let(:get_url) { ci_api("/builds/#{build.id}/artifacts") }
let(:headers) { { "GitLab-Workhorse" => "1.0" } }
let(:headers_with_token) { headers.merge(Ci::API::Helpers::BUILD_TOKEN_HEADER => build.project.token) }
describe "POST /builds/:id/artifacts/authorize" do
context "should authorize posting artifact to running build" do
before do
build.run!
end
it "using token as parameter" do
post authorize_url, { token: build.project.token }, headers
expect(response.status).to eq(200)
expect(json_response["TempPath"]).to_not be_nil
end
it "using token as header" do
post authorize_url, {}, headers_with_token
expect(response.status).to eq(200)
expect(json_response["TempPath"]).to_not be_nil
end
end
context "should fail to post too large artifact" do
before do
build.run!
end
it "using token as parameter" do
settings = Gitlab::CurrentSettings::current_application_settings
settings.update_attributes(max_artifacts_size: 0)
post authorize_url, { token: build.project.token, filesize: 100 }, headers
expect(response.status).to eq(413)
end
it "using token as header" do
settings = Gitlab::CurrentSettings::current_application_settings
settings.update_attributes(max_artifacts_size: 0)
post authorize_url, { filesize: 100 }, headers_with_token
expect(response.status).to eq(413)
end
end
context "should get denied" do
it do
post authorize_url, { token: 'invalid', filesize: 100 }
expect(response.status).to eq(403)
end
end
end
describe "POST /builds/:id/artifacts" do
context "Disable sanitizer" do
before do
# by configuring this path we allow to pass temp file from any path
allow(ArtifactUploader).to receive(:artifacts_upload_path).and_return('/')
end
context "should post artifact to running build" do
before do
build.run!
end
it "uses regual file post" do
upload_artifacts(file_upload, headers_with_token, false)
expect(response.status).to eq(201)
expect(json_response["artifacts_file"]["filename"]).to eq(file_upload.original_filename)
end
it "uses accelerated file post" do
upload_artifacts(file_upload, headers_with_token, true)
expect(response.status).to eq(201)
expect(json_response["artifacts_file"]["filename"]).to eq(file_upload.original_filename)
end
it "updates artifact" do
upload_artifacts(file_upload, headers_with_token)
upload_artifacts(file_upload2, headers_with_token)
expect(response.status).to eq(201)
expect(json_response["artifacts_file"]["filename"]).to eq(file_upload2.original_filename)
end
end
context "should fail to post too large artifact" do
before do
build.run!
end
it do
settings = Gitlab::CurrentSettings::current_application_settings
settings.update_attributes(max_artifacts_size: 0)
upload_artifacts(file_upload, headers_with_token)
expect(response.status).to eq(413)
end
end
context "should fail to post artifacts without file" do
before do
build.run!
end
it do
post post_url, {}, headers_with_token
expect(response.status).to eq(400)
end
end
context "should fail to post artifacts without GitLab-Workhorse" do
before do
build.run!
end
it do
post post_url, { token: build.project.token }, {}
expect(response.status).to eq(403)
end
end
end
context "should fail to post artifacts for outside of tmp path" do
before do
# by configuring this path we allow to pass file from @tmpdir only
# but all temporary files are stored in system tmp directory
@tmpdir = Dir.mktmpdir
allow(ArtifactUploader).to receive(:artifacts_upload_path).and_return(@tmpdir)
build.run!
end
after do
FileUtils.remove_entry @tmpdir
end
it do
upload_artifacts(file_upload, headers_with_token)
expect(response.status).to eq(400)
end
end
def upload_artifacts(file, headers = {}, accelerated = true)
if accelerated
post post_url, {
'file.path' => file.path,
'file.name' => file.original_filename
}, headers
else
post post_url, { file: file }, headers
end
end
end
describe "DELETE /builds/:id/artifacts" do
before do
build.run!
post delete_url, token: build.project.token, file: file_upload
end
it "should delete artifact build" do
build.success
delete delete_url, token: build.project.token
expect(response.status).to eq(200)
end
end
describe "GET /builds/:id/artifacts" do
before do
build.run!
end
it "should download artifact" do
build.update_attributes(artifacts_file: file_upload)
get get_url, token: build.project.token
expect(response.status).to eq(200)
end
it "should fail to download if no artifact uploaded" do
get get_url, token: build.project.token
expect(response.status).to eq(404)
end
end
end
end
end
......@@ -16,7 +16,7 @@ describe 'gitlab:app namespace rake task' do
end
def reenable_backup_sub_tasks
%w{db repo uploads builds}.each do |subtask|
%w{db repo uploads builds artifacts}.each do |subtask|
Rake::Task["gitlab:backup:#{subtask}:create"].reenable
end
end
......@@ -56,6 +56,7 @@ describe 'gitlab:app namespace rake task' do
expect(Rake::Task["gitlab:backup:repo:restore"]).to receive(:invoke)
expect(Rake::Task["gitlab:backup:builds:restore"]).to receive(:invoke)
expect(Rake::Task["gitlab:backup:uploads:restore"]).to receive(:invoke)
expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive(:invoke)
expect(Rake::Task["gitlab:shell:setup"]).to receive(:invoke)
expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
end
......@@ -113,19 +114,20 @@ describe 'gitlab:app namespace rake task' do
it 'should set correct permissions on the tar contents' do
tar_contents, exit_status = Gitlab::Popen.popen(
%W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz}
%W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz}
)
expect(exit_status).to eq(0)
expect(tar_contents).to match('db/')
expect(tar_contents).to match('uploads.tar.gz')
expect(tar_contents).to match('repositories/')
expect(tar_contents).to match('builds.tar.gz')
expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz)\/$/)
expect(tar_contents).to match('artifacts.tar.gz')
expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz)\/$/)
end
it 'should delete temp directories' do
temp_dirs = Dir.glob(
File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds}')
File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts}')
)
expect(temp_dirs).to be_empty
......@@ -161,12 +163,13 @@ describe 'gitlab:app namespace rake task' do
it "does not contain skipped item" do
tar_contents, _exit_status = Gitlab::Popen.popen(
%W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz}
%W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz}
)
expect(tar_contents).to match('db/')
expect(tar_contents).to match('uploads.tar.gz')
expect(tar_contents).to match('builds.tar.gz')
expect(tar_contents).to match('artifacts.tar.gz')
expect(tar_contents).not_to match('repositories/')
end
......@@ -178,6 +181,7 @@ describe 'gitlab:app namespace rake task' do
expect(Rake::Task["gitlab:backup:db:restore"]).to receive :invoke
expect(Rake::Task["gitlab:backup:repo:restore"]).not_to receive :invoke
expect(Rake::Task["gitlab:backup:builds:restore"]).to receive :invoke
expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive :invoke
expect(Rake::Task["gitlab:shell:setup"]).to receive :invoke
expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment