Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
48ae718c
Commit
48ae718c
authored
Nov 09, 2020
by
Shinya Maeda
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Introduce Auto Rollback facility
This commit introduces the auto rollback facility.
parent
0be059c2
Changes
14
Hide whitespace changes
Inline
Side-by-side
Showing
14 changed files
with
335 additions
and
8 deletions
+335
-8
app/models/deployment.rb
app/models/deployment.rb
+6
-2
app/models/environment.rb
app/models/environment.rb
+1
-4
changelogs/unreleased/introduce-auto-rollback-service.yml
changelogs/unreleased/introduce-auto-rollback-service.yml
+5
-0
db/post_migrate/20201112145311_add_index_on_sha_for_initial_deployments.rb
...0201112145311_add_index_on_sha_for_initial_deployments.rb
+21
-0
db/schema_migrations/20201112145311
db/schema_migrations/20201112145311
+1
-0
db/structure.sql
db/structure.sql
+1
-1
ee/app/services/deployments/auto_rollback_service.rb
ee/app/services/deployments/auto_rollback_service.rb
+54
-0
ee/app/workers/all_queues.yml
ee/app/workers/all_queues.yml
+8
-0
ee/app/workers/deployments/auto_rollback_worker.rb
ee/app/workers/deployments/auto_rollback_worker.rb
+18
-0
ee/spec/services/deployments/auto_rollback_service_spec.rb
ee/spec/services/deployments/auto_rollback_service_spec.rb
+129
-0
ee/spec/workers/deployments/auto_rollback_worker_spec.rb
ee/spec/workers/deployments/auto_rollback_worker_spec.rb
+32
-0
lib/gitlab/application_rate_limiter.rb
lib/gitlab/application_rate_limiter.rb
+2
-1
spec/models/deployment_spec.rb
spec/models/deployment_spec.rb
+30
-0
spec/models/environment_spec.rb
spec/models/environment_spec.rb
+27
-0
No files found.
app/models/deployment.rb
View file @
48ae718c
...
...
@@ -41,8 +41,8 @@ class Deployment < ApplicationRecord
scope
:visible
,
->
{
where
(
status:
%i[running success failed canceled]
)
}
scope
:stoppable
,
->
{
where
.
not
(
on_stop:
nil
).
where
.
not
(
deployable_id:
nil
).
success
}
scope
:active
,
->
{
where
(
status:
%i[created running]
)
}
scope
:older_than
,
->
(
deployment
)
{
where
(
'id < ?'
,
deployment
.
id
)
}
scope
:with_deployable
,
->
{
includes
(
:deployable
).
where
(
'deployable_id IS NOT NULL'
)
}
scope
:older_than
,
->
(
deployment
)
{
where
(
'
deployments.
id < ?'
,
deployment
.
id
)
}
scope
:with_deployable
,
->
{
joins
(
'INNER JOIN ci_builds ON ci_builds.id = deployments.deployable_id'
).
preload
(
:deployable
)
}
FINISHED_STATUSES
=
%i[success failed canceled]
.
freeze
...
...
@@ -149,6 +149,10 @@ class Deployment < ApplicationRecord
project
.
repository
.
delete_refs
(
*
ref_paths
.
flatten
)
end
end
def
latest_for_sha
(
sha
)
where
(
sha:
sha
).
order
(
id: :desc
).
take
end
end
def
commit
...
...
app/models/environment.rb
View file @
48ae718c
...
...
@@ -60,6 +60,7 @@ class Environment < ApplicationRecord
addressable_url:
true
delegate
:stop_action
,
:manual_actions
,
to: :last_deployment
,
allow_nil:
true
delegate
:auto_rollback_enabled?
,
to: :project
scope
:available
,
->
{
with_state
(
:available
)
}
scope
:stopped
,
->
{
with_state
(
:stopped
)
}
...
...
@@ -240,10 +241,6 @@ class Environment < ApplicationRecord
def
cancel_deployment_jobs!
jobs
=
active_deployments
.
with_deployable
jobs
.
each
do
|
deployment
|
# guard against data integrity issues,
# for example https://gitlab.com/gitlab-org/gitlab/-/issues/218659#note_348823660
next
unless
deployment
.
deployable
Gitlab
::
OptimisticLocking
.
retry_lock
(
deployment
.
deployable
)
do
|
deployable
|
deployable
.
cancel!
if
deployable
&
.
cancelable?
end
...
...
changelogs/unreleased/introduce-auto-rollback-service.yml
0 → 100644
View file @
48ae718c
---
title
:
Add database index for deployment rollback targets
merge_request
:
47159
author
:
type
:
performance
db/post_migrate/20201112145311_add_index_on_sha_for_initial_deployments.rb
0 → 100644
View file @
48ae718c
# frozen_string_literal: true
class
AddIndexOnShaForInitialDeployments
<
ActiveRecord
::
Migration
[
6.0
]
include
Gitlab
::
Database
::
MigrationHelpers
DOWNTIME
=
false
NEW_INDEX_NAME
=
'index_deployments_on_environment_status_sha'
OLD_INDEX_NAME
=
'index_deployments_on_environment_id_and_status'
disable_ddl_transaction!
def
up
add_concurrent_index
:deployments
,
%i[environment_id status sha]
,
name:
NEW_INDEX_NAME
remove_concurrent_index_by_name
:deployments
,
OLD_INDEX_NAME
end
def
down
add_concurrent_index
:deployments
,
%i[environment_id status]
,
name:
OLD_INDEX_NAME
remove_concurrent_index_by_name
:services
,
NEW_INDEX_NAME
end
end
db/schema_migrations/20201112145311
0 → 100644
View file @
48ae718c
085bb21bdbe3d062b3000d63c111aab5ba75c7e049c32779cccac5c320583759
\ No newline at end of file
db/structure.sql
View file @
48ae718c
...
...
@@ -20697,7 +20697,7 @@ CREATE INDEX index_deployments_on_environment_id_and_id ON deployments USING btr
CREATE
INDEX
index_deployments_on_environment_id_and_iid_and_project_id
ON
deployments
USING
btree
(
environment_id
,
iid
,
project_id
);
CREATE
INDEX
index_deployments_on_environment_
id_and_status
ON
deployments
USING
btree
(
environment_id
,
status
);
CREATE
INDEX
index_deployments_on_environment_
status_sha
ON
deployments
USING
btree
(
environment_id
,
status
,
sha
);
CREATE
INDEX
index_deployments_on_id_and_status_and_created_at
ON
deployments
USING
btree
(
id
,
status
,
created_at
);
...
...
ee/app/services/deployments/auto_rollback_service.rb
0 → 100644
View file @
48ae718c
# frozen_string_literal: true
module
Deployments
class
AutoRollbackService
<
::
BaseService
def
execute
(
environment
)
result
=
validate
(
environment
)
return
result
unless
result
[
:status
]
==
:success
deployment
=
find_rollback_target
(
environment
)
return
error
(
'Failed to find a rollback target.'
)
unless
deployment
new_deployment
=
rollback_to
(
deployment
)
success
(
deployment:
new_deployment
)
end
private
def
validate
(
environment
)
unless
environment
.
auto_rollback_enabled?
return
error
(
'Auto Rollback is not enabled on the project.'
)
end
if
environment
.
has_running_deployments?
return
error
(
'There are running deployments on the environment.'
)
end
if
::
Gitlab
::
ApplicationRateLimiter
.
throttled?
(
:auto_rollback_deployment
,
scope:
[
environment
])
return
error
(
'Auto Rollback was recentlly trigged for the environment. It will be re-activated after a minute.'
)
end
success
end
def
find_rollback_target
(
environment
)
current_deployment
=
environment
.
last_deployment
return
unless
current_deployment
previous_commit_ids
=
current_deployment
.
commit
&
.
parent_ids
return
unless
previous_commit_ids
rollback_target
=
environment
.
successful_deployments
.
with_deployable
.
latest_for_sha
(
previous_commit_ids
)
return
unless
rollback_target
&&
rollback_target
.
deployable
.
retryable?
rollback_target
end
def
rollback_to
(
deployment
)
Ci
::
Build
.
retry
(
deployment
.
deployable
,
deployment
.
deployed_by
).
deployment
end
end
end
ee/app/workers/all_queues.yml
View file @
48ae718c
...
...
@@ -299,6 +299,14 @@
:weight:
1
:idempotent:
:tags: []
-
:name: deployment:deployments_auto_rollback
:feature_category: :continuous_delivery
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight:
3
:idempotent:
true
:tags: []
-
:name: epics:epics_update_epics_dates
:feature_category: :epics
:has_external_dependencies:
...
...
ee/app/workers/deployments/auto_rollback_worker.rb
0 → 100644
View file @
48ae718c
# frozen_string_literal: true
module
Deployments
class
AutoRollbackWorker
include
ApplicationWorker
idempotent!
feature_category
:continuous_delivery
queue_namespace
:deployment
def
perform
(
environment_id
)
Environment
.
find_by_id
(
environment_id
).
try
do
|
environment
|
Deployments
::
AutoRollbackService
.
new
(
environment
.
project
,
nil
)
.
execute
(
environment
)
end
end
end
end
ee/spec/services/deployments/auto_rollback_service_spec.rb
0 → 100644
View file @
48ae718c
# frozen_string_literal: true
require
'spec_helper'
RSpec
.
describe
Deployments
::
AutoRollbackService
,
:clean_gitlab_redis_cache
do
let_it_be
(
:maintainer
)
{
create
(
:user
)
}
let_it_be
(
:project
,
refind:
true
)
{
create
(
:project
,
:repository
)
}
let_it_be
(
:environment
,
refind:
true
)
{
create
(
:environment
,
project:
project
)
}
let_it_be
(
:commits
)
{
project
.
repository
.
commits
(
'master'
,
limit:
2
)
}
let
(
:service
)
{
described_class
.
new
(
project
,
nil
)
}
before_all
do
project
.
add_maintainer
(
maintainer
)
project
.
update!
(
auto_rollback_enabled:
true
)
end
shared_examples_for
'rollback failure'
do
it
'returns an error'
do
expect
(
subject
[
:status
]).
to
eq
(
:error
)
expect
(
subject
[
:message
]).
to
eq
(
message
)
end
end
describe
'#execute'
do
subject
{
service
.
execute
(
environment
)
}
before
do
stub_licensed_features
(
auto_rollback:
true
)
commits
.
reverse_each
{
|
commit
|
create_deployment
(
commit
.
id
)
}
end
it
'successfully roll back a deployment'
do
expect
{
subject
}.
to
change
{
Deployment
.
count
}.
by
(
1
)
expect
(
subject
[
:status
]).
to
eq
(
:success
)
expect
(
subject
[
:deployment
].
sha
).
to
eq
(
commits
[
1
].
id
)
end
context
'when auto_rollback checkbox is disabled on the project'
do
before
do
environment
.
project
.
auto_rollback_enabled
=
false
end
it_behaves_like
'rollback failure'
do
let
(
:message
)
{
'Auto Rollback is not enabled on the project.'
}
end
end
context
'when project does not have an sufficient license'
do
before
do
stub_licensed_features
(
auto_rollback:
false
)
end
it_behaves_like
'rollback failure'
do
let
(
:message
)
{
'Auto Rollback is not enabled on the project.'
}
end
end
context
'when there are running deployments '
do
before
do
create
(
:deployment
,
:running
,
environment:
environment
)
end
it_behaves_like
'rollback failure'
do
let
(
:message
)
{
'There are running deployments on the environment.'
}
end
end
context
'when auto rollback was triggered recently'
do
before
do
allow
(
::
Gitlab
::
ApplicationRateLimiter
).
to
receive
(
:throttled?
)
{
true
}
end
it_behaves_like
'rollback failure'
do
let
(
:message
)
{
'Auto Rollback was recentlly trigged for the environment. It will be re-activated after a minute.'
}
end
end
context
'when there are no deployments on the environment'
do
before
do
environment
.
deployments
.
fast_destroy_all
end
it_behaves_like
'rollback failure'
do
let
(
:message
)
{
'Failed to find a rollback target.'
}
end
end
context
'when there are no deployed commits in the repository'
do
before
do
environment
.
last_deployment
.
update!
(
sha:
'not-exist'
)
end
it_behaves_like
'rollback failure'
do
let
(
:message
)
{
'Failed to find a rollback target.'
}
end
end
context
"when rollback target's deployable is not retryable"
do
before
do
environment
.
all_deployments
.
first
.
deployable
.
degenerate!
end
it_behaves_like
'rollback failure'
do
let
(
:message
)
{
'Failed to find a rollback target.'
}
end
end
context
"when the user who performed deployments is no longer a project member"
do
let
(
:external_user
)
{
create
(
:user
)
}
before
do
environment
.
all_deployments
.
first
.
deployable
.
update!
(
user:
external_user
)
end
it
'raises an error'
do
expect
{
subject
}.
to
raise_error
(
Gitlab
::
Access
::
AccessDeniedError
)
end
end
def
create_deployment
(
commit_id
)
attributes
=
{
project:
project
,
ref:
'master'
,
user:
maintainer
}
pipeline
=
create
(
:ci_pipeline
,
:success
,
sha:
commit_id
,
**
attributes
)
build
=
create
(
:ci_build
,
:success
,
pipeline:
pipeline
,
environment:
environment
.
name
,
**
attributes
)
create
(
:deployment
,
:success
,
environment:
environment
,
deployable:
build
,
sha:
commit_id
,
**
attributes
)
end
end
end
ee/spec/workers/deployments/auto_rollback_worker_spec.rb
0 → 100644
View file @
48ae718c
# frozen_string_literal: true
require
'spec_helper'
RSpec
.
describe
Deployments
::
AutoRollbackWorker
do
let_it_be
(
:environment
)
{
create
(
:environment
)
}
let
(
:worker
)
{
described_class
.
new
}
describe
'#perform'
do
subject
{
worker
.
perform
(
environment_id
)
}
let
(
:environment_id
)
{
environment
.
id
}
it
'executes the rollback service'
do
expect_next_instance_of
(
Deployments
::
AutoRollbackService
,
environment
.
project
,
nil
)
do
|
service
|
expect
(
service
).
to
receive
(
:execute
).
with
(
environment
)
end
subject
end
context
'when an environment does not exist'
do
let
(
:environment_id
)
{
non_existing_record_id
}
it
'does not execute the rollback service'
do
expect
(
Deployments
::
AutoRollbackService
).
not_to
receive
(
:new
)
subject
end
end
end
end
lib/gitlab/application_rate_limiter.rb
View file @
48ae718c
...
...
@@ -34,7 +34,8 @@ module Gitlab
group_testing_hook:
{
threshold:
5
,
interval:
1
.
minute
},
profile_add_new_email:
{
threshold:
5
,
interval:
1
.
minute
},
profile_resend_email_confirmation:
{
threshold:
5
,
interval:
1
.
minute
},
update_environment_canary_ingress:
{
threshold:
1
,
interval:
1
.
minute
}
update_environment_canary_ingress:
{
threshold:
1
,
interval:
1
.
minute
},
auto_rollback_deployment:
{
threshold:
1
,
interval:
3
.
minutes
}
}.
freeze
end
...
...
spec/models/deployment_spec.rb
View file @
48ae718c
...
...
@@ -372,6 +372,7 @@ RSpec.describe Deployment do
it
'retrieves deployments with deployable builds'
do
with_deployable
=
create
(
:deployment
)
create
(
:deployment
,
deployable:
nil
)
create
(
:deployment
,
deployable_type:
'CommitStatus'
,
deployable_id:
non_existing_record_id
)
is_expected
.
to
contain_exactly
(
with_deployable
)
end
...
...
@@ -392,6 +393,35 @@ RSpec.describe Deployment do
end
end
describe
'latest_for_sha'
do
subject
{
described_class
.
latest_for_sha
(
sha
)
}
let_it_be
(
:project
)
{
create
(
:project
,
:repository
)
}
let_it_be
(
:commits
)
{
project
.
repository
.
commits
(
'master'
,
limit:
2
)
}
let_it_be
(
:deployments
)
{
commits
.
reverse
.
map
{
|
commit
|
create
(
:deployment
,
project:
project
,
sha:
commit
.
id
)
}
}
let
(
:sha
)
{
commits
.
map
(
&
:id
)
}
it
'finds the latest deployment with sha'
do
is_expected
.
to
eq
(
deployments
.
last
)
end
context
'when sha is old'
do
let
(
:sha
)
{
commits
.
last
.
id
}
it
'finds the latest deployment with sha'
do
is_expected
.
to
eq
(
deployments
.
first
)
end
end
context
'when sha is nil'
do
let
(
:sha
)
{
nil
}
it
'returns nothing'
do
is_expected
.
to
be_nil
end
end
end
describe
'#includes_commit?'
do
let
(
:project
)
{
create
(
:project
,
:repository
)
}
let
(
:environment
)
{
create
(
:environment
,
project:
project
)
}
...
...
spec/models/environment_spec.rb
View file @
48ae718c
...
...
@@ -1394,4 +1394,31 @@ RSpec.describe Environment, :use_clean_rails_memory_store_caching do
it
{
is_expected
.
to
be
(
false
)
}
end
end
describe
'#cancel_deployment_jobs!'
do
subject
{
environment
.
cancel_deployment_jobs!
}
let_it_be
(
:project
)
{
create
(
:project
,
:repository
)
}
let_it_be
(
:environment
,
reload:
true
)
{
create
(
:environment
,
project:
project
)
}
let!
(
:deployment
)
{
create
(
:deployment
,
project:
project
,
environment:
environment
,
deployable:
build
)
}
let!
(
:build
)
{
create
(
:ci_build
,
:running
,
project:
project
,
environment:
environment
)
}
it
'cancels an active deployment job'
do
subject
expect
(
build
.
reset
).
to
be_canceled
end
context
'when deployable does not exist'
do
before
do
deployment
.
update_column
(
:deployable_id
,
non_existing_record_id
)
end
it
'does not raise an error'
do
expect
{
subject
}.
not_to
raise_error
expect
(
build
.
reset
).
to
be_running
end
end
end
end
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment