Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Léo-Paul Géneau
gitlab-ce
Commits
6db7c232
Commit
6db7c232
authored
Feb 09, 2017
by
Tomasz Maczukin
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add minor updates
parent
48cb391c
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
32 additions
and
9 deletions
+32
-9
app/workers/stuck_ci_builds_worker.rb
app/workers/stuck_ci_builds_worker.rb
+13
-5
spec/workers/stuck_ci_builds_worker_spec.rb
spec/workers/stuck_ci_builds_worker_spec.rb
+19
-4
No files found.
app/workers/stuck_ci_builds_worker.rb
View file @
6db7c232
...
@@ -2,6 +2,8 @@ class StuckCiBuildsWorker
...
@@ -2,6 +2,8 @@ class StuckCiBuildsWorker
include
Sidekiq
::
Worker
include
Sidekiq
::
Worker
include
CronjobQueue
include
CronjobQueue
EXCLUSIVE_LEASE_KEY
=
'stuck_ci_builds_worker_lease'
BUILD_RUNNING_OUTDATED_TIMEOUT
=
1
.
hour
BUILD_RUNNING_OUTDATED_TIMEOUT
=
1
.
hour
BUILD_PENDING_OUTDATED_TIMEOUT
=
1
.
day
BUILD_PENDING_OUTDATED_TIMEOUT
=
1
.
day
BUILD_PENDING_STUCK_TIMEOUT
=
1
.
hour
BUILD_PENDING_STUCK_TIMEOUT
=
1
.
hour
...
@@ -11,20 +13,26 @@ class StuckCiBuildsWorker
...
@@ -11,20 +13,26 @@ class StuckCiBuildsWorker
Rails
.
logger
.
info
"
#{
self
.
class
}
: Cleaning stuck builds"
Rails
.
logger
.
info
"
#{
self
.
class
}
: Cleaning stuck builds"
drop
:running
,
BUILD_RUNNING_OUTDATED_TIMEOUT
drop
:running
,
BUILD_RUNNING_OUTDATED_TIMEOUT
drop
:pending
,
BUILD_PENDING_OUTDATED_TIMEOUT
drop
:pending
,
BUILD_PENDING_OUTDATED_TIMEOUT
drop_stuck
:pending
,
BUILD_PENDING_STUCK_TIMEOUT
drop_stuck
:pending
,
BUILD_PENDING_STUCK_TIMEOUT
remove_lease
end
end
private
private
def
try_obtain_lease
def
try_obtain_lease
Gitlab
::
ExclusiveLease
.
new
(
@uuid
=
Gitlab
::
ExclusiveLease
.
new
(
'stuck_ci_builds_worker_lease'
,
EXCLUSIVE_LEASE_KEY
,
timeout:
30
.
minutes
timeout:
30
.
minutes
).
try_obtain
).
try_obtain
end
end
def
remove_lease
Gitlab
::
ExclusiveLease
.
cancel
(
EXCLUSIVE_LEASE_KEY
,
@uuid
)
end
def
drop
(
status
,
timeout
)
def
drop
(
status
,
timeout
)
search
(
status
,
timeout
)
do
|
build
|
search
(
status
,
timeout
)
do
|
build
|
drop_build
:outdated
,
build
,
status
,
timeout
drop_build
:outdated
,
build
,
status
,
timeout
...
...
spec/workers/stuck_ci_builds_worker_spec.rb
View file @
6db7c232
...
@@ -4,6 +4,7 @@ describe StuckCiBuildsWorker do
...
@@ -4,6 +4,7 @@ describe StuckCiBuildsWorker do
let!
(
:runner
)
{
create
:ci_runner
}
let!
(
:runner
)
{
create
:ci_runner
}
let!
(
:build
)
{
create
:ci_build
,
runner:
runner
}
let!
(
:build
)
{
create
:ci_build
,
runner:
runner
}
let
(
:worker
)
{
described_class
.
new
}
let
(
:worker
)
{
described_class
.
new
}
let
(
:exclusive_lease_uuid
)
{
SecureRandom
.
uuid
}
subject
do
subject
do
build
.
reload
build
.
reload
...
@@ -12,7 +13,7 @@ describe StuckCiBuildsWorker do
...
@@ -12,7 +13,7 @@ describe StuckCiBuildsWorker do
before
do
before
do
build
.
update!
(
status:
status
,
updated_at:
updated_at
)
build
.
update!
(
status:
status
,
updated_at:
updated_at
)
allow_any_instance_of
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:try_obtain
).
and_return
(
true
)
allow_any_instance_of
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:try_obtain
).
and_return
(
exclusive_lease_uuid
)
end
end
shared_examples
'build is dropped'
do
shared_examples
'build is dropped'
do
...
@@ -103,12 +104,26 @@ describe StuckCiBuildsWorker do
...
@@ -103,12 +104,26 @@ describe StuckCiBuildsWorker do
describe
'exclusive lease'
do
describe
'exclusive lease'
do
let
(
:status
)
{
'running'
}
let
(
:status
)
{
'running'
}
let
(
:updated_at
)
{
2
.
days
.
ago
}
let
(
:updated_at
)
{
2
.
days
.
ago
}
let
(
:worker2
)
{
described_class
.
new
}
it
'is guard by exclusive lease'
do
it
'is guard by exclusive lease when executed concurrently'
do
expect
(
worker
).
to
receive
(
:drop
).
at_least
(
:once
)
expect
(
worker2
).
not_to
receive
(
:drop
)
worker
.
perform
worker
.
perform
allow_any_instance_of
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:try_obtain
).
and_return
(
false
)
allow_any_instance_of
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:try_obtain
).
and_return
(
false
)
expect
(
worker
).
not_to
receive
(
:drop
)
worker2
.
perform
end
it
'can be executed in sequence'
do
expect
(
worker
).
to
receive
(
:drop
).
at_least
(
:once
)
expect
(
worker2
).
to
receive
(
:drop
).
at_least
(
:once
)
worker
.
perform
worker2
.
perform
end
it
'cancels exclusive lease after worker perform'
do
expect
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:cancel
).
with
(
described_class
::
EXCLUSIVE_LEASE_KEY
,
exclusive_lease_uuid
)
worker
.
perform
worker
.
perform
end
end
end
end
end
end
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment