Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
74951d38
Commit
74951d38
authored
Oct 07, 2021
by
David Fernandez
Committed by
Bob Van Landuyt
Oct 07, 2021
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Improve readability of the cleanup tags service
parent
c1d94239
Changes
6
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
107 additions
and
83 deletions
+107
-83
app/services/container_expiration_policies/cleanup_service.rb
...services/container_expiration_policies/cleanup_service.rb
+2
-2
app/services/projects/container_repository/cleanup_tags_service.rb
...ces/projects/container_repository/cleanup_tags_service.rb
+96
-72
app/workers/cleanup_container_repository_worker.rb
app/workers/cleanup_container_repository_worker.rb
+2
-2
spec/services/container_expiration_policies/cleanup_service_spec.rb
...ces/container_expiration_policies/cleanup_service_spec.rb
+2
-2
spec/services/projects/container_repository/cleanup_tags_service_spec.rb
...rojects/container_repository/cleanup_tags_service_spec.rb
+2
-2
spec/workers/cleanup_container_repository_worker_spec.rb
spec/workers/cleanup_container_repository_worker_spec.rb
+3
-3
No files found.
app/services/container_expiration_policies/cleanup_service.rb
View file @
74951d38
...
...
@@ -24,8 +24,8 @@ module ContainerExpirationPolicies
begin
service_result
=
Projects
::
ContainerRepository
::
CleanupTagsService
.
new
(
project
,
nil
,
policy_params
.
merge
(
'container_expiration_policy'
=>
true
))
.
execute
(
repository
)
.
new
(
repository
,
nil
,
policy_params
.
merge
(
'container_expiration_policy'
=>
true
))
.
execute
rescue
StandardError
repository
.
cleanup_unfinished!
...
...
app/services/projects/container_repository/cleanup_tags_service.rb
View file @
74951d38
...
...
@@ -2,148 +2,152 @@
module
Projects
module
ContainerRepository
class
CleanupTagsService
<
BaseService
class
CleanupTagsService
include
BaseServiceUtility
include
::
Gitlab
::
Utils
::
StrongMemoize
def
execute
(
container_repository
)
return
error
(
'access denied'
)
unless
can_destroy?
return
error
(
'invalid regex'
)
unless
valid_regex?
def
initialize
(
container_repository
,
user
=
nil
,
params
=
{})
@container_repository
=
container_repository
@current_user
=
user
@params
=
params
.
dup
tags
=
container_repository
.
tags
original_size
=
tags
.
size
@project
=
container_repository
.
project
@tags
=
container_repository
.
tags
tags_size
=
@tags
.
size
@counts
=
{
original_size:
tags_size
,
cached_tags_count:
0
}
end
tags
=
without_latest
(
tags
)
tags
=
filter_by_name
(
tags
)
def
execute
return
error
(
'access denied'
)
unless
can_destroy?
return
error
(
'invalid regex'
)
unless
valid_regex?
before_truncate_size
=
tags
.
size
tags
=
truncate
(
tags
)
after_truncate_size
=
tags
.
size
filter_out_latest
filter_by_name
cached_tags_count
=
populate_tags_from_cache
(
container_repository
,
tags
)
||
0
truncate
populate_from_cache
tags
=
filter_keep_n
(
container_repository
,
tags
)
tags
=
filter_by_older_than
(
container_repository
,
tags
)
filter_keep_n
filter_by_older_than
delete_tags
(
container_repository
,
tags
).
tap
do
|
result
|
result
[
:original_size
]
=
original_size
result
[
:before_truncate_size
]
=
before_truncate_size
result
[
:after_truncate_size
]
=
after_truncate_size
result
[
:cached_tags_count
]
=
cached_tags_count
result
[
:before_delete_size
]
=
tags
.
size
delete_tags
.
merge
(
@counts
).
tap
do
|
result
|
result
[
:before_delete_size
]
=
@tags
.
size
result
[
:deleted_size
]
=
result
[
:deleted
]
&
.
size
result
[
:status
]
=
:error
if
before_truncate_size
!=
after_truncate_size
result
[
:status
]
=
:error
if
@counts
[
:before_truncate_size
]
!=
@counts
[
:after_truncate_size
]
end
end
private
def
delete_tags
(
container_repository
,
tags
)
return
success
(
deleted:
[])
unless
tags
.
any?
tag_names
=
tags
.
map
(
&
:name
)
def
delete_tags
return
success
(
deleted:
[])
unless
@tags
.
any?
service
=
Projects
::
ContainerRepository
::
DeleteTagsService
.
new
(
container_repository
.
project
,
current_user
,
tags:
tag_names
,
container_expiration_policy:
params
[
'container_expiration_policy'
]
@
project
,
@
current_user
,
tags:
@tags
.
map
(
&
:name
)
,
container_expiration_policy:
container_expiration_policy
)
service
.
execute
(
container_repository
)
service
.
execute
(
@
container_repository
)
end
def
without_latest
(
tags
)
tags
.
reject
(
&
:latest?
)
def
filter_out_latest
@tags
.
reject!
(
&
:latest?
)
end
def
order_by_date
(
tags
)
def
order_by_date
now
=
DateTime
.
current
tags
.
sort_by
{
|
tag
|
tag
.
created_at
||
now
}.
reverse
@tags
.
sort_by!
{
|
tag
|
tag
.
created_at
||
now
}
.
reverse!
end
def
filter_by_name
(
tags
)
regex_delete
=
::
Gitlab
::
UntrustedRegexp
.
new
(
"
\\
A
#{
params
[
'name_regex_delete'
]
||
params
[
'name_regex'
]
}
\\
z"
)
regex_retain
=
::
Gitlab
::
UntrustedRegexp
.
new
(
"
\\
A
#{
params
[
'name_regex_keep'
]
}
\\
z"
)
def
filter_by_name
regex_delete
=
::
Gitlab
::
UntrustedRegexp
.
new
(
"
\\
A
#{
name_regex_delete
||
name_regex
}
\\
z"
)
regex_retain
=
::
Gitlab
::
UntrustedRegexp
.
new
(
"
\\
A
#{
name_regex_keep
}
\\
z"
)
tags
.
select
do
|
tag
|
@tags
.
select!
do
|
tag
|
# regex_retain will override any overlapping matches by regex_delete
regex_delete
.
match?
(
tag
.
name
)
&&
!
regex_retain
.
match?
(
tag
.
name
)
end
end
def
filter_keep_n
(
container_repository
,
tags
)
return
tags
unless
params
[
'keep_n'
]
def
filter_keep_n
return
unless
keep_n
tags
=
order_by_date
(
tags
)
cache_tags
(
container_repository
,
tags
.
first
(
keep_n
))
tags
.
drop
(
keep_n
)
order_by_date
cache_tags
(
@tags
.
first
(
keep_n_as_integer
))
@tags
=
@tags
.
drop
(
keep_n_as_integer
)
end
def
filter_by_older_than
(
container_repository
,
tags
)
return
tags
unless
older_than
def
filter_by_older_than
return
unless
older_than
older_than_timestamp
=
older_than_in_seconds
.
ago
tags
,
tags_to_keep
=
tags
.
partition
do
|
tag
|
@tags
,
tags_to_keep
=
@
tags
.
partition
do
|
tag
|
tag
.
created_at
&&
tag
.
created_at
<
older_than_timestamp
end
cache_tags
(
container_repository
,
tags_to_keep
)
tags
cache_tags
(
tags_to_keep
)
end
def
can_destroy?
return
true
if
params
[
'container_expiration_policy'
]
return
true
if
container_expiration_policy
can?
(
current_user
,
:destroy_container_image
,
project
)
can?
(
@current_user
,
:destroy_container_image
,
@
project
)
end
def
valid_regex?
%w(name_regex_delete name_regex name_regex_keep)
.
each
do
|
param_name
|
regex
=
params
[
param_name
]
regex
=
@
params
[
param_name
]
::
Gitlab
::
UntrustedRegexp
.
new
(
regex
)
unless
regex
.
blank?
end
true
rescue
RegexpError
=>
e
::
Gitlab
::
ErrorTracking
.
log_exception
(
e
,
project_id:
project
.
id
)
::
Gitlab
::
ErrorTracking
.
log_exception
(
e
,
project_id:
@
project
.
id
)
false
end
def
truncate
(
tags
)
return
tags
unless
throttling_enabled?
return
tags
if
max_list_size
==
0
def
truncate
@counts
[
:before_truncate_size
]
=
@tags
.
size
@counts
[
:after_truncate_size
]
=
@tags
.
size
return
unless
throttling_enabled?
return
if
max_list_size
==
0
# truncate the list to make sure that after the #filter_keep_n
# execution, the resulting list will be max_list_size
truncated_size
=
max_list_size
+
keep_n
truncated_size
=
max_list_size
+
keep_n
_as_integer
return
tags
if
tags
.
size
<=
truncated_size
return
if
@
tags
.
size
<=
truncated_size
tags
.
sample
(
truncated_size
)
@tags
=
@tags
.
sample
(
truncated_size
)
@counts
[
:after_truncate_size
]
=
@tags
.
size
end
def
populate_
tags_from_cache
(
container_repository
,
tags
)
cache
(
container_repository
).
populate
(
tags
)
if
caching_enabled?
(
container_repository
)
def
populate_
from_cache
@counts
[
:cached_tags_count
]
=
cache
.
populate
(
@tags
)
if
caching_enabled?
end
def
cache_tags
(
container_repository
,
tags
)
cache
(
container_repository
).
insert
(
tags
,
older_than_in_seconds
)
if
caching_enabled?
(
container_repository
)
def
cache_tags
(
tags
)
cache
.
insert
(
tags
,
older_than_in_seconds
)
if
caching_enabled?
end
def
cache
(
container_repository
)
# TODO Implement https://gitlab.com/gitlab-org/gitlab/-/issues/340277 to avoid passing
# the container repository parameter which is bad for a memoized function
def
cache
strong_memoize
(
:cache
)
do
::
Projects
::
ContainerRepository
::
CacheTagsCreatedAtService
.
new
(
container_repository
)
::
Projects
::
ContainerRepository
::
CacheTagsCreatedAtService
.
new
(
@
container_repository
)
end
end
def
caching_enabled?
(
container_repository
)
params
[
'container_expiration_policy'
]
&&
def
caching_enabled?
container_expiration_policy
&&
older_than
.
present?
&&
Feature
.
enabled?
(
:container_registry_expiration_policies_caching
,
container_repository
.
project
)
Feature
.
enabled?
(
:container_registry_expiration_policies_caching
,
@
project
)
end
def
throttling_enabled?
...
...
@@ -155,7 +159,11 @@ module Projects
end
def
keep_n
params
[
'keep_n'
].
to_i
@params
[
'keep_n'
]
end
def
keep_n_as_integer
keep_n
.
to_i
end
def
older_than_in_seconds
...
...
@@ -165,7 +173,23 @@ module Projects
end
def
older_than
params
[
'older_than'
]
@params
[
'older_than'
]
end
def
name_regex_delete
@params
[
'name_regex_delete'
]
end
def
name_regex
@params
[
'name_regex'
]
end
def
name_regex_keep
@params
[
'name_regex_keep'
]
end
def
container_expiration_policy
@params
[
'container_expiration_policy'
]
end
end
end
...
...
app/workers/cleanup_container_repository_worker.rb
View file @
74951d38
...
...
@@ -28,8 +28,8 @@ class CleanupContainerRepositoryWorker
end
result
=
Projects
::
ContainerRepository
::
CleanupTagsService
.
new
(
project
,
current_user
,
params
)
.
execute
(
container_repository
)
.
new
(
container_repository
,
current_user
,
params
)
.
execute
if
run_by_container_expiration_policy?
&&
result
[
:status
]
==
:success
container_repository
.
reset_expiration_policy_started_at!
...
...
spec/services/container_expiration_policies/cleanup_service_spec.rb
View file @
74951d38
...
...
@@ -24,8 +24,8 @@ RSpec.describe ContainerExpirationPolicies::CleanupService do
it
'completely clean up the repository'
do
expect
(
Projects
::
ContainerRepository
::
CleanupTagsService
)
.
to
receive
(
:new
).
with
(
project
,
nil
,
cleanup_tags_service_params
).
and_return
(
cleanup_tags_service
)
expect
(
cleanup_tags_service
).
to
receive
(
:execute
).
with
(
repository
).
and_return
(
status: :success
)
.
to
receive
(
:new
).
with
(
repository
,
nil
,
cleanup_tags_service_params
).
and_return
(
cleanup_tags_service
)
expect
(
cleanup_tags_service
).
to
receive
(
:execute
).
and_return
(
status: :success
)
response
=
subject
...
...
spec/services/projects/container_repository/cleanup_tags_service_spec.rb
View file @
74951d38
...
...
@@ -9,7 +9,7 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_
let_it_be
(
:project
,
reload:
true
)
{
create
(
:project
,
:private
)
}
let
(
:repository
)
{
create
(
:container_repository
,
:root
,
project:
project
)
}
let
(
:service
)
{
described_class
.
new
(
project
,
user
,
params
)
}
let
(
:service
)
{
described_class
.
new
(
repository
,
user
,
params
)
}
let
(
:tags
)
{
%w[latest A Ba Bb C D E]
}
before
do
...
...
@@ -39,7 +39,7 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_
end
describe
'#execute'
do
subject
{
service
.
execute
(
repository
)
}
subject
{
service
.
execute
}
shared_examples
'reading and removing tags'
do
|
caching_enabled:
true
|
context
'when no params are specified'
do
...
...
spec/workers/cleanup_container_repository_worker_spec.rb
View file @
74951d38
...
...
@@ -17,7 +17,7 @@ RSpec.describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_stat
it
'executes the destroy service'
do
expect
(
Projects
::
ContainerRepository
::
CleanupTagsService
).
to
receive
(
:new
)
.
with
(
project
,
user
,
params
.
merge
(
'container_expiration_policy'
=>
false
))
.
with
(
repository
,
user
,
params
.
merge
(
'container_expiration_policy'
=>
false
))
.
and_return
(
service
)
expect
(
service
).
to
receive
(
:execute
)
...
...
@@ -49,7 +49,7 @@ RSpec.describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_stat
expect
(
repository
).
to
receive
(
:start_expiration_policy!
).
and_call_original
expect
(
repository
).
to
receive
(
:reset_expiration_policy_started_at!
).
and_call_original
expect
(
Projects
::
ContainerRepository
::
CleanupTagsService
).
to
receive
(
:new
)
.
with
(
project
,
nil
,
params
.
merge
(
'container_expiration_policy'
=>
true
))
.
with
(
repository
,
nil
,
params
.
merge
(
'container_expiration_policy'
=>
true
))
.
and_return
(
service
)
expect
(
service
).
to
receive
(
:execute
).
and_return
(
status: :success
)
...
...
@@ -62,7 +62,7 @@ RSpec.describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_stat
expect
(
repository
).
to
receive
(
:start_expiration_policy!
).
and_call_original
expect
(
repository
).
not_to
receive
(
:reset_expiration_policy_started_at!
)
expect
(
Projects
::
ContainerRepository
::
CleanupTagsService
).
to
receive
(
:new
)
.
with
(
project
,
nil
,
params
.
merge
(
'container_expiration_policy'
=>
true
))
.
with
(
repository
,
nil
,
params
.
merge
(
'container_expiration_policy'
=>
true
))
.
and_return
(
service
)
expect
(
service
).
to
receive
(
:execute
).
and_return
(
status: :error
,
message:
'timeout while deleting tags'
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment