Commit dc609187 authored by Grzegorz Bizon's avatar Grzegorz Bizon

Merge branch 'container-repository-cleanup-api' into 'master'

Container repository cleanup API

Closes #55978

See merge request gitlab-org/gitlab-ce!24303
parents dfa31d5f 99689c83
# frozen_string_literal: true # frozen_string_literal: true
class ContainerRepository < ActiveRecord::Base class ContainerRepository < ActiveRecord::Base
include Gitlab::Utils::StrongMemoize
belongs_to :project belongs_to :project
validates :name, length: { minimum: 0, allow_nil: false } validates :name, length: { minimum: 0, allow_nil: false }
...@@ -8,6 +10,8 @@ class ContainerRepository < ActiveRecord::Base ...@@ -8,6 +10,8 @@ class ContainerRepository < ActiveRecord::Base
delegate :client, to: :registry delegate :client, to: :registry
scope :ordered, -> { order(:name) }
# rubocop: disable CodeReuse/ServiceClass # rubocop: disable CodeReuse/ServiceClass
def registry def registry
@registry ||= begin @registry ||= begin
...@@ -39,13 +43,14 @@ class ContainerRepository < ActiveRecord::Base ...@@ -39,13 +43,14 @@ class ContainerRepository < ActiveRecord::Base
end end
def tags def tags
return @tags if defined?(@tags)
return [] unless manifest && manifest['tags'] return [] unless manifest && manifest['tags']
@tags = manifest['tags'].map do |tag| strong_memoize(:tags) do
manifest['tags'].sort.map do |tag|
ContainerRegistry::Tag.new(self, tag) ContainerRegistry::Tag.new(self, tag)
end end
end end
end
def blob(config) def blob(config)
ContainerRegistry::Blob.new(self, config) ContainerRegistry::Blob.new(self, config)
......
# frozen_string_literal: true
class ContainerRepositoryPolicy < BasePolicy
delegate { @subject.project }
end
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
class ContainerRepositoryEntity < Grape::Entity class ContainerRepositoryEntity < Grape::Entity
include RequestAwareEntity include RequestAwareEntity
expose :id, :path, :location expose :id, :name, :path, :location, :created_at
expose :tags_path do |repository| expose :tags_path do |repository|
project_registry_repository_tags_path(project, repository, format: :json) project_registry_repository_tags_path(project, repository, format: :json)
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
class ContainerTagEntity < Grape::Entity class ContainerTagEntity < Grape::Entity
include RequestAwareEntity include RequestAwareEntity
expose :name, :location, :revision, :short_revision, :total_size, :created_at expose :name, :path, :location, :digest, :revision, :short_revision, :total_size, :created_at
expose :destroy_path, if: -> (*) { can_destroy? } do |tag| expose :destroy_path, if: -> (*) { can_destroy? } do |tag|
project_registry_repository_tag_path(project, tag.repository, tag.name) project_registry_repository_tag_path(project, tag.repository, tag.name)
......
...@@ -6,9 +6,14 @@ ...@@ -6,9 +6,14 @@
# #
# `#try_obtain_lease` takes a block which will be run if it was able to # `#try_obtain_lease` takes a block which will be run if it was able to
# obtain the lease. Implement `#lease_timeout` to configure the timeout # obtain the lease. Implement `#lease_timeout` to configure the timeout
# for the exclusive lease. Optionally override `#lease_key` to set the # for the exclusive lease.
#
# Optionally override `#lease_key` to set the
# lease key, it defaults to the class name with underscores. # lease key, it defaults to the class name with underscores.
# #
# Optionally override `#lease_release?` to prevent the job to
# be re-executed more often than LEASE_TIMEOUT.
#
module ExclusiveLeaseGuard module ExclusiveLeaseGuard
extend ActiveSupport::Concern extend ActiveSupport::Concern
...@@ -23,7 +28,7 @@ module ExclusiveLeaseGuard ...@@ -23,7 +28,7 @@ module ExclusiveLeaseGuard
begin begin
yield lease yield lease
ensure ensure
release_lease(lease) release_lease(lease) if lease_release?
end end
end end
...@@ -40,6 +45,10 @@ module ExclusiveLeaseGuard ...@@ -40,6 +45,10 @@ module ExclusiveLeaseGuard
"#{self.class.name} does not implement #{__method__}" "#{self.class.name} does not implement #{__method__}"
end end
def lease_release?
true
end
def release_lease(uuid) def release_lease(uuid)
Gitlab::ExclusiveLease.cancel(lease_key, uuid) Gitlab::ExclusiveLease.cancel(lease_key, uuid)
end end
......
# frozen_string_literal: true
module Projects
module ContainerRepository
class CleanupTagsService < BaseService
def execute(container_repository)
return error('feature disabled') unless can_use?
return error('access denied') unless can_admin?
tags = container_repository.tags
tags_by_digest = group_by_digest(tags)
tags = without_latest(tags)
tags = filter_by_name(tags)
tags = with_manifest(tags)
tags = order_by_date(tags)
tags = filter_keep_n(tags)
tags = filter_by_older_than(tags)
deleted_tags = delete_tags(tags, tags_by_digest)
success(deleted: deleted_tags.map(&:name))
end
private
def delete_tags(tags_to_delete, tags_by_digest)
deleted_digests = group_by_digest(tags_to_delete).select do |digest, tags|
delete_tag_digest(digest, tags, tags_by_digest[digest])
end
deleted_digests.values.flatten
end
def delete_tag_digest(digest, tags, other_tags)
# Issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/21405
# we have to remove all tags due
# to Docker Distribution bug unable
# to delete single tag
return unless tags.count == other_tags.count
# delete all tags
tags.map(&:delete)
end
def group_by_digest(tags)
tags.group_by(&:digest)
end
def without_latest(tags)
tags.reject(&:latest?)
end
def with_manifest(tags)
tags.select(&:valid?)
end
def order_by_date(tags)
now = DateTime.now
tags.sort_by { |tag| tag.created_at || now }.reverse
end
def filter_by_name(tags)
regex = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex']}\\z")
tags.select do |tag|
regex.scan(tag.name).any?
end
end
def filter_keep_n(tags)
tags.drop(params['keep_n'].to_i)
end
def filter_by_older_than(tags)
return tags unless params['older_than']
older_than = ChronicDuration.parse(params['older_than']).seconds.ago
tags.select do |tag|
tag.created_at && tag.created_at < older_than
end
end
def can_admin?
can?(current_user, :admin_container_image, project)
end
def can_use?
Feature.enabled?(:container_registry_cleanup, project, default_enabled: true)
end
end
end
end
...@@ -90,13 +90,15 @@ ...@@ -90,13 +90,15 @@
- object_pool:object_pool_join - object_pool:object_pool_join
- object_pool:object_pool_destroy - object_pool:object_pool_destroy
- container_repository:delete_container_repository
- container_repository:cleanup_container_repository
- default - default
- mailers # ActionMailer::DeliveryJob.queue_name - mailers # ActionMailer::DeliveryJob.queue_name
- authorized_projects - authorized_projects
- background_migration - background_migration
- create_gpg_signature - create_gpg_signature
- delete_container_repository
- delete_merged_branches - delete_merged_branches
- delete_user - delete_user
- email_receiver - email_receiver
......
# frozen_string_literal: true
class CleanupContainerRepositoryWorker
include ApplicationWorker
include ExclusiveLeaseGuard
queue_namespace :container_repository
LEASE_TIMEOUT = 1.hour
attr_reader :container_repository, :current_user
def perform(current_user_id, container_repository_id, params)
@current_user = User.find_by_id(current_user_id)
@container_repository = ContainerRepository.find_by_id(container_repository_id)
return unless valid?
try_obtain_lease do
Projects::ContainerRepository::CleanupTagsService
.new(project, current_user, params)
.execute(container_repository)
end
end
private
def valid?
current_user && container_repository && project
end
def project
container_repository&.project
end
# For ExclusiveLeaseGuard concern
def lease_key
@lease_key ||= "container_repository:cleanup_tags:#{container_repository.id}"
end
# For ExclusiveLeaseGuard concern
def lease_timeout
LEASE_TIMEOUT
end
# For ExclusiveLeaseGuard concern
def lease_release?
# we don't allow to execute this worker
# more often than LEASE_TIMEOUT
# for given container repository
false
end
end
...@@ -4,6 +4,8 @@ class DeleteContainerRepositoryWorker ...@@ -4,6 +4,8 @@ class DeleteContainerRepositoryWorker
include ApplicationWorker include ApplicationWorker
include ExclusiveLeaseGuard include ExclusiveLeaseGuard
queue_namespace :container_repository
LEASE_TIMEOUT = 1.hour LEASE_TIMEOUT = 1.hour
attr_reader :container_repository attr_reader :container_repository
......
---
title: Add Container Registry API with cleanup function
merge_request: 24303
author:
type: added
...@@ -47,7 +47,6 @@ ...@@ -47,7 +47,6 @@
- [project_service, 1] - [project_service, 1]
- [delete_user, 1] - [delete_user, 1]
- [todos_destroyer, 1] - [todos_destroyer, 1]
- [delete_container_repository, 1]
- [delete_merged_branches, 1] - [delete_merged_branches, 1]
- [authorized_projects, 1] - [authorized_projects, 1]
- [expire_build_instance_artifacts, 1] - [expire_build_instance_artifacts, 1]
...@@ -81,6 +80,7 @@ ...@@ -81,6 +80,7 @@
- [delete_diff_files, 1] - [delete_diff_files, 1]
- [detect_repository_languages, 1] - [detect_repository_languages, 1]
- [auto_devops, 2] - [auto_devops, 2]
- [container_repository, 1]
- [object_pool, 1] - [object_pool, 1]
- [repository_cleanup, 1] - [repository_cleanup, 1]
- [delete_stored_files, 1] - [delete_stored_files, 1]
......
# frozen_string_literal: true
class MigrateDeleteContainerRepositoryWorker < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
sidekiq_queue_migrate('delete_container_repository', to: 'container_repository:delete_container_repository')
end
def down
sidekiq_queue_migrate('container_repository:delete_container_repository', to: 'delete_container_repository')
end
end
...@@ -16,6 +16,7 @@ The following API resources are available: ...@@ -16,6 +16,7 @@ The following API resources are available:
- [Broadcast messages](broadcast_messages.md) - [Broadcast messages](broadcast_messages.md)
- [Code snippets](snippets.md) - [Code snippets](snippets.md)
- [Commits](commits.md) - [Commits](commits.md)
- [Container Registry](container_registry.md)
- [Custom attributes](custom_attributes.md) - [Custom attributes](custom_attributes.md)
- [Deploy keys](deploy_keys.md), and [deploy keys for multiple projects](deploy_key_multiple_projects.md) - [Deploy keys](deploy_keys.md), and [deploy keys for multiple projects](deploy_key_multiple_projects.md)
- [Deployments](deployments.md) - [Deployments](deployments.md)
......
# Container Registry API
This is the API docs of the [GitLab Container Registry](../user/project/container_registry.md).
## List registry repositories
Get a list of registry repositories in a project.
```
GET /projects/:id/registry/repositories
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
```bash
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories"
```
Example response:
```json
[
{
"id": 1,
"name": "",
"path": "group/project",
"location": "gitlab.example.com:5000/group/project",
"created_at": "2019-01-10T13:38:57.391Z"
},
{
"id": 2,
"name": "releases",
"path": "group/project/releases",
"location": "gitlab.example.com:5000/group/project/releases",
"created_at": "2019-01-10T13:39:08.229Z"
}
]
```
## Delete registry repository
Get a list of repository commits in a project.
This operation is executed asynchronously and might take some time to get executed.
```
DELETE /projects/:id/registry/repositories/:repository_id
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
| `repository_id` | integer | yes | The ID of registry repository. |
```bash
curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2"
```
## List repository tags
Get a list of tags for given registry repository.
```
GET /projects/:id/registry/repositories/:repository_id/tags
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
| `repository_id` | integer | yes | The ID of registry repository. |
```bash
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
Example response:
```json
[
{
"name": "A",
"path": "group/project:A",
"location": "gitlab.example.com:5000/group/project:A"
},
{
"name": "latest",
"path": "group/project:latest",
"location": "gitlab.example.com:5000/group/project:latest"
}
]
```
## Get details of a repository tag
Get details of a registry repository tag.
```
GET /projects/:id/registry/repositories/:repository_id/tags/:tag_name
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
| `repository_id` | integer | yes | The ID of registry repository. |
| `tag_name` | string | yes | The name of tag. |
```bash
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags/v10.0.0"
```
Example response:
```json
{
"name": "v10.0.0",
"path": "group/project:latest",
"location": "gitlab.example.com:5000/group/project:latest",
"revision": "e9ed9d87c881d8c2fd3a31b41904d01ba0b836e7fd15240d774d811a1c248181",
"short_revision": "e9ed9d87c",
"digest": "sha256:c3490dcf10ffb6530c1303522a1405dfaf7daecd8f38d3e6a1ba19ea1f8a1751",
"created_at": "2019-01-06T16:49:51.272+00:00",
"total_size": 350224384
}
```
## Delete a repository tag
Delete a registry repository tag.
```
DELETE /projects/:id/registry/repositories/:repository_id/tags/:tag_name
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
| `repository_id` | integer | yes | The ID of registry repository. |
| `tag_name` | string | yes | The name of tag. |
```bash
curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags/v10.0.0"
```
## Delete repository tags in bulk
Delete repository tags in bulk based on given criteria.
```
DELETE /projects/:id/registry/repositories/:repository_id/tags
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
| `repository_id` | integer | yes | The ID of registry repository. |
| `name_regex` | string | yes | The regex of the name to delete. To delete all tags specify `.*`. |
| `keep_n` | integer | no | The amount of latest tags of given name to keep. |
| `older_than` | string | no | Tags to delete that are older than the given time, written in human readable form `1h`, `1d`, `1month`. |
This API call performs the following operations:
1. It orders all tags by creation date. The creation date is the time of the
manifest creation, not the time of tag push.
1. It removes only the tags matching the given `name_regex`.
1. It never removes the tag named `latest`.
1. It keeps N latest matching tags (if `keep_n` is specified).
1. It only removes tags that are older than X amount of time (if `older_than` is specified).
1. It schedules the asynchronous job to be executed in the background.
These operations are executed asynchronously and it might
take time to get executed. You can run this at most
once an hour for a given container repository.
NOTE: **Note:**
Due to a [Docker Distribution deficiency](https://gitlab.com/gitlab-org/gitlab-ce/issues/21405),
it doesn't remove tags whose manifest is shared by multiple tags.
Examples:
1. Remove tag names that are matching the regex (Git SHA), keep always at least 5,
and remove ones that are older than 2 days:
```bash
curl --request DELETE --data 'name_regex=[0-9a-z]{40}' --data 'keep_n=5' --data 'older_than=2d' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
2. Remove all tags, but keep always the latest 5:
```bash
curl --request DELETE --data 'name_regex=.*' --data 'keep_n=5' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
3. Remove all tags that are older than 1 month:
```bash
curl --request DELETE --data 'name_regex=.*' --data 'older_than=1month' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
...@@ -100,6 +100,7 @@ module API ...@@ -100,6 +100,7 @@ module API
mount ::API::CircuitBreakers mount ::API::CircuitBreakers
mount ::API::Commits mount ::API::Commits
mount ::API::CommitStatuses mount ::API::CommitStatuses
mount ::API::ContainerRegistry
mount ::API::DeployKeys mount ::API::DeployKeys
mount ::API::Deployments mount ::API::Deployments
mount ::API::Environments mount ::API::Environments
......
# frozen_string_literal: true
module API
class ContainerRegistry < Grape::API
include PaginationParams
REGISTRY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(
tag_name: API::NO_SLASH_URL_PART_REGEX)
before { error!('404 Not Found', 404) unless Feature.enabled?(:container_registry_api, user_project, default_enabled: true) }
before { authorize_read_container_images! }
params do
requires :id, type: String, desc: 'The ID of a project'
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'Get a project container repositories' do
detail 'This feature was introduced in GitLab 11.8.'
success Entities::ContainerRegistry::Repository
end
params do
use :pagination
end
get ':id/registry/repositories' do
repositories = user_project.container_repositories.ordered
present paginate(repositories), with: Entities::ContainerRegistry::Repository
end
desc 'Delete repository' do
detail 'This feature was introduced in GitLab 11.8.'
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
end
delete ':id/registry/repositories/:repository_id', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_admin_container_image!
DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id)
status :accepted
end
desc 'Get a list of repositories tags' do
detail 'This feature was introduced in GitLab 11.8.'
success Entities::ContainerRegistry::Tag
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
use :pagination
end
get ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_read_container_image!
tags = Kaminari.paginate_array(repository.tags)
present paginate(tags), with: Entities::ContainerRegistry::Tag
end
desc 'Delete repository tags (in bulk)' do
detail 'This feature was introduced in GitLab 11.8.'
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
requires :name_regex, type: String, desc: 'The tag name regexp to delete, specify .* to delete all'
optional :keep_n, type: Integer, desc: 'Keep n of latest tags with matching name'
optional :older_than, type: String, desc: 'Delete older than: 1h, 1d, 1month'
end
delete ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_admin_container_image!
CleanupContainerRepositoryWorker.perform_async(current_user.id, repository.id,
declared_params.except(:repository_id)) # rubocop: disable CodeReuse/ActiveRecord
status :accepted
end
desc 'Get a details about repository tag' do
detail 'This feature was introduced in GitLab 11.8.'
success Entities::ContainerRegistry::TagDetails
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
requires :tag_name, type: String, desc: 'The name of the tag'
end
get ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_read_container_image!
validate_tag!
present tag, with: Entities::ContainerRegistry::TagDetails
end
desc 'Delete repository tag' do
detail 'This feature was introduced in GitLab 11.8.'
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
requires :tag_name, type: String, desc: 'The name of the tag'
end
delete ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_destroy_container_image!
validate_tag!
tag.delete
status :ok
end
end
helpers do
def authorize_read_container_images!
authorize! :read_container_image, user_project
end
def authorize_read_container_image!
authorize! :read_container_image, repository
end
def authorize_update_container_image!
authorize! :update_container_image, repository
end
def authorize_destroy_container_image!
authorize! :admin_container_image, repository
end
def authorize_admin_container_image!
authorize! :admin_container_image, repository
end
def repository
@repository ||= user_project.container_repositories.find(params[:repository_id])
end
def tag
@tag ||= repository.tag(params[:tag_name])
end
def validate_tag!
not_found!('Tag') unless tag.valid?
end
end
end
end
# frozen_string_literal: true
module API
module Entities
module ContainerRegistry
class Repository < Grape::Entity
expose :id
expose :name
expose :path
expose :location
expose :created_at
end
class Tag < Grape::Entity
expose :name
expose :path
expose :location
end
class TagDetails < Tag
expose :revision
expose :short_revision
expose :digest
expose :created_at
expose :total_size
end
end
end
end
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
module ContainerRegistry module ContainerRegistry
class Tag class Tag
include Gitlab::Utils::StrongMemoize
attr_reader :repository, :name attr_reader :repository, :name
delegate :registry, :client, to: :repository delegate :registry, :client, to: :repository
...@@ -15,6 +17,10 @@ module ContainerRegistry ...@@ -15,6 +17,10 @@ module ContainerRegistry
manifest.present? manifest.present?
end end
def latest?
name == "latest"
end
def v1? def v1?
manifest && manifest['schemaVersion'] == 1 manifest && manifest['schemaVersion'] == 1
end end
...@@ -24,7 +30,9 @@ module ContainerRegistry ...@@ -24,7 +30,9 @@ module ContainerRegistry
end end
def manifest def manifest
@manifest ||= client.repository_manifest(repository.path, name) strong_memoize(:manifest) do
client.repository_manifest(repository.path, name)
end
end end
def path def path
...@@ -42,38 +50,46 @@ module ContainerRegistry ...@@ -42,38 +50,46 @@ module ContainerRegistry
end end
def digest def digest
@digest ||= client.repository_tag_digest(repository.path, name) strong_memoize(:digest) do
client.repository_tag_digest(repository.path, name)
end
end end
def config_blob def config_blob
return @config_blob if defined?(@config_blob)
return unless manifest && manifest['config'] return unless manifest && manifest['config']
@config_blob = repository.blob(manifest['config']) strong_memoize(:config_blob) do
repository.blob(manifest['config'])
end
end end
def config def config
return unless config_blob return unless config_blob&.data
@config ||= ContainerRegistry::Config.new(self, config_blob) if config_blob.data strong_memoize(:config) do
ContainerRegistry::Config.new(self, config_blob)
end
end end
def created_at def created_at
return unless config return unless config
@created_at ||= DateTime.rfc3339(config['created']) strong_memoize(:created_at) do
DateTime.rfc3339(config['created'])
end
end end
def layers def layers
return @layers if defined?(@layers)
return unless manifest return unless manifest
strong_memoize(:layers) do
layers = manifest['layers'] || manifest['fsLayers'] layers = manifest['layers'] || manifest['fsLayers']
@layers = layers.map do |layer| layers.map do |layer|
repository.blob(layer) repository.blob(layer)
end end
end end
end
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def total_size def total_size
......
...@@ -19,7 +19,7 @@ describe Projects::Registry::TagsController do ...@@ -19,7 +19,7 @@ describe Projects::Registry::TagsController do
end end
before do before do
stub_container_registry_tags(repository: /image/, tags: tags) stub_container_registry_tags(repository: /image/, tags: tags, with_manifest: true)
end end
context 'when user can control the registry' do context 'when user can control the registry' do
......
FactoryBot.define do FactoryBot.define do
factory :container_repository do factory :container_repository do
name 'test_container_image' name 'test_image'
project project
transient do transient do
......
...@@ -25,7 +25,7 @@ describe "Container Registry", :js do ...@@ -25,7 +25,7 @@ describe "Container Registry", :js do
context 'when there are image repositories' do context 'when there are image repositories' do
before do before do
stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest]) stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest], with_manifest: true)
project.container_repositories << container_repository project.container_repositories << container_repository
end end
......
...@@ -2,20 +2,27 @@ ...@@ -2,20 +2,27 @@
"type": "object", "type": "object",
"required" : [ "required" : [
"id", "id",
"name",
"path", "path",
"location", "location",
"tags_path" "created_at"
], ],
"properties" : { "properties" : {
"id": { "id": {
"type": "integer" "type": "integer"
}, },
"name": {
"type": "string"
},
"path": { "path": {
"type": "string" "type": "string"
}, },
"location": { "location": {
"type": "string" "type": "string"
}, },
"created_at": {
"type": "date-time"
},
"tags_path": { "tags_path": {
"type": "string" "type": "string"
}, },
......
...@@ -2,15 +2,22 @@ ...@@ -2,15 +2,22 @@
"type": "object", "type": "object",
"required" : [ "required" : [
"name", "name",
"path",
"location" "location"
], ],
"properties" : { "properties" : {
"name": { "name": {
"type": "string" "type": "string"
}, },
"path": {
"type": "string"
},
"location": { "location": {
"type": "string" "type": "string"
}, },
"digest": {
"type": "string"
},
"revision": { "revision": {
"type": "string" "type": "string"
}, },
......
require 'spec_helper'
describe API::ContainerRegistry do
set(:project) { create(:project, :private) }
set(:maintainer) { create(:user) }
set(:developer) { create(:user) }
set(:reporter) { create(:user) }
set(:guest) { create(:user) }
let(:root_repository) { create(:container_repository, :root, project: project) }
let(:test_repository) { create(:container_repository, project: project) }
let(:api_user) { maintainer }
before do
project.add_maintainer(maintainer)
project.add_developer(developer)
project.add_reporter(reporter)
project.add_guest(guest)
stub_feature_flags(container_registry_api: true)
stub_container_registry_config(enabled: true)
root_repository
test_repository
end
shared_examples 'being disallowed' do |param|
context "for #{param}" do
let(:api_user) { public_send(param) }
it 'returns access denied' do
subject
expect(response).to have_gitlab_http_status(:forbidden)
end
end
context "for anonymous" do
let(:api_user) { nil }
it 'returns not found' do
subject
expect(response).to have_gitlab_http_status(:not_found)
end
end
end
describe 'GET /projects/:id/registry/repositories' do
subject { get api("/projects/#{project.id}/registry/repositories", api_user) }
it_behaves_like 'being disallowed', :guest
context 'for reporter' do
let(:api_user) { reporter }
it 'returns a list of repositories' do
subject
expect(json_response.length).to eq(2)
expect(json_response.map { |repository| repository['id'] }).to contain_exactly(
root_repository.id, test_repository.id)
end
it 'returns a matching schema' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response).to match_response_schema('registry/repositories')
end
end
end
describe 'DELETE /projects/:id/registry/repositories/:repository_id' do
subject { delete api("/projects/#{project.id}/registry/repositories/#{root_repository.id}", api_user) }
it_behaves_like 'being disallowed', :developer
context 'for maintainer' do
let(:api_user) { maintainer }
it 'schedules removal of repository' do
expect(DeleteContainerRepositoryWorker).to receive(:perform_async)
.with(maintainer.id, root_repository.id)
subject
expect(response).to have_gitlab_http_status(:accepted)
end
end
end
describe 'GET /projects/:id/registry/repositories/:repository_id/tags' do
subject { get api("/projects/#{project.id}/registry/repositories/#{root_repository.id}/tags", api_user) }
it_behaves_like 'being disallowed', :guest
context 'for reporter' do
let(:api_user) { reporter }
before do
stub_container_registry_tags(repository: root_repository.path, tags: %w(rootA latest))
end
it 'returns a list of tags' do
subject
expect(json_response.length).to eq(2)
expect(json_response.map { |repository| repository['name'] }).to eq %w(latest rootA)
end
it 'returns a matching schema' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response).to match_response_schema('registry/tags')
end
end
end
describe 'DELETE /projects/:id/registry/repositories/:repository_id/tags' do
subject { delete api("/projects/#{project.id}/registry/repositories/#{root_repository.id}/tags", api_user), params: params }
it_behaves_like 'being disallowed', :developer do
let(:params) do
{ name_regex: 'v10.*' }
end
end
context 'for maintainer' do
let(:api_user) { maintainer }
context 'without required parameters' do
let(:params) { }
it 'returns bad request' do
subject
expect(response).to have_gitlab_http_status(:bad_request)
end
end
context 'passes all declared parameters' do
let(:params) do
{ name_regex: 'v10.*',
keep_n: 100,
older_than: '1 day',
other: 'some value' }
end
let(:worker_params) do
{ name_regex: 'v10.*',
keep_n: 100,
older_than: '1 day' }
end
it 'schedules cleanup of tags repository' do
expect(CleanupContainerRepositoryWorker).to receive(:perform_async)
.with(maintainer.id, root_repository.id, worker_params)
subject
expect(response).to have_gitlab_http_status(:accepted)
end
end
end
end
describe 'GET /projects/:id/registry/repositories/:repository_id/tags/:tag_name' do
subject { get api("/projects/#{project.id}/registry/repositories/#{root_repository.id}/tags/rootA", api_user) }
it_behaves_like 'being disallowed', :guest
context 'for reporter' do
let(:api_user) { reporter }
before do
stub_container_registry_tags(repository: root_repository.path, tags: %w(rootA), with_manifest: true)
end
it 'returns a details of tag' do
subject
expect(json_response).to include(
'name' => 'rootA',
'digest' => 'sha256:4c8e63ca4cb663ce6c688cb06f1c372b088dac5b6d7ad7d49cd620d85cf72a15',
'revision' => 'd7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac',
'total_size' => 2319870)
end
it 'returns a matching schema' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response).to match_response_schema('registry/tag')
end
end
end
describe 'DELETE /projects/:id/registry/repositories/:repository_id/tags/:tag_name' do
subject { delete api("/projects/#{project.id}/registry/repositories/#{root_repository.id}/tags/rootA", api_user) }
it_behaves_like 'being disallowed', :developer
context 'for maintainer' do
let(:api_user) { maintainer }
before do
stub_container_registry_tags(repository: root_repository.path, tags: %w(rootA), with_manifest: true)
end
it 'properly removes tag' do
expect_any_instance_of(ContainerRegistry::Client)
.to receive(:delete_repository_tag).with(root_repository.path,
'sha256:4c8e63ca4cb663ce6c688cb06f1c372b088dac5b6d7ad7d49cd620d85cf72a15')
subject
expect(response).to have_gitlab_http_status(:ok)
end
end
end
end
...@@ -16,7 +16,7 @@ describe ContainerTagEntity do ...@@ -16,7 +16,7 @@ describe ContainerTagEntity do
before do before do
stub_container_registry_config(enabled: true) stub_container_registry_config(enabled: true)
stub_container_registry_tags(repository: /image/, tags: %w[test]) stub_container_registry_tags(repository: /image/, tags: %w[test], with_manifest: true)
allow(request).to receive(:project).and_return(project) allow(request).to receive(:project).and_return(project)
allow(request).to receive(:current_user).and_return(user) allow(request).to receive(:current_user).and_return(user)
end end
......
# frozen_string_literal: true
require 'spec_helper'
describe Projects::ContainerRepository::CleanupTagsService do
set(:user) { create(:user) }
set(:project) { create(:project, :private) }
set(:repository) { create(:container_repository, :root, project: project) }
let(:service) { described_class.new(project, user, params) }
before do
project.add_maintainer(user)
stub_feature_flags(container_registry_cleanup: true)
stub_container_registry_config(enabled: true)
stub_container_registry_tags(
repository: repository.path,
tags: %w(latest A Ba Bb C D E))
stub_tag_digest('latest', 'sha256:configA')
stub_tag_digest('A', 'sha256:configA')
stub_tag_digest('Ba', 'sha256:configB')
stub_tag_digest('Bb', 'sha256:configB')
stub_tag_digest('C', 'sha256:configC')
stub_tag_digest('D', 'sha256:configD')
stub_tag_digest('E', nil)
stub_digest_config('sha256:configA', 1.hour.ago)
stub_digest_config('sha256:configB', 5.days.ago)
stub_digest_config('sha256:configC', 1.month.ago)
stub_digest_config('sha256:configD', nil)
end
describe '#execute' do
subject { service.execute(repository) }
context 'when no params are specified' do
let(:params) { {} }
it 'does not remove anything' do
expect_any_instance_of(ContainerRegistry::Client).not_to receive(:delete_repository_tag)
is_expected.to include(status: :success, deleted: [])
end
end
context 'when regex matching everything is specified' do
let(:params) do
{ 'name_regex' => '.*' }
end
it 'does remove B* and C' do
# The :A cannot be removed as config is shared with :latest
# The :E cannot be removed as it does not have valid manifest
expect_delete('sha256:configB').twice
expect_delete('sha256:configC')
expect_delete('sha256:configD')
is_expected.to include(status: :success, deleted: %w(D Bb Ba C))
end
end
context 'when regex matching specific tags is used' do
let(:params) do
{ 'name_regex' => 'C|D' }
end
it 'does remove C and D' do
expect_delete('sha256:configC')
expect_delete('sha256:configD')
is_expected.to include(status: :success, deleted: %w(D C))
end
end
context 'when removing a tagged image that is used by another tag' do
let(:params) do
{ 'name_regex' => 'Ba' }
end
it 'does not remove the tag' do
# Issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/21405
is_expected.to include(status: :success, deleted: [])
end
end
context 'when removing keeping only 3' do
let(:params) do
{ 'name_regex' => '.*',
'keep_n' => 3 }
end
it 'does remove C as it is oldest' do
expect_delete('sha256:configC')
is_expected.to include(status: :success, deleted: %w(C))
end
end
context 'when removing older than 1 day' do
let(:params) do
{ 'name_regex' => '.*',
'older_than' => '1 day' }
end
it 'does remove B* and C as they are older than 1 day' do
expect_delete('sha256:configB').twice
expect_delete('sha256:configC')
is_expected.to include(status: :success, deleted: %w(Bb Ba C))
end
end
context 'when combining all parameters' do
let(:params) do
{ 'name_regex' => '.*',
'keep_n' => 1,
'older_than' => '1 day' }
end
it 'does remove B* and C' do
expect_delete('sha256:configB').twice
expect_delete('sha256:configC')
is_expected.to include(status: :success, deleted: %w(Bb Ba C))
end
end
end
private
def stub_tag_digest(tag, digest)
allow_any_instance_of(ContainerRegistry::Client)
.to receive(:repository_tag_digest)
.with(repository.path, tag) { digest }
allow_any_instance_of(ContainerRegistry::Client)
.to receive(:repository_manifest)
.with(repository.path, tag) do
{ 'config' => { 'digest' => digest } } if digest
end
end
def stub_digest_config(digest, created_at)
allow_any_instance_of(ContainerRegistry::Client)
.to receive(:blob)
.with(repository.path, digest, nil) do
{ 'created' => created_at.to_datetime.rfc3339 }.to_json if created_at
end
end
def expect_delete(digest)
expect_any_instance_of(ContainerRegistry::Client)
.to receive(:delete_repository_tag)
.with(repository.path, digest)
end
end
...@@ -36,31 +36,41 @@ module StubGitlabCalls ...@@ -36,31 +36,41 @@ module StubGitlabCalls
.to receive(:full_access_token).and_return('token') .to receive(:full_access_token).and_return('token')
end end
def stub_container_registry_tags(repository: :any, tags:) def stub_container_registry_tags(repository: :any, tags: [], with_manifest: false)
repository = any_args if repository == :any repository = any_args if repository == :any
allow_any_instance_of(ContainerRegistry::Client) allow_any_instance_of(ContainerRegistry::Client)
.to receive(:repository_tags).with(repository) .to receive(:repository_tags).with(repository)
.and_return({ 'tags' => tags }) .and_return({ 'tags' => tags })
if with_manifest
tags.each do |tag|
allow_any_instance_of(ContainerRegistry::Client)
.to receive(:repository_tag_digest)
.with(repository, tag)
.and_return('sha256:4c8e63ca4cb663ce6c688cb06f1c3' \
'72b088dac5b6d7ad7d49cd620d85cf72a15')
end
allow_any_instance_of(ContainerRegistry::Client) allow_any_instance_of(ContainerRegistry::Client)
.to receive(:repository_manifest).with(repository, anything) .to receive(:repository_manifest).with(repository, anything)
.and_return(stub_container_registry_tag_manifest) .and_return(stub_container_registry_tag_manifest_content)
allow_any_instance_of(ContainerRegistry::Client) allow_any_instance_of(ContainerRegistry::Client)
.to receive(:blob).with(repository, anything, 'application/octet-stream') .to receive(:blob).with(repository, anything, 'application/octet-stream')
.and_return(stub_container_registry_blob) .and_return(stub_container_registry_blob_content)
end
end end
private private
def stub_container_registry_tag_manifest def stub_container_registry_tag_manifest_content
fixture_path = 'spec/fixtures/container_registry/tag_manifest.json' fixture_path = 'spec/fixtures/container_registry/tag_manifest.json'
JSON.parse(File.read(Rails.root + fixture_path)) JSON.parse(File.read(Rails.root + fixture_path))
end end
def stub_container_registry_blob def stub_container_registry_blob_content
fixture_path = 'spec/fixtures/container_registry/config_blob.json' fixture_path = 'spec/fixtures/container_registry/config_blob.json'
File.read(Rails.root + fixture_path) File.read(Rails.root + fixture_path)
......
# frozen_string_literal: true
require 'spec_helper'
describe CleanupContainerRepositoryWorker, :clean_gitlab_redis_shared_state do
let(:repository) { create(:container_repository) }
let(:project) { repository.project }
let(:user) { project.owner }
let(:params) { { key: 'value' } }
subject { described_class.new }
describe '#perform' do
let(:service) { instance_double(Projects::ContainerRepository::CleanupTagsService) }
before do
allow(Projects::ContainerRepository::CleanupTagsService).to receive(:new)
.with(project, user, params).and_return(service)
end
it 'executes the destroy service' do
expect(service).to receive(:execute)
subject.perform(user.id, repository.id, params)
end
it 'does not raise error when user could not be found' do
expect do
subject.perform(-1, repository.id, params)
end.not_to raise_error
end
it 'does not raise error when repository could not be found' do
expect do
subject.perform(user.id, -1, params)
end.not_to raise_error
end
context 'when executed twice in short period' do
it 'executes service only for the first time' do
expect(service).to receive(:execute).once
2.times { subject.perform(user.id, repository.id, params) }
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment