Commit 3bdfc247 authored by Matt Kasa's avatar Matt Kasa

Add Terraform Module Registry

- Adds service discovery
- Adds Terraform Module package type
- Adds API endpoints
- Adds http header token locations to APIAuthentication

Relates to https://gitlab.com/gitlab-org/gitlab/-/issues/321102
parent 421d10b7
......@@ -35,6 +35,7 @@ class Admin::PlanLimitsController < Admin::ApplicationController
npm_max_file_size
nuget_max_file_size
pypi_max_file_size
terraform_module_max_file_size
generic_packages_max_file_size
])
end
......
# frozen_string_literal: true
class Terraform::ServicesController < ApplicationController
skip_before_action :authenticate_user!
feature_category :infrastructure_as_code
def index
render json: { 'modules.v1' => "/api/#{::API::API.version}/packages/terraform/modules/v1/" }
end
end
......@@ -42,7 +42,7 @@ module Packages
end
def filter_by_package_type(packages)
return packages unless package_type
return packages.without_package_type(:terraform_module) unless package_type
raise InvalidPackageTypeError unless ::Packages::Package.package_types.key?(package_type)
packages.with_package_type(package_type)
......@@ -54,6 +54,12 @@ module Packages
packages.search_by_name(params[:package_name])
end
def filter_by_package_version(packages)
return packages unless params[:package_version].present?
packages.with_version(params[:package_version])
end
def filter_with_version(packages)
return packages if params[:include_versionless].present?
......
......@@ -32,6 +32,7 @@ module Packages
packages = filter_with_version(packages)
packages = filter_by_package_type(packages)
packages = filter_by_package_name(packages)
packages = filter_by_package_version(packages)
installable_only ? packages.installable : filter_by_status(packages)
end
......
......@@ -5,7 +5,8 @@ module Types
class PackageTypeEnum < BaseEnum
PACKAGE_TYPE_NAMES = {
pypi: 'PyPI',
npm: 'npm'
npm: 'npm',
terraform_module: 'Terraform Module'
}.freeze
::Packages::Package.package_types.keys.each do |package_type|
......
......@@ -51,6 +51,7 @@ class Packages::Package < ApplicationRecord
validates :name, format: { with: Gitlab::Regex.helm_package_regex }, if: :helm?
validates :name, format: { with: Gitlab::Regex.npm_package_name_regex }, if: :npm?
validates :name, format: { with: Gitlab::Regex.nuget_package_name_regex }, if: :nuget?
validates :name, format: { with: Gitlab::Regex.terraform_module_package_name_regex }, if: :terraform_module?
validates :name, format: { with: Gitlab::Regex.debian_package_name_regex }, if: :debian_package?
validates :name, inclusion: { in: %w[incoming] }, if: :debian_incoming?
validates :version, format: { with: Gitlab::Regex.nuget_version_regex }, if: :nuget?
......@@ -59,7 +60,7 @@ class Packages::Package < ApplicationRecord
validates :version, format: { with: Gitlab::Regex.pypi_version_regex }, if: :pypi?
validates :version, format: { with: Gitlab::Regex.prefixed_semver_regex }, if: :golang?
validates :version, format: { with: Gitlab::Regex.prefixed_semver_regex }, if: :helm?
validates :version, format: { with: Gitlab::Regex.semver_regex }, if: -> { composer_tag_version? || npm? }
validates :version, format: { with: Gitlab::Regex.semver_regex }, if: -> { composer_tag_version? || npm? || terraform_module? }
validates :version,
presence: true,
......@@ -73,7 +74,7 @@ class Packages::Package < ApplicationRecord
enum package_type: { maven: 1, npm: 2, conan: 3, nuget: 4, pypi: 5,
composer: 6, generic: 7, golang: 8, debian: 9,
rubygems: 10, helm: 11 }
rubygems: 10, helm: 11, terraform_module: 12 }
enum status: { default: 0, hidden: 1, processing: 2, error: 3 }
......@@ -85,6 +86,7 @@ class Packages::Package < ApplicationRecord
scope :with_version, ->(version) { where(version: version) }
scope :without_version_like, -> (version) { where.not(arel_table[:version].matches(version)) }
scope :with_package_type, ->(package_type) { where(package_type: package_type) }
scope :without_package_type, ->(package_type) { where.not(package_type: package_type) }
scope :with_status, ->(status) { where(status: status) }
scope :displayable, -> { with_status(DISPLAYABLE_STATUSES) }
scope :installable, -> { with_status(INSTALLABLE_STATUSES) }
......
# frozen_string_literal: true
module Terraform
class ModulesPresenter < Gitlab::View::Presenter::Simple
attr_accessor :packages, :system
presents :modules
def initialize(packages, system)
@packages = packages
@system = system
end
def modules
project_url = @packages.first&.project&.web_url
versions = @packages.map do |package|
{
'version' => package.version,
'submodules' => [],
'root' => {
'dependencies' => [],
'providers' => [
{
'name' => @system,
'version' => ''
}
]
}
}
end
[
{
'versions' => versions,
'source' => project_url
}.compact
]
end
end
end
# frozen_string_literal: true
module Packages
module TerraformModule
class CreatePackageService < ::Packages::CreatePackageService
include Gitlab::Utils::StrongMemoize
def execute
return error('Version is empty.', 400) if params[:module_version].blank?
return error('Package already exists.', 403) if current_package_exists_elsewhere?
return error('Package version already exists.', 403) if current_package_version_exists?
return error('File is too large.', 400) if file_size_exceeded?
ActiveRecord::Base.transaction { create_terraform_module_package! }
end
private
def create_terraform_module_package!
package = create_package!(:terraform_module, name: name, version: params[:module_version])
::Packages::CreatePackageFileService.new(package, file_params).execute
package
end
def current_package_exists_elsewhere?
::Packages::Package
.for_projects(project.root_namespace.all_projects.id_not_in(project.id))
.with_package_type(:terraform_module)
.with_name(name)
.exists?
end
def current_package_version_exists?
project.packages
.with_package_type(:terraform_module)
.with_name(name)
.with_version(params[:module_version])
.exists?
end
def name
strong_memoize(:name) do
"#{params[:module_name]}/#{params[:module_system]}"
end
end
def file_name
strong_memoize(:file_name) do
"#{params[:module_name]}-#{params[:module_system]}-#{params[:module_version]}.tgz"
end
end
def file_params
{
file: params[:file],
size: params[:file].size,
file_sha256: params[:file].sha256,
file_name: file_name,
build: params[:build]
}
end
def file_size_exceeded?
project.actual_limits.exceeded?(:generic_packages_max_file_size, params[:file].size)
end
end
end
end
......@@ -44,6 +44,9 @@
.form-group
= f.label :pypi_max_file_size, _('Maximum PyPI package file size in bytes'), class: 'label-bold'
= f.number_field :pypi_max_file_size, class: 'form-control gl-form-input'
.form-group
= f.label :terraform_module_max_file_size, _('Maximum Terraform Module package file size in bytes'), class: 'label-bold'
= f.number_field :terraform_module_max_file_size, class: 'form-control gl-form-input'
.form-group
= f.label :generic_packages_max_file_size, _('Generic package file size in bytes'), class: 'label-bold'
= f.number_field :generic_packages_max_file_size, class: 'form-control gl-form-input'
......
---
title: Add Terraform Module Registry
merge_request: 55018
author:
type: added
---
key_path: redis_hll_counters.deploy_token_packages.i_package_terraform_module_deploy_token_monthly
description: Number of distinct users authorized via deploy token creating Terraform Module packages in recent 28 days
product_section: ops
product_stage: configure
product_group: group::configure
product_category: infrastructure_as_code
value_type: number
status: implemented
milestone: '13.11'
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55018
time_frame: 28d
data_source: redis_hll
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
---
key_path: redis_hll_counters.user_packages.i_package_terraform_module_user_monthly
description: Number of distinct users creating Terraform Module packages in recent 28 days
product_section: ops
product_stage: configure
product_group: group::configure
product_category: infrastructure_as_code
value_type: number
status: implemented
milestone: '13.11'
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55018
time_frame: 28d
data_source: redis_hll
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
---
key_path: redis_hll_counters.deploy_token_packages.i_package_terraform_module_deploy_token_weekly
description: Number of distinct users authorized via deploy token creating Terraform Module packages in recent 7 days
product_section: ops
product_stage: configure
product_group: group::configure
product_category: infrastructure_as_code
value_type: number
status: implemented
milestone: '13.11'
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55018
time_frame: 7d
data_source: redis_hll
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
---
key_path: redis_hll_counters.user_packages.i_package_terraform_module_user_weekly
description: Number of distinct users creating Terraform Module packages in recent 7 days
product_section: ops
product_stage: configure
product_group: group::configure
product_category: infrastructure_as_code
value_type: number
status: implemented
milestone: '13.11'
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55018
time_frame: 7d
data_source: redis_hll
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
---
key_path: counts.package_events_i_package_terraform_module_delete_package
description: Total count of Terraform Module packages delete events
product_section: ops
product_stage: configure
product_group: group::configure
product_category: infrastructure_as_code
value_type: number
status: implemented
milestone: '13.11'
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55018
time_frame: all
data_source: redis
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
---
key_path: counts.package_events_i_package_terraform_module_pull_package
description: Total count of pull Terraform Module packages events
product_section: ops
product_stage: configure
product_group: group::configure
product_category: infrastructure_as_code
value_type: number
status: implemented
milestone: '13.11'
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55018
time_frame: all
data_source: redis
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
---
key_path: counts.package_events_i_package_terraform_module_push_package
description: Total count of push Terraform Module packages events
product_section: ops
product_stage: configure
product_group: group::configure
product_category: infrastructure_as_code
value_type: number
status: implemented
milestone: '13.11'
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55018
time_frame: all
data_source: redis
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
......@@ -77,6 +77,9 @@ Rails.application.routes.draw do
# Health check
get 'health_check(/:checks)' => 'health_check#index', as: :health_check
# Terraform service discovery
get '.well-known/terraform.json' => 'terraform/services#index', as: :terraform_services
# Begin of the /-/ scope.
# Use this scope for all new global routes.
scope path: '-' do
......
......@@ -14232,6 +14232,7 @@ Values for sorting package.
| <a id="packagetypeenumnuget"></a>`NUGET` | Packages from the Nuget package manager. |
| <a id="packagetypeenumpypi"></a>`PYPI` | Packages from the PyPI package manager. |
| <a id="packagetypeenumrubygems"></a>`RUBYGEMS` | Packages from the Rubygems package manager. |
| <a id="packagetypeenumterraform_module"></a>`TERRAFORM_MODULE` | Packages from the Terraform Module package manager. |
### `PipelineConfigSourceEnum`
......
......@@ -3754,6 +3754,42 @@ Status: `data_available`
Tiers: `free`
### `counts.package_events_i_package_terraform_module_delete_package`
Total count of Terraform Module packages delete events
[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210410012200_package_events_i_package_terraform_module_delete_package.yml)
Group: `group::configure`
Status: `implemented`
Tiers: `free`, `premium`, `ultimate`
### `counts.package_events_i_package_terraform_module_pull_package`
Total count of pull Terraform Module packages events
[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210410012202_package_events_i_package_terraform_module_pull_package.yml)
Group: `group::configure`
Status: `implemented`
Tiers: `free`, `premium`, `ultimate`
### `counts.package_events_i_package_terraform_module_push_package`
Total count of push Terraform Module packages events
[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210410012204_package_events_i_package_terraform_module_push_package.yml)
Group: `group::configure`
Status: `implemented`
Tiers: `free`, `premium`, `ultimate`
### `counts.packages`
Number of packages
......@@ -9898,6 +9934,30 @@ Status: `data_available`
Tiers:
### `redis_hll_counters.deploy_token_packages.i_package_terraform_module_deploy_token_monthly`
Number of distinct users authorized via deploy token creating Terraform Module packages in recent 28 days
[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_28d/20210410012206_i_package_terraform_module_deploy_token_monthly.yml)
Group: `group::configure`
Status: `implemented`
Tiers: `free`, `premium`, `ultimate`
### `redis_hll_counters.deploy_token_packages.i_package_terraform_module_deploy_token_weekly`
Number of distinct users authorized via deploy token creating Terraform Module packages in recent 7 days
[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_7d/20210410012207_i_package_terraform_module_deploy_token_weekly.yml)
Group: `group::configure`
Status: `implemented`
Tiers: `free`, `premium`, `ultimate`
### `redis_hll_counters.ecosystem.ecosystem_total_unique_counts_monthly`
Missing description
......@@ -14866,6 +14926,30 @@ Status: `data_available`
Tiers:
### `redis_hll_counters.user_packages.i_package_terraform_module_user_monthly`
Number of distinct users creating Terraform Module packages in recent 28 days
[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_28d/20210410012208_i_package_terraform_module_user_monthly.yml)
Group: `group::configure`
Status: `implemented`
Tiers: `free`, `premium`, `ultimate`
### `redis_hll_counters.user_packages.i_package_terraform_module_user_weekly`
Number of distinct users creating Terraform Module packages in recent 7 days
[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_7d/20210410012209_i_package_terraform_module_user_weekly.yml)
Group: `group::configure`
Status: `implemented`
Tiers: `free`, `premium`, `ultimate`
### `redis_hll_counters.user_packages.user_packages_total_unique_counts_monthly`
Missing description
......
......@@ -241,6 +241,7 @@ module API
mount ::API::ProjectTemplates
mount ::API::Terraform::State
mount ::API::Terraform::StateVersion
mount ::API::Terraform::Modules::V1::Packages
mount ::API::PersonalAccessTokens
mount ::API::ProtectedBranches
mount ::API::ProtectedTags
......
# frozen_string_literal: true
module API
module Entities
module Terraform
class ModuleVersions < Grape::Entity
expose :modules
end
end
end
end
# frozen_string_literal: true
module API
module Terraform
module Modules
module V1
class Packages < ::API::Base
include ::API::Helpers::Authentication
helpers ::API::Helpers::PackagesHelpers
helpers ::API::Helpers::Packages::BasicAuthHelpers
SEMVER_REGEX = Gitlab::Regex.semver_regex
TERRAFORM_MODULE_REQUIREMENTS = {
module_namespace: API::NO_SLASH_URL_PART_REGEX,
module_name: API::NO_SLASH_URL_PART_REGEX,
module_system: API::NO_SLASH_URL_PART_REGEX
}.freeze
TERRAFORM_MODULE_VERSION_REQUIREMENTS = {
module_version: SEMVER_REGEX
}.freeze
feature_category :package_registry
after_validation do
require_packages_enabled!
end
helpers do
params :module_name do
requires :module_name, type: String, desc: "", regexp: API::NO_SLASH_URL_PART_REGEX
requires :module_system, type: String, regexp: API::NO_SLASH_URL_PART_REGEX
end
params :module_version do
requires :module_version, type: String, desc: 'Module version', regexp: SEMVER_REGEX
end
def module_namespace
strong_memoize(:module_namespace) do
find_namespace(params[:module_namespace])
end
end
def finder_params
{
package_type: :terraform_module,
package_name: "#{params[:module_name]}/#{params[:module_system]}"
}.tap do |finder_params|
finder_params[:package_version] = params[:module_version] if params.has_key?(:module_version)
end
end
def packages
strong_memoize(:packages) do
::Packages::GroupPackagesFinder.new(
current_user,
module_namespace,
finder_params
).execute
end
end
def package
strong_memoize(:package) do
packages.first
end
end
def package_file
strong_memoize(:package_file) do
package.package_files.first
end
end
end
params do
requires :module_namespace, type: String, desc: "Group's ID or slug", regexp: API::NO_SLASH_URL_PART_REGEX
includes :module_name
end
namespace 'packages/terraform/modules/v1/:module_namespace/:module_name/:module_system', requirements: TERRAFORM_MODULE_REQUIREMENTS do
authenticate_with do |accept|
accept.token_types(:personal_access_token, :deploy_token, :job_token)
.sent_through(:http_bearer_token)
end
after_validation do
authorize_read_package!(package || module_namespace)
end
get 'versions' do
presenter = ::Terraform::ModulesPresenter.new(packages, params[:module_system])
present presenter, with: ::API::Entities::Terraform::ModuleVersions
end
params do
includes :module_version
end
namespace '*module_version', requirements: TERRAFORM_MODULE_VERSION_REQUIREMENTS do
after_validation do
not_found! unless package && package_file
end
get 'download' do
module_file_path = api_v4_packages_terraform_modules_v1_module_version_file_path(
module_namespace: params[:module_namespace],
module_name: params[:module_name],
module_system: params[:module_system],
module_version: params[:module_version]
)
jwt_token = Gitlab::TerraformRegistryToken.from_token(token_from_namespace_inheritable).encoded
header 'X-Terraform-Get', module_file_path.sub(%r{module_version/file$}, "#{params[:module_version]}/file?token=#{jwt_token}&archive=tgz")
status :no_content
end
namespace 'file' do
authenticate_with do |accept|
accept.token_types(:deploy_token_from_jwt, :job_token_from_jwt, :personal_access_token_from_jwt).sent_through(:token_param)
end
get do
track_package_event('pull_package', :terraform_module)
present_carrierwave_file!(package_file.file)
end
end
end
end
params do
requires :id, type: String, desc: 'The ID or full path of a project'
includes :module_name
includes :module_version
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
namespace ':id/packages/terraform/modules/:module_name/:module_system/*module_version/file' do
authenticate_with do |accept|
accept.token_types(:deploy_token).sent_through(:http_deploy_token_header)
accept.token_types(:job_token).sent_through(:http_job_token_header)
accept.token_types(:personal_access_token).sent_through(:http_private_token_header)
end
desc 'Workhorse authorize Terraform Module package file' do
detail 'This feature was introduced in GitLab 13.11'
end
put 'authorize' do
authorize_workhorse!(
subject: authorized_user_project,
maximum_size: authorized_user_project.actual_limits.terraform_module_max_file_size
)
end
desc 'Upload Terraform Module package file' do
detail 'This feature was introduced in GitLab 13.11'
end
params do
requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)'
end
put do
authorize_upload!(authorized_user_project)
bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:terraform_module_max_file_size, params[:file].size)
create_package_file_params = {
module_name: params['module_name'],
module_system: params['module_system'],
module_version: params['module_version'],
file: params['file'],
build: current_authenticated_job
}
result = ::Packages::TerraformModule::CreatePackageService
.new(authorized_user_project, current_user, create_package_file_params)
.execute
render_api_error!(result[:message], result[:http_status]) if result[:status] == :error
track_package_event('push_package', :terraform_module)
created!
rescue ObjectStorage::RemoteStoreError => e
Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: authorized_user_project.id })
forbidden!
end
end
end
end
end
end
end
end
......@@ -10,7 +10,17 @@ module Gitlab
attr_reader :location
validates :location, inclusion: { in: %i[http_basic_auth http_token token_param] }
validates :location, inclusion: {
in: %i[
http_basic_auth
http_token
http_bearer_token
http_deploy_token_header
http_job_token_header
http_private_token_header
token_param
]
}
def initialize(location)
@location = location
......@@ -23,6 +33,14 @@ module Gitlab
extract_from_http_basic_auth request
when :http_token
extract_from_http_token request
when :http_bearer_token
extract_from_http_bearer_token request
when :http_deploy_token_header
extract_from_http_deploy_token_header request
when :http_job_token_header
extract_from_http_job_token_header request
when :http_private_token_header
extract_from_http_private_token_header request
when :token_param
extract_from_token_param request
end
......@@ -44,6 +62,34 @@ module Gitlab
UsernameAndPassword.new(nil, password)
end
def extract_from_http_bearer_token(request)
password = request.headers['Authorization']
return unless password.present?
UsernameAndPassword.new(nil, password.split(' ').last)
end
def extract_from_http_deploy_token_header(request)
password = request.headers['Deploy-Token']
return unless password.present?
UsernameAndPassword.new(nil, password)
end
def extract_from_http_job_token_header(request)
password = request.headers['Job-Token']
return unless password.present?
UsernameAndPassword.new(nil, password)
end
def extract_from_http_private_token_header(request)
password = request.headers['Private-Token']
return unless password.present?
UsernameAndPassword.new(nil, password)
end
def extract_from_token_param(request)
password = request.query_parameters['token']
return unless password.present?
......
......@@ -77,6 +77,10 @@ module Gitlab
/x.freeze
end
def terraform_module_package_name_regex
@terraform_module_package_name_regex ||= %r{\A[-a-z0-9]+\/[-a-z0-9]+\z}.freeze
end
def pypi_version_regex
# See the official regex: https://github.com/pypa/packaging/blob/16.7/packaging/version.py#L159
......@@ -149,7 +153,7 @@ module Gitlab
end
def semver_regex
@semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options)
@semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options).freeze
end
# These partial semver regexes are intended for use in composing other
......
......@@ -47,3 +47,6 @@
- i_package_tag_delete_package
- i_package_tag_pull_package
- i_package_tag_push_package
- i_package_terraform_module_delete_package
- i_package_terraform_module_pull_package
- i_package_terraform_module_push_package
......@@ -95,3 +95,11 @@
category: user_packages
aggregation: weekly
redis_slot: package
- name: i_package_terraform_module_deploy_token
category: deploy_token_packages
aggregation: weekly
redis_slot: package
- name: i_package_terraform_module_user
category: user_packages
aggregation: weekly
redis_slot: package
......@@ -20184,6 +20184,9 @@ msgstr ""
msgid "Maximum PyPI package file size in bytes"
msgstr ""
msgid "Maximum Terraform Module package file size in bytes"
msgstr ""
msgid "Maximum Users"
msgstr ""
......
......@@ -129,6 +129,25 @@ FactoryBot.define do
end
end
factory :terraform_module_package do
sequence(:name) { |n| "module-#{n}/system" }
version { '1.0.0' }
package_type { :terraform_module }
after :create do |package|
create :package_file, :terraform_module, package: package
end
trait :with_build do
after :create do |package|
user = package.project.creator
pipeline = create(:ci_pipeline, user: user)
create(:ci_build, user: user, pipeline: pipeline)
create :package_build_info, package: package, pipeline: pipeline
end
end
end
factory :nuget_package do
sequence(:name) { |n| "NugetPackage#{n}"}
sequence(:version) { |n| "1.0.#{n}" }
......
......@@ -254,6 +254,13 @@ FactoryBot.define do
size { 400.kilobytes }
end
trait(:terraform_module) do
file_fixture { 'spec/fixtures/packages/terraform_module/module-system-v1.0.0.tgz' }
file_name { 'module-system-v1.0.0.tgz' }
file_sha1 { 'abf850accb1947c0c0e3ef4b441b771bb5c9ae3c' }
size { 806.bytes }
end
trait(:nuget) do
package
file_fixture { 'spec/fixtures/packages/nuget/package.nupkg' }
......
{
"type": "object",
"required" : ["versions"],
"optional" : ["source"],
"properties" : {
"source": { "type": "string" },
"versions": {
"minItems": 0,
"items": { "$ref": "./version.json" }
}
}
}
{
"type": "array",
"items": { "$ref": "./module.json" }
}
{
"type": "object",
"required": ["version", "submodules", "root"],
"properties": {
"version": {
"type": "string"
},
"submodules": {
"type": "array",
"maxItems": 0
},
"root": {
"type": "object",
"required": ["dependencies", "providers"],
"properties": {
"dependencies": {
"type": "array",
"maxItems": 0
},
"providers": {
"type": "array",
"items": {
"type": "object",
"required": ["name", "version"],
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
}
}
}
}
}
}
}
}
{
"type": "object",
"required" : ["modules"],
"properties" : {
"modules": {
"items": { "$ref": "./module.json" }
}
}
}
......@@ -4,6 +4,6 @@ require 'spec_helper'
RSpec.describe GitlabSchema.types['PackageTypeEnum'] do
it 'exposes all package types' do
expect(described_class.values.keys).to contain_exactly(*%w[MAVEN NPM CONAN NUGET PYPI COMPOSER GENERIC GOLANG DEBIAN RUBYGEMS HELM])
expect(described_class.values.keys).to contain_exactly(*%w[MAVEN NPM CONAN NUGET PYPI COMPOSER GENERIC GOLANG DEBIAN RUBYGEMS HELM TERRAFORM_MODULE])
end
end
......@@ -73,6 +73,90 @@ RSpec.describe Gitlab::APIAuthentication::TokenLocator do
end
end
context 'with :http_bearer_token' do
let(:type) { :http_bearer_token }
context 'without credentials' do
let(:request) { double(headers: {}) }
it 'returns nil' do
expect(subject).to be_nil
end
end
context 'with credentials' do
let(:password) { 'bar' }
let(:request) { double(headers: { "Authorization" => "Bearer #{password}" }) }
it 'returns the credentials' do
expect(subject.password).to eq(password)
end
end
end
context 'with :http_deploy_token_header' do
let(:type) { :http_deploy_token_header }
context 'without credentials' do
let(:request) { double(headers: {}) }
it 'returns nil' do
expect(subject).to be(nil)
end
end
context 'with credentials' do
let(:password) { 'bar' }
let(:request) { double(headers: { 'Deploy-Token' => password }) }
it 'returns the credentials' do
expect(subject.password).to eq(password)
end
end
end
context 'with :http_job_token_header' do
let(:type) { :http_job_token_header }
context 'without credentials' do
let(:request) { double(headers: {}) }
it 'returns nil' do
expect(subject).to be(nil)
end
end
context 'with credentials' do
let(:password) { 'bar' }
let(:request) { double(headers: { 'Job-Token' => password }) }
it 'returns the credentials' do
expect(subject.password).to eq(password)
end
end
end
context 'with :http_private_token_header' do
let(:type) { :http_private_token_header }
context 'without credentials' do
let(:request) { double(headers: {}) }
it 'returns nil' do
expect(subject).to be(nil)
end
end
context 'with credentials' do
let(:password) { 'bar' }
let(:request) { double(headers: { 'Private-Token' => password }) }
it 'returns the credentials' do
expect(subject.password).to eq(password)
end
end
end
context 'with :token_param' do
let(:type) { :token_param }
......
......@@ -427,6 +427,19 @@ RSpec.describe Gitlab::Regex do
it { is_expected.not_to match('%2e%2e%2fmy_package') }
end
describe '.terraform_module_package_name_regex' do
subject { described_class.terraform_module_package_name_regex }
it { is_expected.to match('my-module/my-system') }
it { is_expected.to match('my/module') }
it { is_expected.not_to match('my-module') }
it { is_expected.not_to match('My-Module') }
it { is_expected.not_to match('my_module') }
it { is_expected.not_to match('my.module') }
it { is_expected.not_to match('../../../my-module') }
it { is_expected.not_to match('%2e%2e%2fmy-module') }
end
describe '.pypi_version_regex' do
subject { described_class.pypi_version_regex }
......
......@@ -14,7 +14,7 @@ RSpec.describe Gitlab::UsageDataCounters::PackageEventCounter, :clean_gitlab_red
end
it 'includes the right events' do
expect(described_class::KNOWN_EVENTS.size).to eq 48
expect(described_class::KNOWN_EVENTS.size).to eq 51
end
described_class::KNOWN_EVENTS.each do |event|
......
......@@ -208,6 +208,19 @@ RSpec.describe Packages::Package, type: :model do
it { is_expected.not_to allow_value("@scope%2e%2e%fpackage").for(:name) }
it { is_expected.not_to allow_value("@scope/sub/package").for(:name) }
end
context 'terraform module package' do
subject { build_stubbed(:terraform_module_package) }
it { is_expected.to allow_value('my-module/my-system').for(:name) }
it { is_expected.to allow_value('my/module').for(:name) }
it { is_expected.not_to allow_value('my-module').for(:name) }
it { is_expected.not_to allow_value('My-Module').for(:name) }
it { is_expected.not_to allow_value('my_module').for(:name) }
it { is_expected.not_to allow_value('my.module').for(:name) }
it { is_expected.not_to allow_value('../../../my-module').for(:name) }
it { is_expected.not_to allow_value('%2e%2e%2fmy-module').for(:name) }
end
end
describe '#version' do
......@@ -395,6 +408,7 @@ RSpec.describe Packages::Package, type: :model do
end
it_behaves_like 'validating version to be SemVer compliant for', :npm_package
it_behaves_like 'validating version to be SemVer compliant for', :terraform_module_package
context 'nuget package' do
it_behaves_like 'validating version to be SemVer compliant for', :nuget_package
......@@ -492,6 +506,26 @@ RSpec.describe Packages::Package, type: :model do
end
end
describe '.with_package_type' do
let!(:package1) { create(:terraform_module_package) }
let!(:package2) { create(:npm_package) }
let(:package_type) { :terraform_module }
subject { described_class.with_package_type(package_type) }
it { is_expected.to eq([package1]) }
end
describe '.without_package_type' do
let!(:package1) { create(:npm_package) }
let!(:package2) { create(:terraform_module_package) }
let(:package_type) { :terraform_module }
subject { described_class.without_package_type(package_type) }
it { is_expected.to eq([package1]) }
end
context 'version scopes' do
let!(:package1) { create(:npm_package, version: '1.0.0') }
let!(:package2) { create(:npm_package, version: '1.0.1') }
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe ::Terraform::ModulesPresenter do
let_it_be(:project) { create(:project) }
let_it_be(:module_system) { 'my-system' }
let_it_be(:package_name) { "my-module/#{module_system}" }
let_it_be(:package1) { create(:terraform_module_package, version: '1.0.1', project: project, name: package_name) }
let_it_be(:package2) { create(:terraform_module_package, version: '1.0.10', project: project, name: package_name) }
let(:packages) { project.packages.terraform_module.with_name(package_name) }
let(:presenter) { described_class.new(packages, module_system) }
describe '#modules' do
subject { presenter.modules }
it { is_expected.to be_an(Array) }
it { expect(subject.first).to be_a(Hash) }
it { expect(subject).to match_schema('public_api/v4/packages/terraform/modules/v1/modules') }
end
end
......@@ -37,6 +37,16 @@ RSpec.describe API::ProjectPackages do
end
end
context 'with terraform module package' do
let_it_be(:terraform_module_package) { create(:terraform_module_package, project: project) }
it 'filters out terraform module packages when no package_type filter is set' do
subject
expect(json_response).not_to include(a_hash_including('package_type' => 'terraform_module'))
end
end
context 'project is private' do
let(:project) { create(:project, :private) }
......
This diff is collapsed.
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Terraform::ServicesController do
describe 'GET /.well-known/terraform.json' do
subject { get '/.well-known/terraform.json' }
it 'responds with terraform service discovery' do
subject
expect(json_response['modules.v1']).to eq("/api/#{::API::API.version}/packages/terraform/modules/v1/")
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Packages::TerraformModule::CreatePackageService do
let_it_be(:namespace) { create(:namespace) }
let_it_be(:project) { create(:project, namespace: namespace) }
let_it_be(:user) { create(:user) }
let_it_be(:sha256) { '440e5e148a25331bbd7991575f7d54933c0ebf6cc735a18ee5066ac1381bb590' }
let_it_be(:temp_file) { Tempfile.new('test') }
let_it_be(:file) { UploadedFile.new(temp_file.path, sha256: sha256) }
let(:overrides) { {} }
let(:params) do
{
module_name: 'foo',
module_system: 'bar',
module_version: '1.0.1',
file: file,
file_name: 'foo-bar-1.0.1.tgz'
}.merge(overrides)
end
subject { described_class.new(project, user, params).execute }
describe '#execute' do
context 'valid package' do
it 'creates a package' do
expect { subject }
.to change { ::Packages::Package.count }.by(1)
.and change { ::Packages::Package.terraform_module.count }.by(1)
end
end
context 'package already exists elsewhere' do
let(:project2) { create(:project, namespace: namespace) }
let!(:existing_package) { create(:terraform_module_package, project: project2, name: 'foo/bar', version: '1.0.0') }
it { expect(subject[:http_status]).to eq 403 }
it { expect(subject[:message]).to be 'Package already exists.' }
end
context 'version already exists' do
let!(:existing_version) { create(:terraform_module_package, project: project, name: 'foo/bar', version: '1.0.1') }
it { expect(subject[:http_status]).to eq 403 }
it { expect(subject[:message]).to be 'Package version already exists.' }
end
context 'with empty version' do
let(:overrides) { { module_version: '' } }
it { expect(subject[:http_status]).to eq 400 }
it { expect(subject[:message]).to eq 'Version is empty.' }
end
end
end
# frozen_string_literal: true
RSpec.shared_examples 'when package feature is disabled' do
before do
stub_config(packages: { enabled: false })
end
it_behaves_like 'returning response status', :not_found
end
RSpec.shared_examples 'without authentication' do
it_behaves_like 'returning response status', :unauthorized
end
RSpec.shared_examples 'with authentication' do
where(:user_role, :token_header, :token_type, :valid_token, :status) do
:guest | 'PRIVATE-TOKEN' | :personal_access_token | true | :not_found
:guest | 'PRIVATE-TOKEN' | :personal_access_token | false | :unauthorized
:guest | 'DEPLOY-TOKEN' | :deploy_token | true | :not_found
:guest | 'DEPLOY-TOKEN' | :deploy_token | false | :unauthorized
:guest | 'JOB-TOKEN' | :job_token | true | :not_found
:guest | 'JOB-TOKEN' | :job_token | false | :unauthorized
:reporter | 'PRIVATE-TOKEN' | :personal_access_token | true | :not_found
:reporter | 'PRIVATE-TOKEN' | :personal_access_token | false | :unauthorized
:reporter | 'DEPLOY-TOKEN' | :deploy_token | true | :not_found
:reporter | 'DEPLOY-TOKEN' | :deploy_token | false | :unauthorized
:reporter | 'JOB-TOKEN' | :job_token | true | :not_found
:reporter | 'JOB-TOKEN' | :job_token | false | :unauthorized
:developer | 'PRIVATE-TOKEN' | :personal_access_token | true | :not_found
:developer | 'PRIVATE-TOKEN' | :personal_access_token | false | :unauthorized
:developer | 'DEPLOY-TOKEN' | :deploy_token | true | :not_found
:developer | 'DEPLOY-TOKEN' | :deploy_token | false | :unauthorized
:developer | 'JOB-TOKEN' | :job_token | true | :not_found
:developer | 'JOB-TOKEN' | :job_token | false | :unauthorized
end
with_them do
before do
project.send("add_#{user_role}", user) unless user_role == :anonymous
end
let(:token) { valid_token ? tokens[token_type] : 'invalid-token123' }
let(:headers) { { token_header => token } }
it_behaves_like 'returning response status', params[:status]
end
end
RSpec.shared_examples 'an unimplemented route' do
it_behaves_like 'without authentication'
it_behaves_like 'with authentication'
it_behaves_like 'when package feature is disabled'
end
RSpec.shared_examples 'grants terraform module download' do |user_type, status, add_member = true|
context "for user type #{user_type}" do
before do
group.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
it_behaves_like 'returning response status', status
it 'returns a valid response' do
subject
expect(response.headers).to include 'X-Terraform-Get'
end
end
end
RSpec.shared_examples 'returns terraform module packages' do |user_type, status, add_member = true|
context "for user type #{user_type}" do
before do
group.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
it_behaves_like 'returning response status', status
it 'returning a valid response' do
subject
expect(json_response).to match_schema('public_api/v4/packages/terraform/modules/v1/versions')
end
end
end
RSpec.shared_examples 'returns no terraform module packages' do |user_type, status, add_member = true|
context "for user type #{user_type}" do
before do
group.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
it_behaves_like 'returning response status', status
it 'returns a response with no versions' do
subject
expect(json_response['modules'][0]['versions'].size).to eq(0)
end
end
end
RSpec.shared_examples 'grants terraform module packages access' do |user_type, status, add_member = true|
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
it_behaves_like 'returning response status', status
end
end
RSpec.shared_examples 'grants terraform module package file access' do |user_type, status, add_member = true|
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
it_behaves_like 'returning response status', status
it_behaves_like 'a package tracking event', described_class.name, 'pull_package'
end
end
RSpec.shared_examples 'rejects terraform module packages access' do |user_type, status, add_member = true|
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
it_behaves_like 'returning response status', status
end
end
RSpec.shared_examples 'process terraform module workhorse authorization' do |user_type, status, add_member = true|
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
it_behaves_like 'returning response status', status
it 'has the proper content type' do
subject
expect(response.media_type).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
end
context 'with a request that bypassed gitlab-workhorse' do
let(:headers) do
{ 'HTTP_PRIVATE_TOKEN' => personal_access_token.token }
.merge(workhorse_headers)
.tap { |h| h.delete(Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER) }
end
before do
project.add_maintainer(user)
end
it_behaves_like 'returning response status', :forbidden
end
end
end
RSpec.shared_examples 'process terraform module upload' do |user_type, status, add_member = true|
RSpec.shared_examples 'creates terraform module package files' do
it 'creates package files', :aggregate_failures do
expect { subject }
.to change { project.packages.count }.by(1)
.and change { Packages::PackageFile.count }.by(1)
expect(response).to have_gitlab_http_status(status)
package_file = project.packages.last.package_files.reload.last
expect(package_file.file_name).to eq('mymodule-mysystem-1.0.0.tgz')
end
end
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
context 'with object storage disabled' do
before do
stub_package_file_object_storage(enabled: false)
end
context 'without a file from workhorse' do
let(:send_rewritten_field) { false }
it_behaves_like 'returning response status', :bad_request
end
context 'with correct params' do
it_behaves_like 'package workhorse uploads'
it_behaves_like 'creates terraform module package files'
it_behaves_like 'a package tracking event', described_class.name, 'push_package'
end
end
context 'with object storage enabled' do
let(:tmp_object) do
fog_connection.directories.new(key: 'packages').files.create( # rubocop:disable Rails/SaveBang
key: "tmp/uploads/#{file_name}",
body: 'content'
)
end
let(:fog_file) { fog_to_uploaded_file(tmp_object) }
let(:params) { { file: fog_file, 'file.remote_id' => file_name } }
context 'and direct upload enabled' do
let(:fog_connection) do
stub_package_file_object_storage(direct_upload: true)
end
it_behaves_like 'creates terraform module package files'
['123123', '../../123123'].each do |remote_id|
context "with invalid remote_id: #{remote_id}" do
let(:params) do
{
file: fog_file,
'file.remote_id' => remote_id
}
end
it_behaves_like 'returning response status', :forbidden
end
end
end
context 'and direct upload disabled' do
context 'and background upload disabled' do
let(:fog_connection) do
stub_package_file_object_storage(direct_upload: false, background_upload: false)
end
it_behaves_like 'creates terraform module package files'
end
context 'and background upload enabled' do
let(:fog_connection) do
stub_package_file_object_storage(direct_upload: false, background_upload: true)
end
it_behaves_like 'creates terraform module package files'
end
end
end
end
end
......@@ -205,6 +205,7 @@ RSpec.shared_examples 'filters on each package_type' do |is_project: false|
let_it_be(:package9) { create(:debian_package, project: project) }
let_it_be(:package10) { create(:rubygems_package, project: project) }
let_it_be(:package11) { create(:helm_package, project: project) }
let_it_be(:package12) { create(:terraform_module_package, project: project) }
Packages::Package.package_types.keys.each do |package_type|
context "for package type #{package_type}" do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment