Commit f0c4fbea authored by Filipa Lacerda's avatar Filipa Lacerda

Merge branch 'master' into 22643-manual-job-page

* master: (24 commits)
  Allow local tests to use a modified Gitaly
  Update check.md
  add deprecation and removal issue to docs
  Add status attribute to runner api entity
  Fix typos in a code comment
  Just try to detect and assign once
  Modify `LDAP::Person` to return username value based on attributes
  Create Kubernetes based on Application Templates
  Fix a bug where charlock_holmes was used needlessly to encode strings
  Ignore the Migration/Datetime cop in a migration that fix a column type to datetime_with_timezone
  Use `__` instead of `s__` when context is not required
  Enclose props in quotes
  Update some Gitaly annotations in Gitlab::Shell
  Prepare Gitlab::Git::Repository#rebase for Gitaly migration
  add missing changelog
  refactor spec, add docs
  deprecate check integrity task
  add lock specs
  add locks chek
  remove max-depth flag so it works with subgroups
  ...
parents 2cf2df8e 96ef3e70
......@@ -42,28 +42,28 @@ export default {
v-if="isGroup"
css-class="number-subgroups"
icon-name="folder"
:title="s__('Subgroups')"
:value=item.subgroupCount
:title="__('Subgroups')"
:value="item.subgroupCount"
/>
<item-stats-value
v-if="isGroup"
css-class="number-projects"
icon-name="bookmark"
:title="s__('Projects')"
:value=item.projectCount
:title="__('Projects')"
:value="item.projectCount"
/>
<item-stats-value
v-if="isGroup"
css-class="number-users"
icon-name="users"
:title="s__('Members')"
:value=item.memberCount
:title="__('Members')"
:value="item.memberCount"
/>
<item-stats-value
v-if="isProject"
css-class="project-stars"
icon-name="star"
:value=item.starCount
:value="item.starCount"
/>
<item-stats-value
css-class="item-visibility"
......
module DeploymentPlatform
def deployment_platform
@deployment_platform ||=
find_cluster_platform_kubernetes ||
find_kubernetes_service_integration ||
build_cluster_and_deployment_platform
end
private
def find_cluster_platform_kubernetes
clusters.find_by(enabled: true)&.platform_kubernetes
end
def find_kubernetes_service_integration
services.deployment.reorder(nil).find_by(active: true)
end
def build_cluster_and_deployment_platform
return unless kubernetes_service_template
cluster = ::Clusters::Cluster.create(cluster_attributes_from_service_template)
cluster.platform_kubernetes if cluster.persisted?
end
def kubernetes_service_template
@kubernetes_service_template ||= KubernetesService.active.find_by_template
end
def cluster_attributes_from_service_template
{
name: 'kubernetes-template',
projects: [self],
provider_type: :user,
platform_type: :kubernetes,
platform_kubernetes_attributes: platform_kubernetes_attributes_from_service_template
}
end
def platform_kubernetes_attributes_from_service_template
{
api_url: kubernetes_service_template.api_url,
ca_pem: kubernetes_service_template.ca_pem,
token: kubernetes_service_template.token,
namespace: kubernetes_service_template.namespace
}
end
end
......@@ -19,6 +19,7 @@ class Project < ActiveRecord::Base
include Routable
include GroupDescendant
include Gitlab::SQL::Pattern
include DeploymentPlatform
extend Gitlab::ConfigHelper
extend Gitlab::CurrentSettings
......@@ -904,12 +905,6 @@ class Project < ActiveRecord::Base
@ci_service ||= ci_services.reorder(nil).find_by(active: true)
end
# TODO: This will be extended for multiple enviroment clusters
def deployment_platform
@deployment_platform ||= clusters.find_by(enabled: true)&.platform_kubernetes
@deployment_platform ||= services.where(category: :deployment).reorder(nil).find_by(active: true)
end
def monitoring_services
services.where(category: :monitoring)
end
......
......@@ -44,6 +44,7 @@ class Service < ActiveRecord::Base
scope :pipeline_hooks, -> { where(pipeline_events: true, active: true) }
scope :wiki_page_hooks, -> { where(wiki_page_events: true, active: true) }
scope :external_issue_trackers, -> { issue_trackers.active.without_defaults }
scope :deployment, -> { where(category: 'deployment') }
default_value_for :category, 'common'
......@@ -271,6 +272,10 @@ class Service < ActiveRecord::Base
nil
end
def self.find_by_template
find_by(template: true)
end
private
def cache_project_has_external_issue_tracker
......
---
title: Fix gitlab-rake gitlab:import:repos import schedule
merge_request: 15931
author:
type: fixed
---
title: Allow automatic creation of Kubernetes Integration from template
merge_request: 16104
author:
type: added
---
title: Add online and status attribute to runner api entity
merge_request: 11750
author:
type: added
---
title: Modify `LDAP::Person` to return username value based on attributes
merge_request:
author:
type: fixed
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
# rubocop:disable Migration/Datetime
class ScheduleIssuesClosedAtTypeChange < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
......
......@@ -28,19 +28,25 @@ exactly which repositories are causing the trouble.
### Check all GitLab repositories
>**Note:**
>
> - `gitlab:repo:check` has been deprecated in favor of `gitlab:git:fsck`
> - [Deprecated][ce-15931] in GitLab 10.4.
> - `gitlab:repo:check` will be removed in the future. [Removal issue][ce-41699]
This task loops through all repositories on the GitLab server and runs the
3 integrity checks described previously.
**Omnibus Installation**
```
sudo gitlab-rake gitlab:repo:check
sudo gitlab-rake gitlab:git:fsck
```
**Source Installation**
```bash
sudo -u git -H bundle exec rake gitlab:repo:check RAILS_ENV=production
sudo -u git -H bundle exec rake gitlab:git:fsck RAILS_ENV=production
```
### Check repositories for a specific user
......@@ -76,3 +82,6 @@ The LDAP check Rake task will test the bind_dn and password credentials
(if configured) and will list a sample of LDAP users. This task is also
executed as part of the `gitlab:check` task, but can run independently.
See [LDAP Rake Tasks - LDAP Check](ldap.md#check) for details.
[ce-15931]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/15931
[ce-41699]: https://gitlab.com/gitlab-org/gitlab-ce/issues/41699
......@@ -30,14 +30,18 @@ Example response:
"description": "test-1-20150125",
"id": 6,
"is_shared": false,
"name": null
"name": null,
"online": true,
"status": "online"
},
{
"active": true,
"description": "test-2-20150125",
"id": 8,
"is_shared": false,
"name": null
"name": null,
"online": false,
"status": "offline"
}
]
```
......@@ -69,28 +73,36 @@ Example response:
"description": "shared-runner-1",
"id": 1,
"is_shared": true,
"name": null
"name": null,
"online": true,
"status": "online"
},
{
"active": true,
"description": "shared-runner-2",
"id": 3,
"is_shared": true,
"name": null
"name": null,
"online": false
"status": "offline"
},
{
"active": true,
"description": "test-1-20150125",
"id": 6,
"is_shared": false,
"name": null
"name": null,
"online": true
"status": "paused"
},
{
"active": true,
"description": "test-2-20150125",
"id": 8,
"is_shared": false,
"name": null
"name": null,
"online": false,
"status": "offline"
}
]
```
......@@ -122,6 +134,8 @@ Example response:
"is_shared": false,
"contacted_at": "2016-01-25T16:39:48.066Z",
"name": null,
"online": true,
"status": "online",
"platform": null,
"projects": [
{
......@@ -176,6 +190,8 @@ Example response:
"is_shared": false,
"contacted_at": "2016-01-25T16:39:48.066Z",
"name": null,
"online": true,
"status": "online",
"platform": null,
"projects": [
{
......@@ -327,14 +343,18 @@ Example response:
"description": "test-2-20150125",
"id": 8,
"is_shared": false,
"name": null
"name": null,
"online": false,
"status": "offline"
},
{
"active": true,
"description": "development_runner",
"id": 5,
"is_shared": true,
"name": null
"name": null,
"online": true
"status": "paused"
}
]
```
......@@ -364,7 +384,9 @@ Example response:
"description": "test-2016-02-01",
"id": 9,
"is_shared": false,
"name": null
"name": null,
"online": true,
"status": "online"
}
```
......
......@@ -97,6 +97,29 @@ describe 'Gitaly Request count tests' do
end
```
## Running tests with a locally modified version of Gitaly
Normally, gitlab-ce/ee tests use a local clone of Gitaly in `tmp/tests/gitaly`
pinned at the version specified in GITALY_SERVER_VERSION. If you want
to run tests locally against a modified version of Gitaly you can
replace `tmp/tests/gitaly` with a symlink.
```shell
rm -rf tmp/tests/gitaly
ln -s /path/to/gitaly tmp/tests/gitaly
```
Make sure you run `make` in your local Gitaly directory before running
tests. Otherwise, Gitaly will fail to boot.
If you make changes to your local Gitaly in between test runs you need
to manually run `make` again.
Note that CI tests will not use your locally modified version of
Gitaly. To use a custom Gitaly version in CI you need to update
GITALY_SERVER_VERSION. You can use the format `=revision` to use a
non-tagged commit from https://gitlab.com/gitlab-org/gitaly in CI.
---
[Return to Development documentation](README.md)
......@@ -862,6 +862,8 @@ module API
expose :active
expose :is_shared
expose :name
expose :online?, as: :online
expose :status
end
class RunnerDetails < Runner
......
......@@ -14,14 +14,7 @@ module Gitlab
ENCODING_CONFIDENCE_THRESHOLD = 50
def encode!(message)
return nil unless message.respond_to?(:force_encoding)
return message if message.encoding == Encoding::UTF_8 && message.valid_encoding?
if message.respond_to?(:frozen?) && message.frozen?
message = message.dup
end
message.force_encoding("UTF-8")
message = force_encode_utf8(message)
return message if message.valid_encoding?
# return message if message type is binary
......@@ -35,6 +28,8 @@ module Gitlab
# encode and clean the bad chars
message.replace clean(message)
rescue ArgumentError
return nil
rescue
encoding = detect ? detect[:encoding] : "unknown"
"--broken encoding: #{encoding}"
......@@ -54,8 +49,8 @@ module Gitlab
end
def encode_utf8(message)
return nil unless message.is_a?(String)
return message if message.encoding == Encoding::UTF_8 && message.valid_encoding?
message = force_encode_utf8(message)
return message if message.valid_encoding?
detect = CharlockHolmes::EncodingDetector.detect(message)
if detect && detect[:encoding]
......@@ -69,6 +64,8 @@ module Gitlab
else
clean(message)
end
rescue ArgumentError
return nil
end
def encode_binary(s)
......@@ -83,6 +80,15 @@ module Gitlab
private
def force_encode_utf8(message)
raise ArgumentError unless message.respond_to?(:force_encoding)
return message if message.encoding == Encoding::UTF_8 && message.valid_encoding?
message = message.dup if message.respond_to?(:frozen?) && message.frozen?
message.force_encoding("UTF-8")
end
def clean(message)
message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
.encode("UTF-8")
......
......@@ -11,7 +11,7 @@ module Gitlab
include Gitlab::EncodingHelper
def ref_name(ref)
encode_utf8(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '')
encode!(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '')
end
def branch_name(ref)
......
......@@ -50,12 +50,21 @@ module Gitlab
# to the caller to limit the number of blobs and blob_size_limit.
#
# Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798
def batch(repository, blob_references, blob_size_limit: nil)
blob_size_limit ||= MAX_DATA_DISPLAY_SIZE
def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE)
Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled|
if is_enabled
Gitlab::GitalyClient.allow_n_plus_1_calls do
blob_references.map do |sha, path|
find_by_gitaly(repository, sha, path, limit: blob_size_limit)
end
end
else
blob_references.map do |sha, path|
find_by_rugged(repository, sha, path, limit: blob_size_limit)
end
end
end
end
# Find LFS blobs given an array of sha ids
# Returns array of Gitlab::Git::Blob
......@@ -122,13 +131,23 @@ module Gitlab
)
end
def find_by_gitaly(repository, sha, path)
def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE)
path = path.sub(/\A\/*/, '')
path = '/' if path.empty?
name = File.basename(path)
entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE)
# Gitaly will think that setting the limit to 0 means unlimited, while
# the client might only need the metadata and thus set the limit to 0.
# In this method we'll then set the limit to 1, but clear the byte of data
# that we got back so for the outside world it looks like the limit was
# actually 0.
req_limit = limit == 0 ? 1 : limit
entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit)
return unless entry
entry.data = "" if limit == 0
case entry.type
when :COMMIT
new(
......
......@@ -97,6 +97,11 @@ module Gitlab
end
end
def update_branch(branch_name, newrev, oldrev)
ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name
update_ref_in_hooks(ref, newrev, oldrev)
end
private
# Returns [newrev, should_run_after_create, should_run_after_create_branch]
......
......@@ -1219,9 +1219,16 @@ module Gitlab
rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)
env = git_env_for_user(user)
if remote_repository.is_a?(RemoteRepository)
env.merge!(remote_repository.fetch_env)
remote_repo_path = GITALY_INTERNAL_URL
else
remote_repo_path = remote_repository.path
end
with_worktree(rebase_path, branch, env: env) do
run_git!(
%W(pull --rebase #{remote_repository.path} #{remote_branch}),
%W(pull --rebase #{remote_repo_path} #{remote_branch}),
chdir: rebase_path, env: env
)
......
......@@ -74,7 +74,7 @@ module Gitlab
def user_options(fields, value, limit)
options = {
attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq,
attributes: Gitlab::LDAP::Person.ldap_attributes(config),
base: config.base
}
......
......@@ -148,7 +148,7 @@ module Gitlab
def default_attributes
{
'username' => %w(uid userid sAMAccountName),
'username' => %w(uid sAMAccountName userid),
'email' => %w(mail email userPrincipalName),
'name' => 'cn',
'first_name' => 'givenName',
......
......@@ -6,6 +6,8 @@ module Gitlab
# Source: http://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/
AD_USER_DISABLED = Net::LDAP::Filter.ex("userAccountControl:1.2.840.113556.1.4.803", "2")
InvalidEntryError = Class.new(StandardError)
attr_accessor :entry, :provider
def self.find_by_uid(uid, adapter)
......@@ -29,11 +31,12 @@ module Gitlab
def self.ldap_attributes(config)
[
'dn', # Used in `dn`
config.uid, # Used in `uid`
*config.attributes['name'], # Used in `name`
*config.attributes['email'] # Used in `email`
]
'dn',
config.uid,
*config.attributes['name'],
*config.attributes['email'],
*config.attributes['username']
].compact.uniq
end
def self.normalize_dn(dn)
......@@ -60,6 +63,8 @@ module Gitlab
Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" }
@entry = entry
@provider = provider
validate_entry
end
def name
......@@ -71,7 +76,13 @@ module Gitlab
end
def username
uid
username = attribute_value(:username)
# Depending on the attribute, multiple values may
# be returned. We need only one for username.
# Ex. `uid` returns only one value but `mail` may
# return an array of multiple email addresses.
[username].flatten.first
end
def email
......@@ -104,6 +115,19 @@ module Gitlab
entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend
end
def validate_entry
allowed_attrs = self.class.ldap_attributes(config).map(&:downcase)
# Net::LDAP::Entry transforms keys to symbols. Change to strings to compare.
entry_attrs = entry.attribute_names.map { |n| n.to_s.downcase }
invalid_attrs = entry_attrs - allowed_attrs
if invalid_attrs.any?
raise InvalidEntryError,
"#{self.class.name} initialized with Net::LDAP::Entry containing invalid attributes(s): #{invalid_attrs}"
end
end
end
end
end
module Gitlab
module SetupHelper
class << self
# We cannot create config.toml files for all possible Gitaly configuations.
# For instance, if Gitaly is running on another machine then it makes no
# sense to write a config.toml file on the current machine. This method will
# only generate a configuration for the most common and simplest case: when
# we have exactly one Gitaly process and we are sure it is running locally
# because it uses a Unix socket.
# For development and testing purposes, an extra storage is added to gitaly,
# which is not known to Rails, but must be explicitly stubbed.
def gitaly_configuration_toml(gitaly_dir, gitaly_ruby: true)
storages = []
address = nil
Gitlab.config.repositories.storages.each do |key, val|
if address
if address != val['gitaly_address']
raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address."
end
elsif URI(val['gitaly_address']).scheme != 'unix'
raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses."
else
address = val['gitaly_address']
end
storages << { name: key, path: val['path'] }
end
if Rails.env.test?
storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s }
end
config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages }
config[:auth] = { token: 'secret' } if Rails.env.test?
config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby
config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
config[:bin_dir] = Gitlab.config.gitaly.client_path
TOML.dump(config)
end
# rubocop:disable Rails/Output
def create_gitaly_configuration(dir, force: false)
config_path = File.join(dir, 'config.toml')
FileUtils.rm_f(config_path) if force
File.open(config_path, File::WRONLY | File::CREAT | File::EXCL) do |f|
f.puts gitaly_configuration_toml(dir)
end
rescue Errno::EEXIST
puts "Skipping config.toml generation:"
puts "A configuration file already exists."
rescue ArgumentError => e
puts "Skipping config.toml generation:"
puts e.message
end
# rubocop:enable Rails/Output
end
end
end
......@@ -71,7 +71,6 @@ module Gitlab
# Ex.
# add_repository("/path/to/storage", "gitlab/gitlab-ci")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
def add_repository(storage, name)
relative_path = name.dup
relative_path << '.git' unless relative_path.end_with?('.git')
......@@ -100,7 +99,7 @@ module Gitlab
# Ex.
# import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874
def import_repository(storage, name, url)
# The timeout ensures the subprocess won't hang forever
cmd = gitlab_projects(storage, "#{name}.git")
......@@ -122,7 +121,6 @@ module Gitlab
# Ex.
# fetch_remote(my_repo, "upstream")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false)
gitaly_migrate(:fetch_remote) do |is_enabled|
if is_enabled
......@@ -142,7 +140,7 @@ module Gitlab
# Ex.
# mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873
def mv_repository(storage, path, new_path)
gitlab_projects(storage, "#{path}.git").mv_project("#{new_path}.git")
end
......@@ -156,7 +154,7 @@ module Gitlab
# Ex.
# fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci")
#
# Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one.
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817
def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path)
gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git")
.fork_repository(forked_to_storage, "#{forked_to_disk_path}.git")
......@@ -170,7 +168,7 @@ module Gitlab
# Ex.
# remove_repository("/path/to/storage", "gitlab/gitlab-ci")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873
def remove_repository(storage, name)
gitlab_projects(storage, "#{name}.git").rm_project
end
......@@ -221,7 +219,6 @@ module Gitlab
# Ex.
# add_namespace("/path/to/storage", "gitlab")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
def add_namespace(storage, name)
Gitlab::GitalyClient.migrate(:add_namespace) do |enabled|
if enabled
......@@ -243,7 +240,6 @@ module Gitlab
# Ex.
# rm_namespace("/path/to/storage", "gitlab")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
def rm_namespace(storage, name)
Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled|
if enabled
......@@ -261,7 +257,6 @@ module Gitlab
# Ex.
# mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
def mv_namespace(storage, old_name, new_name)
Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled|
if enabled
......
......@@ -387,14 +387,8 @@ namespace :gitlab do
namespace :repo do
desc "GitLab | Check the integrity of the repositories managed by GitLab"
task check: :environment do
Gitlab.config.repositories.storages.each do |name, repository_storage|
namespace_dirs = Dir.glob(File.join(repository_storage['path'], '*'))
namespace_dirs.each do |namespace_dir|
repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
end
end
puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red)
Rake::Task["gitlab:git:fsck"].execute
end
end
......@@ -461,35 +455,4 @@ namespace :gitlab do
puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red)
end
end
def check_repo_integrity(repo_dir)
puts "\nChecking repo at #{repo_dir.color(:yellow)}"
git_fsck(repo_dir)
check_config_lock(repo_dir)
check_ref_locks(repo_dir)
end
def git_fsck(repo_dir)
puts "Running `git fsck`".color(:yellow)
system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: repo_dir)
end
def check_config_lock(repo_dir)
config_exists = File.exist?(File.join(repo_dir, 'config.lock'))
config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green)
puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}"
end
def check_ref_locks(repo_dir)
lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock'))
if lock_files.present?
puts "Ref lock files exist:".color(:red)
lock_files.each do |lock_file|
puts " #{lock_file}"
end
else
puts "No ref lock files exist".color(:green)
end
end
end
......@@ -30,6 +30,20 @@ namespace :gitlab do
end
end
desc 'GitLab | Git | Check all repos integrity'
task fsck: :environment do
failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo|
check_config_lock(repo)
check_ref_locks(repo)
end
if failures.empty?
puts "Done".color(:green)
else
output_failures(failures)
end
end
def perform_git_cmd(cmd, message)
puts "Starting #{message} on all repositories"
......@@ -40,6 +54,8 @@ namespace :gitlab do
else
failures << repo
end
yield(repo) if block_given?
end
failures
......@@ -49,5 +65,24 @@ namespace :gitlab do
puts "The following repositories reported errors:".color(:red)
failures.each { |f| puts "- #{f}" }
end
def check_config_lock(repo_dir)
config_exists = File.exist?(File.join(repo_dir, 'config.lock'))
config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green)
puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}"
end
def check_ref_locks(repo_dir)
lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock'))
if lock_files.present?
puts "Ref lock files exist:".color(:red)
lock_files.each { |lock_file| puts " #{lock_file}" }
else
puts "No ref lock files exist".color(:green)
end
end
end
end
......@@ -21,8 +21,8 @@ namespace :gitlab do
command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test?
Gitlab::SetupHelper.create_gitaly_configuration(args.dir)
Dir.chdir(args.dir) do
create_gitaly_configuration
# In CI we run scripts/gitaly-test-build instead of this command
unless ENV['CI'].present?
Bundler.with_original_env { run_command!(command) }
......@@ -39,60 +39,7 @@ namespace :gitlab do
# Exclude gitaly-ruby configuration because that depends on the gitaly
# installation directory.
puts gitaly_configuration_toml(gitaly_ruby: false)
end
private
# We cannot create config.toml files for all possible Gitaly configuations.
# For instance, if Gitaly is running on another machine then it makes no
# sense to write a config.toml file on the current machine. This method will
# only generate a configuration for the most common and simplest case: when
# we have exactly one Gitaly process and we are sure it is running locally
# because it uses a Unix socket.
# For development and testing purposes, an extra storage is added to gitaly,
# which is not known to Rails, but must be explicitly stubbed.
def gitaly_configuration_toml(gitaly_ruby: true)
storages = []
address = nil
Gitlab.config.repositories.storages.each do |key, val|
if address
if address != val['gitaly_address']
raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address."
end
elsif URI(val['gitaly_address']).scheme != 'unix'
raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses."
else
address = val['gitaly_address']
end
storages << { name: key, path: val['path'] }
end
if Rails.env.test?
storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s }
end
config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages }
config[:auth] = { token: 'secret' } if Rails.env.test?
config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby
config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
config[:bin_dir] = Gitlab.config.gitaly.client_path
TOML.dump(config)
end
def create_gitaly_configuration
File.open("config.toml", File::WRONLY | File::CREAT | File::EXCL) do |f|
f.puts gitaly_configuration_toml
end
rescue Errno::EEXIST
puts "Skipping config.toml generation:"
puts "A configuration file already exists."
rescue ArgumentError => e
puts "Skipping config.toml generation:"
puts e.message
puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false)
end
end
end
......@@ -130,7 +130,7 @@ module Gitlab
def all_repos
Gitlab.config.repositories.storages.each_value do |repository_storage|
IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -type d -name *.git)) do |find|
find.each_line do |path|
yield path.chomp
end
......
......@@ -2,7 +2,7 @@ require 'spec_helper'
# Integration test that exports a file using the Import/Export feature
# It looks up for any sensitive word inside the JSON, so if a sensitive word is found
# we''l have to either include it adding the model that includes it to the +safe_list+
# we'll have to either include it adding the model that includes it to the +safe_list+
# or make sure the attribute is blacklisted in the +import_export.yml+ configuration
feature 'Import/Export - project export integration test', :js do
include Select2Helper
......
......@@ -120,6 +120,24 @@ describe Gitlab::EncodingHelper do
it 'returns empty string on conversion errors' do
expect { ext_class.encode_utf8('') }.not_to raise_error(ArgumentError)
end
context 'with strings that can be forcefully encoded into utf8' do
let(:test_string) do
"refs/heads/FixSymbolsTitleDropdown".encode("ASCII-8BIT")
end
let(:expected_string) do
"refs/heads/FixSymbolsTitleDropdown".encode("UTF-8")
end
subject { ext_class.encode_utf8(test_string) }
it "doesn't use CharlockHolmes if the encoding can be forced into utf_8" do
expect(CharlockHolmes::EncodingDetector).not_to receive(:detect)
expect(subject).to eq(expected_string)
expect(subject.encoding.name).to eq('UTF-8')
end
end
end
describe '#clean' do
......
......@@ -202,16 +202,6 @@ describe Gitlab::Git::Blob, seed_helper: true do
context 'limiting' do
subject { described_class.batch(repository, blob_references, blob_size_limit: blob_size_limit) }
context 'default' do
let(:blob_size_limit) { nil }
it 'limits to MAX_DATA_DISPLAY_SIZE' do
stub_const('Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE', 100)
expect(subject.first.data.size).to eq(100)
end
end
context 'positive' do
let(:blob_size_limit) { 10 }
......@@ -221,7 +211,10 @@ describe Gitlab::Git::Blob, seed_helper: true do
context 'zero' do
let(:blob_size_limit) { 0 }
it { expect(subject.first.data).to eq('') }
it 'only loads the metadata' do
expect(subject.first.size).not_to be(0)
expect(subject.first.data).to eq('')
end
end
context 'negative' do
......
......@@ -16,7 +16,7 @@ describe Gitlab::LDAP::Adapter do
expect(adapter).to receive(:ldap_search) do |arg|
expect(arg[:filter].to_s).to eq('(uid=johndoe)')
expect(arg[:base]).to eq('dc=example,dc=com')
expect(arg[:attributes]).to match(%w{dn uid cn mail email userPrincipalName})
expect(arg[:attributes]).to match(ldap_attributes)
end.and_return({})
adapter.users('uid', 'johndoe')
......@@ -26,7 +26,7 @@ describe Gitlab::LDAP::Adapter do
expect(adapter).to receive(:ldap_search).with(
base: 'uid=johndoe,ou=users,dc=example,dc=com',
scope: Net::LDAP::SearchScope_BaseObject,
attributes: %w{dn uid cn mail email userPrincipalName},
attributes: ldap_attributes,
filter: nil
).and_return({})
......@@ -63,7 +63,7 @@ describe Gitlab::LDAP::Adapter do
it 'uses the right uid attribute when non-default' do
stub_ldap_config(uid: 'sAMAccountName')
expect(adapter).to receive(:ldap_search).with(
hash_including(attributes: %w{dn sAMAccountName cn mail email userPrincipalName})
hash_including(attributes: ldap_attributes)
).and_return({})
adapter.users('sAMAccountName', 'johndoe')
......@@ -137,4 +137,8 @@ describe Gitlab::LDAP::Adapter do
end
end
end
def ldap_attributes
Gitlab::LDAP::Person.ldap_attributes(Gitlab::LDAP::Config.new('ldapmain'))
end
end
......@@ -8,13 +8,16 @@ describe Gitlab::LDAP::Person do
before do
stub_ldap_config(
options: {
'uid' => 'uid',
'attributes' => {
'name' => 'cn',
'email' => %w(mail email userPrincipalName)
'email' => %w(mail email userPrincipalName),
'username' => username_attribute
}
}
)
end
let(:username_attribute) { %w(uid sAMAccountName userid) }
describe '.normalize_dn' do
subject { described_class.normalize_dn(given) }
......@@ -44,6 +47,34 @@ describe Gitlab::LDAP::Person do
end
end
describe '.ldap_attributes' do
it 'returns a compact and unique array' do
stub_ldap_config(
options: {
'uid' => nil,
'attributes' => {
'name' => 'cn',
'email' => 'mail',
'username' => %w(uid mail memberof)
}
}
)
config = Gitlab::LDAP::Config.new('ldapmain')
ldap_attributes = described_class.ldap_attributes(config)
expect(ldap_attributes).to match_array(%w(dn uid cn mail memberof))
end
end
describe '.validate_entry' do
it 'raises InvalidEntryError' do
entry['foo'] = 'bar'
expect { described_class.new(entry, 'ldapmain') }
.to raise_error(Gitlab::LDAP::Person::InvalidEntryError)
end
end
describe '#name' do
it 'uses the configured name attribute and handles values as an array' do
name = 'John Doe'
......@@ -72,6 +103,44 @@ describe Gitlab::LDAP::Person do
end
end
describe '#username' do
context 'with default uid username attribute' do
let(:username_attribute) { 'uid' }
it 'returns the proper username value' do
attr_value = 'johndoe'
entry[username_attribute] = attr_value
person = described_class.new(entry, 'ldapmain')
expect(person.username).to eq(attr_value)
end
end
context 'with a different username attribute' do
let(:username_attribute) { 'sAMAccountName' }
it 'returns the proper username value' do
attr_value = 'johndoe'
entry[username_attribute] = attr_value
person = described_class.new(entry, 'ldapmain')
expect(person.username).to eq(attr_value)
end
end
context 'with a non-standard username attribute' do
let(:username_attribute) { 'mail' }
it 'returns the proper username value' do
attr_value = 'john.doe@example.com'
entry[username_attribute] = attr_value
person = described_class.new(entry, 'ldapmain')
expect(person.username).to eq(attr_value)
end
end
end
def assert_generic_test(test_description, got, expected)
test_failure_message = "Failed test description: '#{test_description}'\n\n expected: #{expected}\n got: #{got}"
expect(got).to eq(expected), test_failure_message
......
......@@ -275,6 +275,26 @@ describe Gitlab::OAuth::User do
end
end
context 'and a corresponding LDAP person with a non-default username' do
before do
allow(ldap_user).to receive(:uid) { uid }
allow(ldap_user).to receive(:username) { 'johndoe@example.com' }
allow(ldap_user).to receive(:email) { %w(johndoe@example.com john2@example.com) }
allow(ldap_user).to receive(:dn) { dn }
end
context 'and no account for the LDAP user' do
it 'creates a user favoring the LDAP username and strips email domain' do
allow(Gitlab::LDAP::Person).to receive(:find_by_uid).and_return(ldap_user)
oauth_user.save
expect(gl_user).to be_valid
expect(gl_user.username).to eql 'johndoe'
end
end
end
context "and no corresponding LDAP person" do
before do
allow(Gitlab::LDAP::Person).to receive(:find_by_uid).and_return(nil)
......
require 'rails_helper'
describe DeploymentPlatform do
let(:project) { create(:project) }
describe '#deployment_platform' do
subject { project.deployment_platform }
context 'with no Kubernetes configuration on CI/CD, no Kubernetes Service and a Kubernetes template configured' do
let!(:kubernetes_service) { create(:kubernetes_service, template: true) }
it 'returns a platform kubernetes' do
expect(subject).to be_a_kind_of(Clusters::Platforms::Kubernetes)
end
it 'creates a cluster and a platform kubernetes' do
expect { subject }
.to change { Clusters::Cluster.count }.by(1)
.and change { Clusters::Platforms::Kubernetes.count }.by(1)
end
it 'includes appropriate attributes for Cluster' do
cluster = subject.cluster
expect(cluster.name).to eq('kubernetes-template')
expect(cluster.project).to eq(project)
expect(cluster.provider_type).to eq('user')
expect(cluster.platform_type).to eq('kubernetes')
end
it 'creates a platform kubernetes' do
expect { subject }.to change { Clusters::Platforms::Kubernetes.count }.by(1)
end
it 'copies attributes from Clusters::Platform::Kubernetes template into the new Cluster::Platforms::Kubernetes' do
expect(subject.api_url).to eq(kubernetes_service.api_url)
expect(subject.ca_pem).to eq(kubernetes_service.ca_pem)
expect(subject.token).to eq(kubernetes_service.token)
expect(subject.namespace).to eq(kubernetes_service.namespace)
end
end
context 'with no Kubernetes configuration on CI/CD, no Kubernetes Service and no Kubernetes template configured' do
it { is_expected.to be_nil }
end
context 'when user configured kubernetes from CI/CD > Clusters' do
let!(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) }
let(:platform_kubernetes) { cluster.platform_kubernetes }
it 'returns the Kubernetes platform' do
expect(subject).to eq(platform_kubernetes)
end
end
context 'when user configured kubernetes integration from project services' do
let!(:kubernetes_service) { create(:kubernetes_service, project: project) }
it 'returns the Kubernetes service' do
expect(subject).to eq(kubernetes_service)
end
end
context 'when the cluster creation fails' do
let!(:kubernetes_service) { create(:kubernetes_service, template: true) }
before do
allow_any_instance_of(Clusters::Cluster).to receive(:persisted?).and_return(false)
end
it { is_expected.to be_nil }
end
end
end
......@@ -3137,25 +3137,6 @@ describe Project do
end
end
describe '#deployment_platform' do
subject { project.deployment_platform }
let(:project) { create(:project) }
context 'when user configured kubernetes from Integration > Kubernetes' do
let!(:kubernetes_service) { create(:kubernetes_service, project: project) }
it { is_expected.to eq(kubernetes_service) }
end
context 'when user configured kubernetes from CI/CD > Clusters' do
let!(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) }
let(:platform_kubernetes) { cluster.platform_kubernetes }
it { is_expected.to eq(platform_kubernetes) }
end
end
describe '#write_repository_config' do
set(:project) { create(:project, :repository) }
......
......@@ -272,4 +272,12 @@ describe Service do
expect(service.deprecation_message).to be_nil
end
end
describe '.find_by_template' do
let!(:kubernetes_service) { create(:kubernetes_service, template: true) }
it 'returns service template' do
expect(KubernetesService.find_by_template).to eq(kubernetes_service)
end
end
end
require 'rspec/mocks'
require 'toml'
module TestEnv
extend self
......@@ -147,6 +148,9 @@ module TestEnv
version: Gitlab::GitalyClient.expected_server_version,
task: "gitlab:gitaly:install[#{gitaly_dir}]") do
# Always re-create config, in case it's outdated. This is fast anyway.
Gitlab::SetupHelper.create_gitaly_configuration(gitaly_dir, force: true)
start_gitaly(gitaly_dir)
end
end
......@@ -347,6 +351,9 @@ module TestEnv
end
def component_needs_update?(component_folder, expected_version)
# Allow local overrides of the component for tests during development
return false if Rails.env.test? && File.symlink?(component_folder)
version = File.read(File.join(component_folder, 'VERSION')).strip
# Notice that this will always yield true when using branch versions
......
require 'rake_helper'
describe 'gitlab:git rake tasks' do
before do
Rake.application.rake_require 'tasks/gitlab/git'
storages = { 'default' => { 'path' => Settings.absolute('tmp/tests/default_storage') } }
FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git'))
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
allow_any_instance_of(String).to receive(:color) { |string, _color| string }
stub_warn_user_is_not_gitlab
end
after do
FileUtils.rm_rf(Settings.absolute('tmp/tests/default_storage'))
end
describe 'fsck' do
it 'outputs the integrity check for a repo' do
expect { run_rake_task('gitlab:git:fsck') }.to output(/Performed Checking integrity at .*@hashed\/1\/2\/test.git/).to_stdout
end
it 'errors out about config.lock issues' do
FileUtils.touch(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/config.lock'))
expect { run_rake_task('gitlab:git:fsck') }.to output(/file exists\? ... yes/).to_stdout
end
it 'errors out about ref lock issues' do
FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/refs/heads'))
FileUtils.touch(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/refs/heads/blah.lock'))
expect { run_rake_task('gitlab:git:fsck') }.to output(/Ref lock files exist:/).to_stdout
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment