Commit 3aef2204 authored by Micaël Bergeron's avatar Micaël Bergeron

squash: before rebase

parent 97be319f
...@@ -197,8 +197,8 @@ ActiveRecord::Schema.define(version: 20180612175636) do ...@@ -197,8 +197,8 @@ ActiveRecord::Schema.define(version: 20180612175636) do
t.string "external_authorization_service_url" t.string "external_authorization_service_url"
t.string "external_authorization_service_default_label" t.string "external_authorization_service_default_label"
t.boolean "pages_domain_verification_enabled", default: true, null: false t.boolean "pages_domain_verification_enabled", default: true, null: false
t.float "external_authorization_service_timeout", default: 0.5, null: false
t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false
t.float "external_authorization_service_timeout", default: 0.5
t.text "external_auth_client_cert" t.text "external_auth_client_cert"
t.text "encrypted_external_auth_client_key" t.text "encrypted_external_auth_client_key"
t.string "encrypted_external_auth_client_key_iv" t.string "encrypted_external_auth_client_key_iv"
...@@ -549,12 +549,10 @@ ActiveRecord::Schema.define(version: 20180612175636) do ...@@ -549,12 +549,10 @@ ActiveRecord::Schema.define(version: 20180612175636) do
t.integer "config_source" t.integer "config_source"
t.boolean "protected" t.boolean "protected"
t.integer "failure_reason" t.integer "failure_reason"
t.integer "iid"
end end
add_index "ci_pipelines", ["auto_canceled_by_id"], name: "index_ci_pipelines_on_auto_canceled_by_id", using: :btree add_index "ci_pipelines", ["auto_canceled_by_id"], name: "index_ci_pipelines_on_auto_canceled_by_id", using: :btree
add_index "ci_pipelines", ["pipeline_schedule_id"], name: "index_ci_pipelines_on_pipeline_schedule_id", using: :btree add_index "ci_pipelines", ["pipeline_schedule_id"], name: "index_ci_pipelines_on_pipeline_schedule_id", using: :btree
add_index "ci_pipelines", ["project_id", "iid"], name: "index_ci_pipelines_on_project_id_and_iid", unique: true, where: "(iid IS NOT NULL)", using: :btree
add_index "ci_pipelines", ["project_id", "ref", "status", "id"], name: "index_ci_pipelines_on_project_id_and_ref_and_status_and_id", using: :btree add_index "ci_pipelines", ["project_id", "ref", "status", "id"], name: "index_ci_pipelines_on_project_id_and_ref_and_status_and_id", using: :btree
add_index "ci_pipelines", ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree add_index "ci_pipelines", ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree
add_index "ci_pipelines", ["project_id"], name: "index_ci_pipelines_on_project_id", using: :btree add_index "ci_pipelines", ["project_id"], name: "index_ci_pipelines_on_project_id", using: :btree
...@@ -1632,7 +1630,6 @@ ActiveRecord::Schema.define(version: 20180612175636) do ...@@ -1632,7 +1630,6 @@ ActiveRecord::Schema.define(version: 20180612175636) do
t.text "title_html" t.text "title_html"
t.text "description_html" t.text "description_html"
t.integer "time_estimate" t.integer "time_estimate"
t.boolean "squash", default: false, null: false
t.integer "cached_markdown_version" t.integer "cached_markdown_version"
t.datetime "last_edited_at" t.datetime "last_edited_at"
t.integer "last_edited_by_id" t.integer "last_edited_by_id"
...@@ -2022,9 +2019,9 @@ ActiveRecord::Schema.define(version: 20180612175636) do ...@@ -2022,9 +2019,9 @@ ActiveRecord::Schema.define(version: 20180612175636) do
t.datetime "next_execution_timestamp" t.datetime "next_execution_timestamp"
t.string "status" t.string "status"
t.string "jid" t.string "jid"
t.text "last_error"
t.datetime_with_timezone "last_update_at" t.datetime_with_timezone "last_update_at"
t.datetime_with_timezone "last_successful_update_at" t.datetime_with_timezone "last_successful_update_at"
t.text "last_error"
end end
add_index "project_mirror_data", ["jid"], name: "index_project_mirror_data_on_jid", using: :btree add_index "project_mirror_data", ["jid"], name: "index_project_mirror_data_on_jid", using: :btree
...@@ -2279,7 +2276,6 @@ ActiveRecord::Schema.define(version: 20180612175636) do ...@@ -2279,7 +2276,6 @@ ActiveRecord::Schema.define(version: 20180612175636) do
end end
add_index "redirect_routes", ["path"], name: "index_redirect_routes_on_path", unique: true, using: :btree add_index "redirect_routes", ["path"], name: "index_redirect_routes_on_path", unique: true, using: :btree
add_index "redirect_routes", ["path"], name: "index_redirect_routes_on_path_text_pattern_ops", using: :btree, opclasses: {"path"=>"varchar_pattern_ops"}
add_index "redirect_routes", ["source_type", "source_id"], name: "index_redirect_routes_on_source_type_and_source_id", using: :btree add_index "redirect_routes", ["source_type", "source_id"], name: "index_redirect_routes_on_source_type_and_source_id", using: :btree
create_table "releases", force: :cascade do |t| create_table "releases", force: :cascade do |t|
...@@ -2677,13 +2673,13 @@ ActiveRecord::Schema.define(version: 20180612175636) do ...@@ -2677,13 +2673,13 @@ ActiveRecord::Schema.define(version: 20180612175636) do
t.boolean "notified_of_own_activity" t.boolean "notified_of_own_activity"
t.boolean "support_bot" t.boolean "support_bot"
t.string "preferred_language" t.string "preferred_language"
t.string "rss_token"
t.boolean "email_opted_in" t.boolean "email_opted_in"
t.string "email_opted_in_ip" t.string "email_opted_in_ip"
t.integer "email_opted_in_source_id" t.integer "email_opted_in_source_id"
t.datetime "email_opted_in_at" t.datetime "email_opted_in_at"
t.integer "theme_id", limit: 2 t.integer "theme_id", limit: 2
t.integer "accepted_term_id" t.integer "accepted_term_id"
t.string "feed_token"
end end
add_index "users", ["admin"], name: "index_users_on_admin", using: :btree add_index "users", ["admin"], name: "index_users_on_admin", using: :btree
...@@ -2691,14 +2687,13 @@ ActiveRecord::Schema.define(version: 20180612175636) do ...@@ -2691,14 +2687,13 @@ ActiveRecord::Schema.define(version: 20180612175636) do
add_index "users", ["created_at"], name: "index_users_on_created_at", using: :btree add_index "users", ["created_at"], name: "index_users_on_created_at", using: :btree
add_index "users", ["email"], name: "index_users_on_email", unique: true, using: :btree add_index "users", ["email"], name: "index_users_on_email", unique: true, using: :btree
add_index "users", ["email"], name: "index_users_on_email_trigram", using: :gin, opclasses: {"email"=>"gin_trgm_ops"} add_index "users", ["email"], name: "index_users_on_email_trigram", using: :gin, opclasses: {"email"=>"gin_trgm_ops"}
add_index "users", ["feed_token"], name: "index_users_on_feed_token", using: :btree
add_index "users", ["ghost"], name: "index_users_on_ghost", using: :btree add_index "users", ["ghost"], name: "index_users_on_ghost", using: :btree
add_index "users", ["incoming_email_token"], name: "index_users_on_incoming_email_token", using: :btree add_index "users", ["incoming_email_token"], name: "index_users_on_incoming_email_token", using: :btree
add_index "users", ["name"], name: "index_users_on_name", using: :btree add_index "users", ["name"], name: "index_users_on_name", using: :btree
add_index "users", ["name"], name: "index_users_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"} add_index "users", ["name"], name: "index_users_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"}
add_index "users", ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true, using: :btree add_index "users", ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true, using: :btree
add_index "users", ["rss_token"], name: "index_users_on_rss_token", using: :btree
add_index "users", ["state"], name: "index_users_on_state", using: :btree add_index "users", ["state"], name: "index_users_on_state", using: :btree
add_index "users", ["state"], name: "index_users_on_state_and_internal_attrs", where: "((ghost <> true) AND (support_bot <> true))", using: :btree
add_index "users", ["support_bot"], name: "index_users_on_support_bot", using: :btree add_index "users", ["support_bot"], name: "index_users_on_support_bot", using: :btree
add_index "users", ["username"], name: "index_users_on_username", using: :btree add_index "users", ["username"], name: "index_users_on_username", using: :btree
add_index "users", ["username"], name: "index_users_on_username_trigram", using: :gin, opclasses: {"username"=>"gin_trgm_ops"} add_index "users", ["username"], name: "index_users_on_username_trigram", using: :gin, opclasses: {"username"=>"gin_trgm_ops"}
...@@ -2828,8 +2823,6 @@ ActiveRecord::Schema.define(version: 20180612175636) do ...@@ -2828,8 +2823,6 @@ ActiveRecord::Schema.define(version: 20180612175636) do
add_foreign_key "clusters", "users", on_delete: :nullify add_foreign_key "clusters", "users", on_delete: :nullify
add_foreign_key "clusters_applications_helm", "clusters", on_delete: :cascade add_foreign_key "clusters_applications_helm", "clusters", on_delete: :cascade
add_foreign_key "clusters_applications_ingress", "clusters", name: "fk_753a7b41c1", on_delete: :cascade add_foreign_key "clusters_applications_ingress", "clusters", name: "fk_753a7b41c1", on_delete: :cascade
add_foreign_key "clusters_applications_jupyter", "clusters", on_delete: :cascade
add_foreign_key "clusters_applications_jupyter", "oauth_applications", on_delete: :nullify
add_foreign_key "clusters_applications_prometheus", "clusters", name: "fk_557e773639", on_delete: :cascade add_foreign_key "clusters_applications_prometheus", "clusters", name: "fk_557e773639", on_delete: :cascade
add_foreign_key "clusters_applications_runners", "ci_runners", column: "runner_id", name: "fk_02de2ded36", on_delete: :nullify add_foreign_key "clusters_applications_runners", "ci_runners", column: "runner_id", name: "fk_02de2ded36", on_delete: :nullify
add_foreign_key "clusters_applications_runners", "clusters", on_delete: :cascade add_foreign_key "clusters_applications_runners", "clusters", on_delete: :cascade
......
# See http://doc.gitlab.com/ce/development/migration_style_guide.html # See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab. # for more information on how to write migrations for GitLab.
class AddEltDumpEnabledToApplicationSettings < ActiveRecord::Migration class AddPseudonymizerEnabledToApplicationSettings < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers include Gitlab::Database::MigrationHelpers
# Set this constant to true if this migration requires downtime. # Set this constant to true if this migration requires downtime.
......
diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake
new file mode 100644
index 00000000000..bfca4bfb3f7
--- /dev/null
+++ b/lib/tasks/gitlab/artifacts/migrate.rake
@@ -0,0 +1,25 @@
+require 'logger'
+require 'resolv-replace'
+
+desc "GitLab | Migrate files for artifacts to comply with new storage format"
+namespace :gitlab do
+ namespace :artifacts do
+ task migrate: :environment do
+ logger = Logger.new(STDOUT)
+ logger.info('Starting transfer of artifacts')
+
+ Ci::Build.joins(:project)
+ .with_artifacts_stored_locally
+ .find_each(batch_size: 10) do |build|
+ begin
+ build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE)
+ build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE)
+
+ logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage")
+ rescue => e
+ logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
+ end
+ end
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/exclusive_lease.rake b/lib/tasks/gitlab/exclusive_lease.rake
new file mode 100644
index 00000000000..83722bf6d94
--- /dev/null
+++ b/lib/tasks/gitlab/exclusive_lease.rake
@@ -0,0 +1,9 @@
+namespace :gitlab do
+ namespace :exclusive_lease do
+ desc 'GitLab | Clear existing exclusive leases for specified scope (default: *)'
+ task :clear, [:scope] => [:environment] do |_, args|
+ args[:scope].nil? ? Gitlab::ExclusiveLease.reset_all! : Gitlab::ExclusiveLease.reset_all!(args[:scope])
+ puts 'All exclusive lease entries were removed.'
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake
new file mode 100644
index 00000000000..a45e5ca91e0
--- /dev/null
+++ b/lib/tasks/gitlab/lfs/migrate.rake
@@ -0,0 +1,22 @@
+require 'logger'
+
+desc "GitLab | Migrate LFS objects to remote storage"
+namespace :gitlab do
+ namespace :lfs do
+ task migrate: :environment do
+ logger = Logger.new(STDOUT)
+ logger.info('Starting transfer of LFS files to object storage')
+
+ LfsObject.with_files_stored_locally
+ .find_each(batch_size: 10) do |lfs_object|
+ begin
+ lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
+
+ logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage")
+ rescue => e
+ logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
+ end
+ end
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index 1ce2eedb89c..56f3a916c87 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -69,7 +69,7 @@ namespace :gitlab do
if File.exist?(path_to_repo)
print '-'
else
- if Gitlab::Shell.new.add_repository(project.repository_storage,
+ if Gitlab::Shell.new.create_repository(project.repository_storage,
project.disk_path)
print '.'
else
diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake
new file mode 100644
index 00000000000..c26c3ccb3be
--- /dev/null
+++ b/lib/tasks/gitlab/uploads/migrate.rake
@@ -0,0 +1,33 @@
+namespace :gitlab do
+ namespace :uploads do
+ desc 'GitLab | Uploads | Migrate the uploaded files to object storage'
+ task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args|
+ batch_size = ENV.fetch('BATCH', 200).to_i
+ @to_store = ObjectStorage::Store::REMOTE
+ @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym
+ @uploader_class = args.uploader_class.constantize
+ @model_class = args.model_class.constantize
+
+ uploads.each_batch(of: batch_size, &method(:enqueue_batch)) # rubocop: disable Cop/InBatches
+ end
+
+ def enqueue_batch(batch, index)
+ job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch,
+ @mounted_as,
+ @to_store)
+ puts "Enqueued job ##{index}: #{job}"
+ rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e
+ # continue for the next batch
+ puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red)
+ end
+
+ def uploads
+ Upload.class_eval { include EachBatch } unless Upload < EachBatch
+
+ Upload
+ .where.not(store: @to_store)
+ .where(uploader: @uploader_class.to_s,
+ model_type: @model_class.base_class.sti_name)
+ end
+ end
+end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment