Automatic merge of gitlab-org/gitlab-ce master

app/models/namespace.rb

@@ -35,6 +35,8 @@ class Namespace < ApplicationRecord
   belongs_to :parent, class_name: "Namespace"
   has_many :children, class_name: "Namespace", foreign_key: :parent_id
   has_one :chat_team, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+  has_one :root_storage_statistics, class_name: 'Namespace::RootStorageStatistics'
+  has_one :aggregation_schedule, class_name: 'Namespace::AggregationSchedule'
 
   validates :owner, presence: true, unless: ->(n) { n.type == "Group" }
   validates :name,

app/models/namespace/aggregation_schedule.rb

+# frozen_string_literal: true
+
+class Namespace::AggregationSchedule < ApplicationRecord
+  self.primary_key = :namespace_id
+
+  belongs_to :namespace
+end

app/models/namespace/root_storage_statistics.rb

+# frozen_string_literal: true
+
+class Namespace::RootStorageStatistics < ApplicationRecord
+  self.primary_key = :namespace_id
+
+  belongs_to :namespace
+  has_one :route, through: :namespace
+
+  delegate :all_projects, to: :namespace
+end

app/services/pages_domains/obtain_lets_encrypt_certificate_service.rb

@@ -2,6 +2,14 @@
 
 module PagesDomains
   class ObtainLetsEncryptCertificateService
+    # time for processing validation requests for acme challenges
+    # 5-15 seconds is usually enough
+    CHALLENGE_PROCESSING_DELAY = 1.minute.freeze
+
+    # time LetsEncrypt ACME server needs to generate the certificate
+    # no particular SLA, usually takes 10-15 seconds
+    CERTIFICATE_PROCESSING_DELAY = 1.minute.freeze
+
     attr_reader :pages_domain
 
     def initialize(pages_domain)
@@ -14,6 +22,7 @@ module PagesDomains
 
       unless acme_order
         ::PagesDomains::CreateAcmeOrderService.new(pages_domain).execute
+        PagesDomainSslRenewalWorker.perform_in(CHALLENGE_PROCESSING_DELAY, pages_domain.id)
         return
       end
 
@@ -23,6 +32,7 @@ module PagesDomains
       case api_order.status
       when 'ready'
         api_order.request_certificate(private_key: acme_order.private_key, domain: pages_domain.domain)
+        PagesDomainSslRenewalWorker.perform_in(CERTIFICATE_PROCESSING_DELAY, pages_domain.id)
       when 'valid'
         save_certificate(acme_order.private_key, api_order)
         acme_order.destroy!

config/initializers/1_settings.rb

@@ -456,7 +456,7 @@ Settings.cron_jobs['pages_domain_removal_cron_worker']['cron'] ||= '47 0 * * *'
 Settings.cron_jobs['pages_domain_removal_cron_worker']['job_class'] = 'PagesDomainRemovalCronWorker'
 
 Settings.cron_jobs['pages_domain_ssl_renewal_cron_worker'] ||= Settingslogic.new({})
-Settings.cron_jobs['pages_domain_ssl_renewal_cron_worker']['cron'] ||= '*/5 * * * *'
+Settings.cron_jobs['pages_domain_ssl_renewal_cron_worker']['cron'] ||= '*/10 * * * *'
 Settings.cron_jobs['pages_domain_ssl_renewal_cron_worker']['job_class'] = 'PagesDomainSslRenewalCronWorker'
 
 Settings.cron_jobs['issue_due_scheduler_worker'] ||= Settingslogic.new({})

db/migrate/20190531153110_create_namespace_root_storage_statistics.rb

+# frozen_string_literal: true
+
+class CreateNamespaceRootStorageStatistics < ActiveRecord::Migration[5.1]
+  DOWNTIME = false
+
+  def change
+    create_table :namespace_root_storage_statistics, id: false, primary_key: :namespace_id do |t|
+      t.integer :namespace_id, null: false, primary_key: true
+      t.datetime_with_timezone :updated_at, null: false
+
+      t.bigint :repository_size, null: false, default: 0
+      t.bigint :lfs_objects_size, null: false, default: 0
+      t.bigint :wiki_size, null: false, default: 0
+      t.bigint :build_artifacts_size, null: false, default: 0
+      t.bigint :storage_size, null: false, default: 0
+      t.bigint :packages_size, null: false, default: 0
+
+      t.index :namespace_id, unique: true
+      t.foreign_key :namespaces, column: :namespace_id, on_delete: :cascade
+    end
+  end
+end

db/migrate/20190605184422_create_namespace_aggregation_schedules.rb

+# frozen_string_literal: true
+
+class CreateNamespaceAggregationSchedules < ActiveRecord::Migration[5.1]
+  DOWNTIME = false
+
+  def change
+    create_table :namespace_aggregation_schedules, id: false, primary_key: :namespace_id do |t|
+      t.integer :namespace_id, null: false, primary_key: true
+
+      t.index :namespace_id, unique: true
+      t.foreign_key :namespaces, column: :namespace_id, on_delete: :cascade
+    end
+  end
+end

db/schema.rb

@@ -2055,6 +2055,21 @@ ActiveRecord::Schema.define(version: 20190620112608) do
     t.index ["title"], name: "index_milestones_on_title_trigram", using: :gin, opclasses: {"title"=>"gin_trgm_ops"}
   end
 
+  create_table "namespace_aggregation_schedules", primary_key: "namespace_id", id: :integer, default: nil, force: :cascade do |t|
+    t.index ["namespace_id"], name: "index_namespace_aggregation_schedules_on_namespace_id", unique: true, using: :btree
+  end
+
+  create_table "namespace_root_storage_statistics", primary_key: "namespace_id", id: :integer, default: nil, force: :cascade do |t|
+    t.datetime_with_timezone "updated_at", null: false
+    t.bigint "repository_size", default: 0, null: false
+    t.bigint "lfs_objects_size", default: 0, null: false
+    t.bigint "wiki_size", default: 0, null: false
+    t.bigint "build_artifacts_size", default: 0, null: false
+    t.bigint "storage_size", default: 0, null: false
+    t.bigint "packages_size", default: 0, null: false
+    t.index ["namespace_id"], name: "index_namespace_root_storage_statistics_on_namespace_id", unique: true, using: :btree
+  end
+
   create_table "namespace_statistics", id: :serial, force: :cascade do |t|
     t.integer "namespace_id", null: false
     t.integer "shared_runners_seconds", default: 0, null: false
@@ -3757,6 +3772,8 @@ ActiveRecord::Schema.define(version: 20190620112608) do
   add_foreign_key "merge_trains", "users", on_delete: :cascade
   add_foreign_key "milestones", "namespaces", column: "group_id", name: "fk_95650a40d4", on_delete: :cascade
   add_foreign_key "milestones", "projects", name: "fk_9bd0a0c791", on_delete: :cascade
+  add_foreign_key "namespace_aggregation_schedules", "namespaces", on_delete: :cascade
+  add_foreign_key "namespace_root_storage_statistics", "namespaces", on_delete: :cascade
   add_foreign_key "namespace_statistics", "namespaces", on_delete: :cascade
   add_foreign_key "namespaces", "namespaces", column: "custom_project_templates_group_id", name: "fk_e7a0b20a6b", on_delete: :nullify
   add_foreign_key "namespaces", "plans", name: "fk_fdd12e5b80", on_delete: :nullify

spec/factories/namespace/aggregation_schedules.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :namespace_aggregation_schedules, class: Namespace::AggregationSchedule do
+    namespace
+  end
+end

spec/factories/namespace/root_storage_statistics.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :namespace_root_storage_statistics, class: Namespace::RootStorageStatistics do
+    namespace
+  end
+end

spec/factories/namespaces.rb

@@ -19,5 +19,13 @@ FactoryBot.define do
         owner.namespace = namespace
       end
     end
+
+    trait :with_aggregation_schedule do
+      association :aggregation_schedule, factory: :namespace_aggregation_schedules
+    end
+
+    trait :with_root_storage_statistics do
+      association :root_storage_statistics, factory: :namespace_root_storage_statistics
+    end
   end
 end

spec/models/namespace/aggregation_schedule_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Namespace::AggregationSchedule, type: :model do
+  it { is_expected.to belong_to :namespace }
+end

spec/models/namespace/root_storage_statistics_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Namespace::RootStorageStatistics, type: :model do
+  it { is_expected.to belong_to :namespace }
+  it { is_expected.to have_one(:route).through(:namespace) }
+
+  it { is_expected.to delegate_method(:all_projects).to(:namespace) }
+end

spec/models/namespace_spec.rb

@@ -15,6 +15,8 @@ describe Namespace do
     it { is_expected.to have_many :project_statistics }
     it { is_expected.to belong_to :parent }
     it { is_expected.to have_many :children }
+    it { is_expected.to have_one :root_storage_statistics }
+    it { is_expected.to have_one :aggregation_schedule }
   end
 
   describe 'validations' do

spec/services/pages_domains/obtain_lets_encrypt_certificate_service_spec.rb

@@ -34,8 +34,12 @@ describe PagesDomains::ObtainLetsEncryptCertificateService do
   end
 
   context 'when there is no acme order' do
-    it 'creates acme order' do
+    it 'creates acme order and schedules next step' do
       expect_to_create_acme_challenge
+      expect(PagesDomainSslRenewalWorker).to(
+        receive(:perform_in).with(described_class::CHALLENGE_PROCESSING_DELAY, pages_domain.id)
+          .and_return(nil).once
+      )
 
       service.execute
     end
@@ -82,8 +86,12 @@ describe PagesDomains::ObtainLetsEncryptCertificateService do
       stub_lets_encrypt_order(existing_order.url, 'ready')
     end
 
-    it 'request certificate' do
+    it 'request certificate and schedules next step' do
       expect(api_order).to receive(:request_certificate).and_call_original
+      expect(PagesDomainSslRenewalWorker).to(
+        receive(:perform_in).with(described_class::CERTIFICATE_PROCESSING_DELAY, pages_domain.id)
+          .and_return(nil).once
+      )
 
       service.execute
     end

spec/support/sidekiq.rb

@@ -30,6 +30,8 @@ RSpec.configure do |config|
   end
 
   config.after(:each, :sidekiq, :redis) do
-    Sidekiq.redis { |redis| redis.flushdb }
+    Sidekiq.redis do |connection|
+      connection.redis.flushdb
+    end
   end
 end