Automatic merge of gitlab-org/gitlab master

changelogs/unreleased/270093-add-primary-key-to-mrcc-diff-files.yml

+---
+title: Add primary key to merge_request_context_commit_diff_files
+merge_request: 49024
+author:
+type: changed

changelogs/unreleased/jimcser-master-patch-94937.yml

+---
+title: Add link in Access Request API
+merge_request: 48081
+author: jimcser
+type: fixed

config/object_store_settings.rb

@@ -128,6 +128,7 @@ class ObjectStoreSettings
 
       if section['enabled'] && target_config['bucket'].blank?
         missing_bucket_for(store_type)
+        next
       end
 
       # If a storage type such as Pages defines its own connection and does not

db/migrate/20201202155913_add_primary_key_to_merge_request_context_commit_diff_files.rb

+# frozen_string_literal: true
+
+class AddPrimaryKeyToMergeRequestContextCommitDiffFiles < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    execute(<<~SQL)
+      DELETE FROM merge_request_context_commit_diff_files
+      WHERE merge_request_context_commit_id IS NULL;
+
+      DELETE FROM merge_request_context_commit_diff_files df1
+      USING merge_request_context_commit_diff_files df2
+      WHERE df1.ctid < df2.ctid
+        AND df1.merge_request_context_commit_id = df2.merge_request_context_commit_id
+        AND df1.relative_order = df2.relative_order;
+
+      ALTER TABLE merge_request_context_commit_diff_files
+      ADD CONSTRAINT merge_request_context_commit_diff_files_pkey PRIMARY KEY (merge_request_context_commit_id, relative_order);
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      ALTER TABLE merge_request_context_commit_diff_files
+      DROP CONSTRAINT merge_request_context_commit_diff_files_pkey,
+      ALTER COLUMN merge_request_context_commit_id DROP NOT NULL
+    SQL
+  end
+end

db/migrate/20201202161021_remove_redundant_index_on_merge_request_context_commit_diff_files.rb

+# frozen_string_literal: true
+
+class RemoveRedundantIndexOnMergeRequestContextCommitDiffFiles < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    remove_concurrent_index_by_name :merge_request_context_commit_diff_files, 'idx_mr_cc_diff_files_on_mr_cc_id'
+  end
+
+  def down
+    # no-op: this index is not tracked in structure.sql, and is redundant due to idx_mr_cc_diff_files_on_mr_cc_id_and_sha
+  end
+end

db/schema_migrations/20201202155913

+926f54b5756fa9495e71f2340823418b5679195d5720212dddda0d0c6396629e
\ No newline at end of file

db/schema_migrations/20201202161021

+696c1d9f8cc90337549530e129e6abf4429d218f151cefaacacacb6e3f33d1c7
\ No newline at end of file

db/structure.sql

@@ -13695,7 +13695,7 @@ CREATE TABLE merge_request_context_commit_diff_files (
     old_path text NOT NULL,
     diff text,
     "binary" boolean,
-    merge_request_context_commit_id bigint
+    merge_request_context_commit_id bigint NOT NULL
 );
 
 CREATE TABLE merge_request_context_commits (
@@ -19550,6 +19550,9 @@ ALTER TABLE ONLY merge_request_blocks
 ALTER TABLE ONLY merge_request_cleanup_schedules
     ADD CONSTRAINT merge_request_cleanup_schedules_pkey PRIMARY KEY (merge_request_id);
 
+ALTER TABLE ONLY merge_request_context_commit_diff_files
+    ADD CONSTRAINT merge_request_context_commit_diff_files_pkey PRIMARY KEY (merge_request_context_commit_id, relative_order);
+
 ALTER TABLE ONLY merge_request_context_commits
     ADD CONSTRAINT merge_request_context_commits_pkey PRIMARY KEY (id);
 

doc/administration/pages/index.md

@@ -427,6 +427,42 @@ Authority (CA) in the system certificate store.
 
 For Omnibus, this is fixed by [installing a custom CA in Omnibus GitLab](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates).
 
+### Zip serving and cache configuration
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/-/merge_requests/392) in GitLab 13.7.
+
+DANGER: **Warning:**
+These are advanced settings. The recommended default values are set inside GitLab Pages. You should
+change these settings only if absolutely necessary. Use extreme caution.
+
+GitLab Pages can serve content from zip archives through object storage (an
+[issue](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/485) exists for supporting disk storage
+as well). It uses an in-memory cache to increase the performance when serving content from a zip
+archive. You can modify the cache behavior by changing the following configuration flags.
+
+| Setting | Description |
+| ------- | ----------- |
+| `zip_cache_expiration` | The cache expiration interval of zip archives. Must be greater than zero to avoid serving stale content. Default is 60s. |
+| `zip_cache_cleanup` | The interval at which archives are cleaned from memory if they have already expired. Default is 30s. |
+| `zip_cache_refresh` | The time interval in which an archive is extended in memory if accessed before `zip_cache_expiration`. This works together with `zip_cache_expiration` to determine if an archive is extended in memory. See the [example below](#zip-cache-refresh-example) for important details. Default is 30s. |
+| `zip_open_timeout` | The maximum time allowed to open a zip archive. Increase this time for big archives or slow network connections, as doing so may affect the latency of serving Pages. Default is 30s. |
+
+#### Zip cache refresh example
+
+Archives are refreshed in the cache (extending the time they are held in memory) if they're accessed
+before `zip_cache_expiration`, and the time left before expiring is less than or equal to
+`zip_cache_refresh`. For example, if `archive.zip` is accessed at time 0s, it expires in 60s (the
+default for `zip_cache_expiration`). In the example below, if the archive is opened again after 15s
+it is **not** refreshed because the time left for expiry (45s) is greater than `zip_cache_refresh`
+(default 30s). However, if the archive is accessed again after 45s (from the first time it was
+opened) it's refreshed. This extends the time the archive remains in memory from
+`45s + zip_cache_expiration (60s)`, for a total of 105s.
+
+After an archive reaches `zip_cache_expiration`, it's marked as expired and removed on the next
+`zip_cache_cleanup` interval.
+
+![Zip cache configuration](img/zip_cache_configuration.png)
+
 ## Activate verbose logging for daemon
 
 Verbose logging was [introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/2533) in

doc/api/access_requests.md

@@ -7,7 +7,7 @@ type: reference, api
 
 # Group and project access requests API
 
-> Introduced in GitLab 8.11.
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/18583) in GitLab 8.11.
 
 ## Valid access levels
 

doc/development/api_graphql_styleguide.md

@@ -611,7 +611,7 @@ descriptions:
 Example:
 
 ```ruby
-field :id, GraphQL::ID_TYPE, description: 'ID of the Issue'
+field :id, GraphQL::ID_TYPE, description: 'ID of the issue'
 field :confidential, GraphQL::BOOLEAN_TYPE, description: 'Indicates the issue is confidential'
 field :closed_at, Types::TimeType, description: 'Timestamp of when the issue was closed'
 ```
@@ -1190,7 +1190,7 @@ argument :project_path, GraphQL::ID_TYPE,
 
 argument :iid, GraphQL::STRING_TYPE,
          required: true,
-         description: "The iid of the merge request to mutate"
+         description: "The IID of the merge request to mutate"
 
 argument :wip,
          GraphQL::BOOLEAN_TYPE,

doc/development/documentation/styleguide/index.md

@@ -970,7 +970,7 @@ search engine optimization (SEO), use the imperative, where possible.
 
 For guidelines on capitalizing headings, see the section on [capitalization](#capitalization).
 
-NOTE: **Note:**
+NOTE:
 If you change an existing title, be careful. In-page [anchor links](#anchor-links),
 links in the GitLab application, and links from external sites can break.
 
@@ -979,7 +979,7 @@ links in the GitLab application, and links from external sites can break.
 Headings generate anchor links when rendered. `## This is an example` generates
 the anchor `#this-is-an-example`.
 
-NOTE: **Note:**
+NOTE:
 [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39717) in
 GitLab 13.4, [product badges](#product-tier-badges) used in headings aren't
 included in the generated anchor links. For example, when you link to
@@ -1510,100 +1510,72 @@ example:
 
 ## Alert boxes
 
-When you need to call special attention to particular sentences, use the
-following markup to create highlighted alert boxes.
+Use alert boxes to call attention to information.
 
-Alert boxes work for one paragraph only. Multiple paragraphs, lists, and headers
-don't render correctly. For multiple lines, use [blockquotes](#blockquotes)
-instead.
+Alert boxes are generated when the words `NOTE:` or `WARNING:` are followed by a
+line break. For example:
 
-Alert boxes render only on the GitLab documentation site (<https://docs.gitlab.com>).
-In the GitLab product help, alert boxes appear as plain Markdown text.
+```markdown
+NOTE:
+This is something to note.
+```
 
-These alert boxes are used in the GitLab documentation. These aren't strict
-guidelines, but for consistency you should try to use these values:
+To display an alert box for multiple paragraphs, lists, or headers, use
+[blockquotes](#blockquotes) instead.
 
-| Color  | Markup     | Default keyword | Alternative keywords                                                 |
-|--------|------------|-----------------|----------------------------------------------------------------------|
-| Blue   | `NOTE:`    | `**Note:**`     |                                                                      |
-| Yellow | `CAUTION:` | `**Caution:**`  | `**Warning:**`, `**Important:**`                                     |
-| Red    | `DANGER:`  | `**Danger:**`   | `**Warning:**`, `**Important:**`, `**Deprecated:**`, `**Required:**` |
-| Green  | `TIP:`     | `**Tip:**`      |                                                                      |
+Alert boxes render only on the GitLab documentation site (<https://docs.gitlab.com>).
+In the GitLab product help, alert boxes appear as plain text.
 
 ### Note
 
-Notes indicate additional information that's of special use to the reader.
-Notes are most effective when used _sparingly_. Try to avoid them. Too many notes
-can make topics difficult to scan, and create an overly busy page.
+Use notes sparingly. Too many notes can make topics difficult to scan.
 
-Instead of adding a note, try one of these alternatives:
+Instead of adding a note:
 
-- Re-write the sentence as part of the most-relevant paragraph.
-- Put the information into its own standalone paragraph.
-- Put the content under a new subheading that introduces the topic, which makes
-  it more visible.
+- Re-write the sentence as part of a paragraph.
+- Put the information into its own paragraph.
+- Put the content under a new subheading.
 
-If you must use a note, use the following formatting:
+If you must use a note, use this format:
 
 ```markdown
-NOTE: **Note:**
+NOTE:
 This is something to note.
 ```
 
-How it renders on the GitLab documentation site:
+It renders on the GitLab documentation site as:
 
-NOTE: **Note:**
+NOTE:
 This is something to note.
 
-### Tip
-
-```markdown
-TIP: **Tip:**
-This is a tip.
-```
-
-How it renders on the GitLab documentation site:
-
-TIP: **Tip:**
-This is a tip.
-
-### Caution
-
-```markdown
-CAUTION: **Caution:**
-This is something to be cautious about.
-```
-
-How it renders on the GitLab documentation site:
-
-CAUTION: **Caution:**
-This is something to be cautious about.
+### Warning
 
-### Danger
+Use a warning to indicate deprecated features, or to provide a warning about
+procedures that have the potential for data loss. 
 
 ```markdown
-DANGER: **Warning:**
-This is a breaking change, a bug, or something very important to note.
+WARNING:
+This is something to be warned about.
 ```
 
-How it renders on the GitLab documentation site:
+It renders on the GitLab documentation site as:
 
-DANGER: **Warning:**
-This is a breaking change, a bug, or something very important to note.
+WARNING:
+This is something to be warned about.
 
 ## Blockquotes
 
-For highlighting a text inside a blue blockquote, use this format:
+For highlighting a text inside a blockquote, use this format:
 
 ```markdown
 > This is a blockquote.
 ```
 
-which renders on the [GitLab documentation site](https://docs.gitlab.com) as:
+It renders on the GitLab documentation site as:
 
 > This is a blockquote.
 
-If the text spans across multiple lines it's OK to split the line.
+If the text spans multiple lines, you can split them.
 
 For multiple paragraphs, use the symbol `>` before every line:
 
@@ -1616,7 +1588,7 @@ For multiple paragraphs, use the symbol `>` before every line:
 > - Second item in the list
 ```
 
-Which renders to:
+It renders on the GitLab documentation site as:
 
 > This is the first paragraph.
 >
@@ -1752,11 +1724,10 @@ If a feature is deprecated, include a link to a replacement (when available):
 ```
 
 You can also describe the replacement in surrounding text, if available. If the
-deprecation isn't obvious in existing text, you may want to include a warning,
-such as:
+deprecation isn't obvious in existing text, you may want to include a warning:
 
 ```markdown
-DANGER: **Deprecated:**
+WARNING:
 This feature was [deprecated](link-to-issue) in GitLab 12.3 and replaced by
 [Feature name](link-to-feature-documentation).
 ```
@@ -1780,15 +1751,14 @@ voters to agree.
 
 #### End-of-life for features or products
 
-When a feature or product enters the end-of-life process, indicate its
-status prominently. Use the `Danger` [alert](#alert-boxes) with the `**Important**`
-keyword directly below the page header, or the sub-header for the feature or product.
-Link to the deprecation and removal issues, if possible.
+When a feature or product enters its end-of-life, indicate its status by
+creating a [warning alert](#alert-boxes) directly following its relevant header.
+If possible, link to its deprecation and removal issues.
 
 For example:
 
 ```markdown
-DANGER: **Important:**
+WARNING:
 This feature is in its end-of-life process. It is [deprecated](link-to-issue)
 for use in GitLab X.X, and is planned for [removal](link-to-issue) in GitLab X.X.
 ```

doc/user/markdown.md

@@ -477,7 +477,7 @@ In addition to this, links to some objects are also recognized and formatted. So
 
 ### Task lists
 
-> If this is not rendered correctly, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#task-lists).
+If this section is not rendered correctly, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#task-lists).
 
 You can add task lists anywhere Markdown is supported, but you can only "click"
 to toggle the boxes if they are in issues, merge requests, or comments. In other
@@ -780,6 +780,7 @@ But let's throw in a <b>tag</b>.
 
 There are multiple ways to emphasize text in Markdown. You can italicize, bold, strikethrough,
 as well as combine these emphasis styles together.
+Strikethrough is not part of the core Markdown standard, but is part of GFM.
 
 Examples:
 
@@ -801,9 +802,6 @@ Combined emphasis with **asterisks and _underscores_**.
 
 Strikethrough uses two tildes. ~~Scratch this.~~
 
-NOTE: **Note:**
-Strikethrough is not part of the core Markdown standard, but is part of GFM.
-
 #### Multiple underscores in words and mid-word emphasis
 
 If this section is not rendered correctly, [view it in GitLab](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#multiple-underscores-in-words).

qa/Rakefile

@@ -3,6 +3,7 @@ require_relative 'qa/tools/delete_subgroups'
 require_relative 'qa/tools/generate_perf_testdata'
 require_relative 'qa/tools/delete_test_ssh_keys'
 require_relative 'qa/tools/initialize_gitlab_auth'
+require_relative 'qa/tools/delete_projects'
 
 desc "Revokes all personal access tokens"
 task :revoke_personal_access_tokens do
@@ -51,3 +52,8 @@ desc "Deletes test ssh keys a user"
 task :delete_test_ssh_keys, [:title_portion, :delete_before, :dry_run] do |t, args|
   QA::Tools::DeleteTestSSHKeys.new(args).run
 end
+
+desc "Deletes projects directly under the provided group"
+task :delete_projects do
+  QA::Tools::DeleteProjects.new.run
+end

qa/qa/tools/delete_projects.rb

+# frozen_string_literal: true
+
+require_relative '../../qa'
+
+# This script deletes all projects directly under a group specified by ENV['TOP_LEVEL_GROUP_NAME']
+# Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS
+# Optional environment variable: TOP_LEVEL_GROUP_NAME (defaults to 'gitlab-qa-sandbox-group')
+# Run `rake delete_projects`
+
+module QA
+  module Tools
+    class DeleteProjects
+      include Support::Api
+
+      def initialize
+        raise ArgumentError, "Please provide GITLAB_ADDRESS environment variable" unless ENV['GITLAB_ADDRESS']
+        raise ArgumentError, "Please provide GITLAB_QA_ACCESS_TOKEN environment variable" unless ENV['GITLAB_QA_ACCESS_TOKEN']
+
+        @api_client = Runtime::API::Client.new(ENV['GITLAB_ADDRESS'], personal_access_token: ENV['GITLAB_QA_ACCESS_TOKEN'])
+      end
+
+      def run
+        STDOUT.puts 'Running...'
+
+        # Fetch group's id
+        group_id = fetch_group_id
+
+        projects_head_response = head Runtime::API::Request.new(@api_client, "/groups/#{group_id}/projects", per_page: "100").url
+        total_project_pages = projects_head_response.headers[:x_total_pages]
+
+        # Do not delete projects that are less than 4 days old (for debugging purposes)
+        project_ids = fetch_project_ids(group_id, total_project_pages)
+        STDOUT.puts "Number of projects to be deleted: #{project_ids.length}"
+
+        delete_projects(project_ids) unless project_ids.empty?
+        STDOUT.puts "\nDone"
+      end
+
+      private
+
+      def delete_projects(project_ids)
+        STDOUT.puts "Deleting #{project_ids.length} projects..."
+        project_ids.each do |project_id|
+          delete_response = delete Runtime::API::Request.new(@api_client, "/projects/#{project_id}").url
+          dot_or_f = delete_response.code.between?(200, 300) ? "\e[32m.\e[0m" : "\e[31mF\e[0m"
+          print dot_or_f
+        end
+      end
+
+      def fetch_group_id
+        group_name = ENV['TOP_LEVEL_GROUP_NAME'] || 'gitlab-qa-sandbox-group'
+        group_search_response = get Runtime::API::Request.new(@api_client, "/groups/#{group_name}").url
+        JSON.parse(group_search_response.body)["id"]
+      end
+
+      def fetch_project_ids(group_id, total_project_pages)
+        projects_ids = []
+
+        total_project_pages.to_i.times do |page_no|
+          projects_response = get Runtime::API::Request.new(@api_client, "/groups/#{group_id}/projects", page: (page_no + 1).to_s, per_page: "100").url
+          projects_ids.concat(JSON.parse(projects_response.body).select { |project| Date.parse(project["created_at"]) < Date.today - 3 }.map { |project| project["id"] })
+        end
+
+        projects_ids.uniq
+      end
+    end
+  end
+end

spec/config/object_store_settings_spec.rb

@@ -92,6 +92,7 @@ RSpec.describe ObjectStoreSettings do
         config['object_store']['objects']['pages'].delete('bucket')
 
         expect { subject }.not_to raise_error
+        expect(settings.pages['object_store']).to eq(nil)
       end
 
       it 'allows pages to define its own connection' do