Add latest changes from gitlab-org/gitlab@master

app/models/repository.rb

@@ -1127,8 +1127,8 @@ class Repository
 
   private
 
-  # TODO Generice finder, later split this on finders by Ref or Oid
-  # https://gitlab.com/gitlab-org/gitlab-foss/issues/39239
+  # TODO Genericize finder, later split this on finders by Ref or Oid
+  # https://gitlab.com/gitlab-org/gitlab/issues/19877
   def find_commit(oid_or_ref)
     commit = if oid_or_ref.is_a?(Gitlab::Git::Commit)
                oid_or_ref

changelogs/unreleased/22166-fix-sast-template.yml

+---
+title: Check both SAST_DISABLE and SAST_DISABLE_DIND when executing SAST job template
+merge_request: 22166
+author:
+type: fixed

changelogs/unreleased/39951-fix-dependency-scanning-regex.yml

+---
+title: Fix regex matching for gemnasium dependency scanning jobs
+merge_request: 22025
+author: Maximilian Stendler
+type: fix

changelogs/unreleased/remove-ancestor-flag.yml

+---
+title: Remove N+1 query issue when checking group root ancestor.
+merge_request:
+author:
+type: performance

db/migrate/20191126134210_rename_packages_package_tags.rb

+# frozen_string_literal: true
+
+class RenamePackagesPackageTags < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+
+  def change
+    rename_table(:packages_package_tags, :packages_tags)
+  end
+end

db/migrate/20200106085831_add_timestamps_to_packages_tags.rb

+# frozen_string_literal: true
+
+class AddTimestampsToPackagesTags < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  # We disable these cops here because adding this column is safe. The table does not
+  # have any data in it.
+  # rubocop: disable Migration/AddIndex
+  def up
+    add_timestamps_with_timezone(:packages_tags, null: false)
+    add_index(:packages_tags, [:package_id, :updated_at], order: { updated_at: :desc })
+  end
+
+  # We disable these cops here because adding this column is safe. The table does not
+  # have any data in it.
+  # rubocop: disable Migration/RemoveIndex
+  def down
+    remove_index(:packages_tags, [:package_id, :updated_at])
+    remove_timestamps(:packages_tags)
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_01_06_071113) do
+ActiveRecord::Schema.define(version: 2020_01_06_085831) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
@@ -2938,12 +2938,6 @@ ActiveRecord::Schema.define(version: 2020_01_06_071113) do
     t.index ["package_id", "file_name"], name: "index_packages_package_files_on_package_id_and_file_name"
   end
 
-  create_table "packages_package_tags", force: :cascade do |t|
-    t.integer "package_id", null: false
-    t.string "name", limit: 255, null: false
-    t.index ["package_id"], name: "index_packages_package_tags_on_package_id"
-  end
-
   create_table "packages_packages", force: :cascade do |t|
     t.integer "project_id", null: false
     t.datetime_with_timezone "created_at", null: false
@@ -2956,6 +2950,15 @@ ActiveRecord::Schema.define(version: 2020_01_06_071113) do
     t.index ["project_id"], name: "index_packages_packages_on_project_id"
   end
 
+  create_table "packages_tags", force: :cascade do |t|
+    t.integer "package_id", null: false
+    t.string "name", limit: 255, null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.index ["package_id", "updated_at"], name: "index_packages_tags_on_package_id_and_updated_at", order: { updated_at: :desc }
+    t.index ["package_id"], name: "index_packages_tags_on_package_id"
+  end
+
   create_table "pages_domain_acme_orders", force: :cascade do |t|
     t.integer "pages_domain_id", null: false
     t.datetime_with_timezone "expires_at", null: false
@@ -4705,8 +4708,8 @@ ActiveRecord::Schema.define(version: 2020_01_06_071113) do
   add_foreign_key "packages_dependency_links", "packages_packages", column: "package_id", on_delete: :cascade
   add_foreign_key "packages_maven_metadata", "packages_packages", column: "package_id", name: "fk_be88aed360", on_delete: :cascade
   add_foreign_key "packages_package_files", "packages_packages", column: "package_id", name: "fk_86f0f182f8", on_delete: :cascade
-  add_foreign_key "packages_package_tags", "packages_packages", column: "package_id", on_delete: :cascade
   add_foreign_key "packages_packages", "projects", on_delete: :cascade
+  add_foreign_key "packages_tags", "packages_packages", column: "package_id", on_delete: :cascade
   add_foreign_key "pages_domain_acme_orders", "pages_domains", on_delete: :cascade
   add_foreign_key "pages_domains", "projects", name: "fk_ea2f6dfc6f", on_delete: :cascade
   add_foreign_key "path_locks", "projects", name: "fk_5265c98f24", on_delete: :cascade

doc/administration/gitaly/index.md

@@ -164,11 +164,21 @@ Git operations in GitLab will result in an API error.
    postgresql['enable'] = false
    redis['enable'] = false
    nginx['enable'] = false
-   prometheus['enable'] = false
    unicorn['enable'] = false
    sidekiq['enable'] = false
    gitlab_workhorse['enable'] = false
 
+   # If you don't want to run monitoring services uncomment the following (not recommended)
+   # alertmanager['enable'] = false
+   # gitlab_exporter['enable'] = false
+   # grafana['enable'] = false
+   # node_exporter['enable'] = false
+   # prometheus['enable'] = false
+
+   # Enable prometheus monitoring - comment out if you disable monitoring services above.
+   # This makes Prometheus listen on all interfaces. You must use firewalls to restrict access to this address/port.
+   prometheus['listen_address'] = '0.0.0.0:9090'
+
    # Prevent database connections during 'gitlab-ctl reconfigure'
    gitlab_rails['rake_cache_clear'] = false
    gitlab_rails['auto_migrate'] = false
@@ -191,6 +201,11 @@ Git operations in GitLab will result in an API error.
 
 1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server:
 
+   <!--
+   updates to following example must also be made at
+   https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab
+   -->
+
    On `gitaly1.internal`:
 
    ```
@@ -561,14 +576,14 @@ a few things that you need to do:
 
 1. Make sure the [`git` user home directory](https://docs.gitlab.com/omnibus/settings/configuration.html#moving-the-home-directory-for-a-user) is on local disk.
 1. Configure [database lookup of SSH keys](../operations/fast_ssh_key_lookup.md)
-   to eliminate the need for a shared authorized_keys file.
+   to eliminate the need for a shared `authorized_keys` file.
 1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage)
    including [incremental logging](../job_logs.md#new-incremental-logging-architecture).
 1. Configure [object storage for LFS objects](../lfs/lfs_administration.md#storing-lfs-objects-in-remote-object-storage).
 1. Configure [object storage for uploads](../uploads.md#using-object-storage-core-only).
-1. Configure [object storage for Merge Request Diffs](../merge_request_diffs.md#using-object-storage).
-1. Configure [object storage for Packages](../packages/index.md#using-object-storage) (Optional Feature).
-1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (Optional Feature).
+1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage).
+1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature).
+1. Configure [object storage for dependency proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature).
 
 NOTE: **Note:**
 One current feature of GitLab that still requires a shared directory (NFS) is
@@ -862,7 +877,7 @@ remote: GitLab: 401 Unauthorized
 You will need to sync your `gitlab-secrets.json` file with your GitLab
 app nodes.
 
-### Client side GRPC logs
+### Client side gRPC logs
 
 Gitaly uses the [gRPC](https://grpc.io/) RPC framework. The Ruby gRPC
 client has its own log file which may contain useful information when

doc/administration/packages/container_registry.md

@@ -632,6 +632,29 @@ mounting the docker-daemon and setting `privileged = false` in the Runner's
 
 Additional information about this: [issue 18239](https://gitlab.com/gitlab-org/gitlab-foss/issues/18239).
 
+### `unauthorized: authentication required` when pushing large images
+
+Example error:
+
+```shell
+docker push gitlab.example.com/myproject/docs:latest
+The push refers to a repository [gitlab.example.com/myproject/docs]
+630816f32edb: Preparing
+530d5553aec8: Preparing
+...
+4b0bab9ff599: Waiting
+d1c800db26c7: Waiting
+42755cf4ee95: Waiting
+unauthorized: authentication required
+```
+
+GitLab has a default token expiration of 5 minutes for the registry. When pushing
+larger images, or images that take longer than 5 minutes to push, users may
+encounter this error.
+
+Administrators can increase the token duration in **Admin area > Settings >
+Container Registry > Authorization token duration (minutes)**.
+
 ### AWS S3 with the GitLab registry error when pushing large images
 
 When using AWS S3 with the GitLab registry, an error may occur when pushing

doc/development/code_review.md

@@ -274,6 +274,12 @@ time frame, let the author know as soon as possible and try to help them find
 another reviewer or maintainer who will be able to, so that they can be unblocked
 and get on with their work quickly.
 
+If you think you are at capacity and are unable to accept any more reviews until
+some have been completed, communicate this through your GitLab status by setting
+the `:red_circle:` emoji and mentioning that you are at capacity in the status
+text. This will guide contributors to pick a different reviewer, helping us to
+meet the SLO.
+
 Of course, if you are out of office and have
 [communicated](https://about.gitlab.com/handbook/paid-time-off/#communicating-your-time-off)
 this through your GitLab.com Status, authors are expected to realize this and

doc/integration/elasticsearch.md

@@ -36,6 +36,9 @@ it yourself or by using the
 service. Running Elasticsearch on the same server as GitLab is not recommended
 and it will likely cause performance degradation on the GitLab installation.
 
+NOTE: **Note:**
+**For a single node Elasticsearch cluster the functional cluster health status will be yellow** (will never be green) because the primary shard is allocated but replicas can not be as there is no other node to which Elasticsearch can assign a replica.
+
 Once the data is added to the database or repository and [Elasticsearch is
 enabled in the admin area](#enabling-elasticsearch) the search index will be
 updated automatically.
@@ -592,6 +595,23 @@ Here are some common pitfalls and how to overcome them:
   for this setting ("Maximum Size of HTTP Request Payloads"), based on the size of
   the underlying instance.
   
+- **My single node Elasticsearch cluster status never goes from `yellow` to `green` even though everything seems to be running properly**
+
+  **For a single node Elasticsearch cluster the functional cluster health status will be yellow** (will never be green) because the primary shard is allocated but replicas can not be as there is no other node to which Elasticsearch can assign a replica. This also applies if you are using using the
+[Amazon Elasticsearch](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-handling-errors.html#aes-handling-errors-yellow-cluster-status) service.
+
+  CAUTION: **Warning**: Setting the number of replicas to `0` is not something that we recommend (this is not allowed in the GitLab Elasticsearch Integration menu). If you are planning to add more Elasticsearch nodes (for a total of more than 1 Elasticsearch) the number of replicas will need to be set to an integer value larger than `0`. Failure to do so will result in lack of redundancy (losing one node will corupt the index).
+
+  If you have a **hard requirement to have a green status for your single node Elasticsearch cluster**, please make sure you understand the risks outlined in the previous paragraph and then simply run the following query to set the number of replicas to `0`(the cluster will no longer try to create any shard replicas):
+
+  ```bash
+  curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' --data '{
+  "index" : {
+     "number_of_replicas" : 0
+    }
+  }'
+  ```
+
 ### Reverting to basic search
 
 Sometimes there may be issues with your Elasticsearch index data and as such

doc/security/webhooks.md

@@ -35,8 +35,8 @@ to endpoints like `http://localhost:123/some-resource/delete`.
 To prevent this type of exploitation from happening, starting with GitLab 10.6,
 all Webhook requests to the current GitLab instance server address and/or in a
 private network will be forbidden by default. That means that all requests made
-to 127.0.0.1, ::1 and 0.0.0.0, as well as IPv4 10.0.0.0/8, 172.16.0.0/12,
-192.168.0.0/16 and IPv6 site-local (ffc0::/10) addresses won't be allowed.
+to `127.0.0.1`, `::1` and `0.0.0.0`, as well as IPv4 `10.0.0.0/8`, `172.16.0.0/12`,
+`192.168.0.0/16` and IPv6 site-local (`ffc0::/10`) addresses won't be allowed.
 
 This behavior can be overridden by enabling the option *"Allow requests to the
 local network from web hooks and services"* in the *"Outbound requests"* section

doc/user/packages/dependency_proxy/index.md

@@ -12,7 +12,7 @@ receiving a request and returning the upstream image from a registry, acting
 as a pull-through cache.
 
 The dependency proxy is available in the group level. To access it, navigate to
-a group's **Overview > Dependency Proxy**.
+a group's **Packages > Dependency Proxy**.
 
 ![Dependency Proxy group page](img/group_dependency_proxy.png)
 
@@ -33,7 +33,7 @@ The following dependency proxies are supported.
 With the Docker dependency proxy, you can use GitLab as a source for a Docker image.
 To get a Docker image into the dependency proxy:
 
-1. Find the proxy URL on your group's page under **Overview > Dependency Proxy**,
+1. Find the proxy URL on your group's page under **Packages > Dependency Proxy**,
    for example `gitlab.com/groupname/dependency_proxy/containers`.
 1. Trigger GitLab to pull the Docker image you want (e.g., `alpine:latest` or
    `linuxserver/nextcloud:latest`) and store it in the proxy storage by using

doc/user/packages/npm_registry/index.md

@@ -134,8 +134,8 @@ Add a corresponding section to your `.npmrc` file:
 
 ```ini
 @foo:registry=https://gitlab.com/api/v4/packages/npm/
-//gitlab.com/api/v4/packages/npm/:_authToken=${env.CI_JOB_TOKEN}
-//gitlab.com/api/v4/projects/{env.CI_PROJECT_ID>/packages/npm/:_authToken=${env.CI_JOB_TOKEN}
+//gitlab.com/api/v4/packages/npm/:_authToken=${CI_JOB_TOKEN}
+//gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=${CI_JOB_TOKEN}
 ```
 
 ## Uploading packages
@@ -242,3 +242,27 @@ Starting from GitLab 12.6, new packages published to the GitLab NPM Registry exp
   - bundleDependencies
   - peerDependencies
   - deprecated
+
+## NPM distribution tags
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9425) in GitLab Premium 12.7.
+
+Dist Tags for newly published packages are supported, and they follow NPM's convention where they are optional, and each tag can only be assigned to 1 package at
+You can add [distribution tags](https://docs.npmjs.com/cli/dist-tag) for newly
+published packages. They follow NPM's convention where they are optional, and
+each tag can only be assigned to one package at a time. The latest tag is added
+by default when a package is published without a tag. The same goes to installing
+a package without specifying the tag or version.
+
+Examples of the supported `dist-tag` commands and using tags in general:
+
+```sh
+npm publish @scope/package --tag               # Publish new package with new tag
+npm dist-tag add @scope/package@version my-tag # Add a tag to an existing package
+npm dist-tag ls @scope/package                 # List all tags under the package
+npm dist-tag rm @scope/package@version my-tag  # Delete a tag from the package
+npm install @scope/package@my-tag              # Install a specific tag
+```
+
+CAUTION: **Warning:**
+Due to a bug in NPM 6.9.0, deleting dist tags fails. Make sure your NPM version is greater than 6.9.1.

lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml

@@ -88,7 +88,7 @@ gemnasium-dependency_scanning:
   only:
     variables:
       - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ &&
-        $DS_DEFAULT_ANALYZERS =~ /gemnasium/ &&
+        $DS_DEFAULT_ANALYZERS =~ /gemnasium([^-]|$)/ &&
         $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby|javascript|php/
 
 gemnasium-maven-dependency_scanning:

lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml

@@ -55,6 +55,7 @@ sast:
   services: []
   except:
     variables:
+      - $SAST_DISABLED
       - $SAST_DISABLE_DIND == 'false'
   script:
     - /analyzer run