Automatic merge of gitlab-org/gitlab master

.gitlab/ci/test-metadata.gitlab-ci.yml

 .tests-metadata-state:
+  variables:
+    TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache"
   before_script:
     - source scripts/utils.sh
   artifacts:

app/assets/javascripts/merge_conflicts/components/diff_file_editor.js

 /* eslint-disable no-param-reassign */
-/* global ace */
 
 import Vue from 'vue';
+import { debounce } from 'lodash';
 import axios from '~/lib/utils/axios_utils';
 import { deprecatedCreateFlash as flash } from '~/flash';
 import { __ } from '~/locale';
-import getModeByFileExtension from '~/lib/utils/ace_utils';
 
 (global => {
   global.mergeConflicts = global.mergeConflicts || {};
@@ -28,7 +27,6 @@ import getModeByFileExtension from '~/lib/utils/ace_utils';
     data() {
       return {
         saved: false,
-        loading: false,
         fileLoaded: false,
         originalContent: '',
       };
@@ -37,7 +35,6 @@ import getModeByFileExtension from '~/lib/utils/ace_utils';
       classObject() {
         return {
           saved: this.saved,
-          'is-loading': this.loading,
         };
       },
     },
@@ -45,7 +42,7 @@ import getModeByFileExtension from '~/lib/utils/ace_utils';
       'file.showEditor': function showEditorWatcher(val) {
         this.resetEditorContent();
 
-        if (!val || this.fileLoaded || this.loading) {
+        if (!val || this.fileLoaded) {
           return;
         }
 
@@ -59,30 +56,25 @@ import getModeByFileExtension from '~/lib/utils/ace_utils';
     },
     methods: {
       loadEditor() {
-        this.loading = true;
+        const EditorPromise = import(/* webpackChunkName: 'EditorLite' */ '~/editor/editor_lite');
+        const DataPromise = axios.get(this.file.content_path);
 
-        axios
-          .get(this.file.content_path)
-          .then(({ data }) => {
-            const content = this.$el.querySelector('pre');
-            const fileContent = document.createTextNode(data.content);
+        Promise.all([EditorPromise, DataPromise])
+          .then(([{ default: EditorLite }, { data: { content, new_path: path } }]) => {
+            const contentEl = this.$el.querySelector('.editor');
 
-            content.textContent = fileContent.textContent;
-
-            this.originalContent = data.content;
+            this.originalContent = content;
             this.fileLoaded = true;
-            this.editor = ace.edit(content);
-            this.editor.$blockScrolling = Infinity; // Turn off annoying warning
-            this.editor.getSession().setMode(getModeByFileExtension(data.new_path));
-            this.editor.on('change', () => {
-              this.saveDiffResolution();
+
+            this.editor = new EditorLite().createInstance({
+              el: contentEl,
+              blobPath: path,
+              blobContent: content,
             });
-            this.saveDiffResolution();
-            this.loading = false;
+            this.editor.onDidChangeModelContent(debounce(this.saveDiffResolution.bind(this), 250));
           })
           .catch(() => {
             flash(__('An error occurred while loading the file'));
-            this.loading = false;
           });
       },
       saveDiffResolution() {
@@ -95,7 +87,7 @@ import getModeByFileExtension from '~/lib/utils/ace_utils';
       },
       resetEditorContent() {
         if (this.fileLoaded) {
-          this.editor.setValue(this.originalContent, -1);
+          this.editor.setValue(this.originalContent);
         }
       },
       cancelDiscardConfirmation(file) {

app/assets/stylesheets/pages/editor.scss

@@ -8,18 +8,18 @@
     background: $gray-normal;
   }
 
-  #editor {
-    border: 0;
-    border-radius: 0;
+  #editor,
+  .editor {
+    @include gl-border-0;
+    @include gl-m-0;
+    @include gl-p-0;
+    @include gl-relative;
+    @include gl-w-full;
     height: 500px;
-    margin: 0;
-    padding: 0;
-    position: relative;
-    width: 100%;
 
     .editor-loading-content {
-      height: 100%;
-      border: 0;
+      @include gl-h-full;
+      @include gl-border-0;
     }
   }
 

app/views/projects/merge_requests/conflicts/components/_diff_file_editor.html.haml

@@ -7,7 +7,4 @@
           %button.btn.btn-sm.btn-close{ "@click" => "acceptDiscardConfirmation(file)" } Discard changes
           %button.btn.btn-sm{ "@click" => "cancelDiscardConfirmation(file)" } Cancel
     .editor-wrap{ ":class" => "classObject" }
-      .loading
-        .spinner.spinner-md
-      .editor
-        %pre{ "style" => "height: 350px" }
+      .editor{ "style" => "height: 350px", data: { 'editor-loading': true } }

changelogs/unreleased/273016-replace-ace-merge-conflicts.yml

+---
+title: Replace ACE with Editor Lite
+merge_request: 46250
+author:
+type: changed

doc/administration/operations/moving_repositories.md

@@ -5,10 +5,35 @@ info: To determine the technical writer assigned to the Stage/Group associated w
 type: reference
 ---
 
-# Moving repositories managed by GitLab
+# Moving repositories managed by GitLab **(CORE ONLY)**
 
 Sometimes you need to move all repositories managed by GitLab to
-another file system or another server. In this document we look
+another file system or another server.
+
+## Moving data within a GitLab instance
+
+The recommended way to move Git repositories between servers, between different storage, and
+from unclustered to clustered Gitaly (Praefect) is using the API.
+
+Read more:
+
+- [Configuring additional storage for Gitaly](../gitaly/index.md#network-architecture)
+  - Within this example, additional storage called `storage1` and `storage2` is configured.
+- [The API documentation](../../api/project_repository_storage_moves.md) details the endpoints for quering and scheduling repository moves.
+- [Migrate existing repositories to Gitaly Cluster](../gitaly/praefect.md#migrate-existing-repositories-to-gitaly-cluster)
+
+### Limitations
+
+Read more in the [API documentation](../../api/project_repository_storage_moves.md#limitations).
+
+## Migrating to another GitLab instance
+
+Using the API isn't an option if you are migrating to a new GitLab environment, for example:
+
+- From a single-node GitLab to a scaled-out architecture.
+- From a GitLab instance in your private datacenter to a cloud provider.
+
+The rest of the document will look
 at some of the ways you can copy all your repositories from
 `/var/opt/gitlab/git-data/repositories` to `/mnt/gitlab/repositories`.
 
@@ -22,7 +47,14 @@ DANGER: **Warning:**
 Each of the approaches we list can or does overwrite data in the target directory
 `/mnt/gitlab/repositories`. Do not mix up the source and the target.
 
-## Target directory is empty: use a `tar` pipe
+### Recommended approach in all cases
+
+GitLab's [backup and restore capability](../../raketasks/backup_restore.md) should be used. Git repositories are accessed, managed and stored on GitLab servers by the Gitaly component of the product as a database. Data loss can result from directly accessing and copying Gitaly's files using tools like `rsync`.
+
+- From GitLab 13.3, backup performance can be improved by [processing multiple repositories concurrently](../../raketasks/backup_restore.md#back-up-git-repositories-concurrently).
+- Backups can be created of just the repositories using the [skip feature](../../raketasks/backup_restore.md#excluding-specific-directories-from-the-backup)
+
+### Target directory is empty: use a `tar` pipe
 
 If the target directory `/mnt/gitlab/repositories` is empty the
 simplest thing to do is to use a `tar` pipe. This method has low
@@ -37,7 +69,7 @@ sudo -u git sh -c 'tar -C /var/opt/gitlab/git-data/repositories -cf - -- . |\
 
 If you want to see progress, replace `-xf` with `-xvf`.
 
-### `tar` pipe to another server
+#### `tar` pipe to another server
 
 You can also use a `tar` pipe to copy data to another server. If your
 `git` user has SSH access to the new server as `git@newserver`, you
@@ -51,7 +83,12 @@ sudo -u git sh -c 'tar -C /var/opt/gitlab/git-data/repositories -cf - -- . |\
 If you want to compress the data before it goes over the network
 (which costs you CPU cycles) you can replace `ssh` with `ssh -C`.
 
-## The target directory contains an outdated copy of the repositories: use `rsync`
+### The target directory contains an outdated copy of the repositories: use `rsync`
+
+DANGER: **Warning:**
+Using `rsync` to migrate
+Git data can cause data loss and repository corruption.
+[These instructions are being reviewed](https://gitlab.com/gitlab-org/gitlab/-/issues/270422).
 
 If the target directory already contains a partial / outdated copy
 of the repositories it may be wasteful to copy all the data again
@@ -68,7 +105,12 @@ The `/.` in the command above is very important, without it you can
 easily get the wrong directory structure in the target directory.
 If you want to see progress, replace `-a` with `-av`.
 
-### Single `rsync` to another server
+#### Single `rsync` to another server
+
+DANGER: **Warning:**
+Using `rsync` to migrate
+Git data can cause data loss and repository corruption.
+[These instructions are being reviewed](https://gitlab.com/gitlab-org/gitlab/-/issues/270422).
 
 If the `git` user on your source system has SSH access to the target
 server you can send the repositories over the network with `rsync`.
@@ -78,7 +120,12 @@ sudo -u git sh -c 'rsync -a --delete /var/opt/gitlab/git-data/repositories/. \
   git@newserver:/mnt/gitlab/repositories'
 ```
 
-## Thousands of Git repositories: use one `rsync` per repository
+### Thousands of Git repositories: use one `rsync` per repository
+
+DANGER: **Warning:**
+Using `rsync` to migrate
+Git data can cause data loss and repository corruption.
+[These instructions are being reviewed](https://gitlab.com/gitlab-org/gitlab/-/issues/270422).
 
 Every time you start an `rsync` job it has to inspect all files in
 the source directory, all files in the target directory, and then
@@ -93,11 +140,14 @@ This utility is not included in GitLab so you need to install it yourself with `
 or `yum`. Also note that the GitLab scripts we used below were added in GitLab 8.1.
 
 **This process does not clean up repositories at the target location that no
-longer exist at the source.** If you start using your GitLab instance with
-`/mnt/gitlab/repositories`, you need to run `gitlab-rake gitlab:cleanup:repos`
-after switching to the new repository storage directory.
+longer exist at the source.**
+
+#### Parallel `rsync` for all repositories known to GitLab
 
-### Parallel `rsync` for all repositories known to GitLab
+DANGER: **Warning:**
+Using `rsync` to migrate
+Git data can cause data loss and repository corruption.
+[These instructions are being reviewed](https://gitlab.com/gitlab-org/gitlab/-/issues/270422).
 
 This syncs repositories with 10 `rsync` processes at a time. We keep
 track of progress so that the transfer can be restarted if necessary.
@@ -154,7 +204,12 @@ cat /home/git/transfer-logs/* | sort | uniq -u |\
 `
 ```
 
-### Parallel `rsync` only for repositories with recent activity
+#### Parallel `rsync` only for repositories with recent activity
+
+DANGER: **Warning:**
+Using `rsync` to migrate
+Git data can cause data loss and repository corruption.
+[These instructions are being reviewed](https://gitlab.com/gitlab-org/gitlab/-/issues/270422).
 
 Suppose you have already done one sync that started after 2015-10-1 12:00 UTC.
 Then you might only want to sync repositories that were changed via GitLab

doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md

@@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
 type: reference
 ---
 
-# GitLab Rails Console Cheat Sheet
+# GitLab Rails Console Cheat Sheet **(CORE ONLY)**
 
 This is the GitLab Support Team's collection of information regarding the GitLab Rails
 console, for use while troubleshooting. It is listed here for transparency,
@@ -46,6 +46,40 @@ instance_of_object.method(:foo).source_location
 project.method(:private?).source_location
 ```
 
+## Attributes
+
+View available attributes, formatted using pretty print (`pp`).
+
+For example, determine what attributes contain users' names and email addresses:
+
+```ruby
+u = User.find_by_username('someuser')
+pp u.attributes
+```
+
+Partial output:
+
+```plaintext
+{"id"=>1234,
+ "email"=>"someuser@example.com",
+ "sign_in_count"=>99,
+ "name"=>"S User",
+ "username"=>"someuser",
+ "first_name"=>nil,
+ "last_name"=>nil,
+ "bot_type"=>nil}
+```
+
+Then make use of the attributes, [testing SMTP, for example](https://docs.gitlab.com/omnibus/settings/smtp.html#testing-the-smtp-configuration):
+
+```ruby
+e = u.email
+n = u.name
+Notify.test_email(e, "Test email for #{n}", 'Test email').deliver_now
+#
+Notify.test_email(u.email, "Test email for #{u.name}", 'Test email').deliver_now
+```
+
 ## Query the database using an ActiveRecord Model
 
 ```ruby
@@ -148,7 +182,7 @@ project.repository.expire_exists_cache
 Project.update_all(visibility_level: 0)
 ```
 
-### Find & remove projects that are pending deletion
+### Find projects that are pending deletion
 
 ```ruby
 #
@@ -177,8 +211,6 @@ project = Project.find_by_full_path('group-changeme/project-changeme')
 ::Projects::DestroyService.new(project, user, {}).execute
 ```
 
-Next, run `sudo gitlab-rake gitlab:cleanup:repos` on the command line to finish.
-
 ### Destroy a project
 
 ```ruby

doc/api/project_repository_storage_moves.md

@@ -30,6 +30,11 @@ read-only. Please try again later.` message if they try to push new commits.
 
 This API requires you to [authenticate yourself](README.md#authentication) as an administrator.
 
+## Limitations
+
+- [The repositories associated with snippets can't currently be moved with the API](https://gitlab.com/groups/gitlab-org/-/epics/3393).
+- [Group level wikis can't currently be moved with the API](https://gitlab.com/gitlab-org/gitlab/-/issues/219003).
+
 ## Retrieve all project repository storage moves
 
 ```plaintext

doc/development/testing_guide/ci.md

@@ -6,7 +6,7 @@ Our current CI parallelization setup is as follows:
 
 1. The `retrieve-tests-metadata` job in the `prepare` stage ensures we have a
    `knapsack/report-master.json` file:
-   - The `knapsack/report-master.json` file is fetched from the latest `master` artifacts, if it's not here
+   - The `knapsack/report-master.json` file is fetched from S3, if it's not here
      we initialize the file with `{}`.
 1. Each `[rspec|rspec-ee] [unit|integration|system|geo] n m` job are run with
    `knapsack rspec` and should have an evenly distributed share of tests:
@@ -19,7 +19,7 @@ Our current CI parallelization setup is as follows:
 1. The `update-tests-metadata` job (which only runs on scheduled pipelines for
    [the canonical project](https://gitlab.com/gitlab-org/gitlab) takes all the
    `knapsack/rspec*_pg_*.json` files and merge them all together into a single
-   `knapsack/report-master.json` file that is saved as artifact.
+   `knapsack/report-master.json` file that is then uploaded to S3.
 
 After that, the next pipeline will use the up-to-date `knapsack/report-master.json` file.
 

scripts/rspec_helpers.sh

@@ -4,11 +4,11 @@ function retrieve_tests_metadata() {
   mkdir -p knapsack/ rspec_flaky/ rspec_profiling/
 
   if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
-    wget -O "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "https://gitlab.com/gitlab-org/gitlab/-/jobs/artifacts/master/raw/${KNAPSACK_RSPEC_SUITE_REPORT_PATH}?job=retrieve-tests-metadata" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+    wget -O "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
   fi
 
   if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
-    wget -O "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "https://gitlab.com/gitlab-org/gitlab/-/jobs/artifacts/master/raw/${FLAKY_RSPEC_SUITE_REPORT_PATH}?job=retrieve-tests-metadata" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+    wget -O "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
   fi
 }
 
@@ -16,11 +16,29 @@ function update_tests_metadata() {
   echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
 
   scripts/merge-reports "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" knapsack/rspec*.json
+  if [[ -n "${TESTS_METADATA_S3_BUCKET}" ]]; then
+    if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
+      scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+    else
+      echo "Not uplaoding report to S3 as the pipeline is not a scheduled one."
+    fi
+  fi
+
   rm -f knapsack/rspec*.json
 
-  export FLAKY_RSPEC_GENERATE_REPORT="true"
   scripts/merge-reports "${FLAKY_RSPEC_SUITE_REPORT_PATH}" rspec_flaky/all_*.json
+
+  export FLAKY_RSPEC_GENERATE_REPORT="true"
   scripts/flaky_examples/prune-old-flaky-examples "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+
+  if [[ -n ${TESTS_METADATA_S3_BUCKET} ]]; then
+    if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
+      scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+    else
+      echo "Not uploading report to S3 as the pipeline is not a scheduled one."
+    fi
+  fi
+
   rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
 
   if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then

spec/features/merge_request/user_resolves_conflicts_spec.rb

@@ -3,6 +3,8 @@
 require 'spec_helper'
 
 RSpec.describe 'Merge request > User resolves conflicts', :js do
+  include Spec::Support::Helpers::Features::EditorLiteSpecHelpers
+
   let(:project) { create(:project, :repository) }
   let(:user) { project.creator }
 
@@ -64,15 +66,13 @@ RSpec.describe 'Merge request > User resolves conflicts', :js do
       within find('.files-wrapper .diff-file', text: 'files/ruby/popen.rb') do
         click_button 'Edit inline'
         wait_for_requests
-        find('.files-wrapper .diff-file pre')
-        execute_script('ace.edit($(".files-wrapper .diff-file pre")[0]).setValue("One morning");')
+        editor_set_value("One morning")
       end
 
       within find('.files-wrapper .diff-file', text: 'files/ruby/regex.rb') do
         click_button 'Edit inline'
         wait_for_requests
-        find('.files-wrapper .diff-file pre')
-        execute_script('ace.edit($(".files-wrapper .diff-file pre")[1]).setValue("Gregor Samsa woke from troubled dreams");')
+        editor_set_value("Gregor Samsa woke from troubled dreams")
       end
 
       find_button('Commit to source branch').send_keys(:return)