Merge remote-tracking branch 'upstream/master' into qa-allow-setting-sandbox-group

* upstream/master: (27 commits)
  Set initial password for instance in LDAP QA test
  Backport EE changes to some hashed storage documentation to CE
  Remove allow_n_plus_1 from Git::Repository#branches_filter
  Bumps Gitlab Shell version to 6.0.3
  Make resetting column information overridable in EE
  Added 'clear' button to ci lint editor
  Issues and merge requests in subgroups docs
  Update docs labels CE
  Refactored merge_requests/show path in dispatcher.js
  wording
  don't check against a hardcoded user name
  10.5 Update the dependencies license list
  10.5 Update the .gitignore, .gitlab-ci.yml, and Dockerfile templates
  Create update guide for 10.5
  Update 10.5 source install guide
  Add docs for MR link in commit page
  Add groups to OpenID Connect claims
  Replaced $.get with axois.get
  Memoize MergeRequest#rebase_in_progress? to prevent N+1 queries in Gitaly
  [docs] Info rescheduling background migrations
  ...

GITLAB_SHELL_VERSION

-6.0.2
+6.0.3

app/assets/javascripts/boards/components/new_list_dropdown.js

 /* eslint-disable func-names, no-new, space-before-function-paren, one-var,
    promise/catch-or-return */
+import axios from '~/lib/utils/axios_utils';
 import _ from 'underscore';
 import CreateLabelDropdown from '../../create_label';
 
@@ -28,9 +29,9 @@ gl.issueBoards.newListDropdownInit = () => {
 
     $this.glDropdown({
       data(term, callback) {
-        $.get($this.attr('data-list-labels-path'))
-          .then((resp) => {
-            callback(resp);
+        axios.get($this.attr('data-list-labels-path'))
+          .then(({ data }) => {
+            callback(data);
           });
       },
       renderRow (label) {

app/assets/javascripts/dispatcher.js

 /* eslint-disable func-names, space-before-function-paren, no-var, prefer-arrow-callback, wrap-iife, no-shadow, consistent-return, one-var, one-var-declaration-per-line, camelcase, default-case, no-new, quotes, no-duplicate-case, no-case-declarations, no-fallthrough, max-len */
-import MergeRequest from './merge_request';
 import Flash from './flash';
 import GfmAutoComplete from './gfm_auto_complete';
-import ZenMode from './zen_mode';
-import initNotes from './init_notes';
-import initIssuableSidebar from './init_issuable_sidebar';
 import { convertPermissionToBoolean } from './lib/utils/common_utils';
 import GlFieldErrors from './gl_field_errors';
 import Shortcuts from './shortcuts';
-import ShortcutsIssuable from './shortcuts_issuable';
-import Diff from './diff';
 import SearchAutocomplete from './search_autocomplete';
 
 var Dispatcher;
@@ -262,17 +256,10 @@ var Dispatcher;
             .catch(fail);
           break;
         case 'projects:merge_requests:show':
-          new Diff();
-          new ZenMode();
-
-          initIssuableSidebar();
-          initNotes();
-
-          const mrShowNode = document.querySelector('.merge-request');
-          window.mergeRequest = new MergeRequest({
-            action: mrShowNode.dataset.mrAction,
-          });
-          shortcut_handler = new ShortcutsIssuable(true);
+          import('./pages/projects/merge_requests/show')
+            .then(callDefault)
+            .catch(fail);
+          shortcut_handler = true;
           break;
         case 'dashboard:activity':
           import('./pages/dashboard/activity')

app/assets/javascripts/gl_form.js

-/* global autosize */
-
+import autosize from 'autosize';
 import GfmAutoComplete from './gfm_auto_complete';
 import dropzoneInput from './dropzone_input';
 import textUtils from './lib/utils/text_markdown';

app/assets/javascripts/pages/ci/lints/ci_lint_editor.js

@@ -2,11 +2,18 @@ export default class CILintEditor {
   constructor() {
     this.editor = window.ace.edit('ci-editor');
     this.textarea = document.querySelector('#content');
+    this.clearYml = document.querySelector('.clear-yml');
 
     this.editor.getSession().setMode('ace/mode/yaml');
     this.editor.on('input', () => {
       const content = this.editor.getSession().getValue();
       this.textarea.value = content;
     });
+
+    this.clearYml.addEventListener('click', this.clear.bind(this));
+  }
+
+  clear() {
+    this.editor.setValue('');
   }
 }

app/assets/javascripts/pages/projects/merge_requests/show/index.js

+import MergeRequest from '~/merge_request';
+import ZenMode from '~/zen_mode';
+import initNotes from '~/init_notes';
+import initIssuableSidebar from '~/init_issuable_sidebar';
+import ShortcutsIssuable from '~/shortcuts_issuable';
+import Diff from '~/diff';
+import { handleLocationHash } from '~/lib/utils/common_utils';
+
+export default () => {
+  new Diff(); // eslint-disable-line no-new
+  new ZenMode(); // eslint-disable-line no-new
+
+  initIssuableSidebar(); // eslint-disable-line no-new
+  initNotes(); // eslint-disable-line no-new
+
+  const mrShowNode = document.querySelector('.merge-request');
+
+  window.mergeRequest = new MergeRequest({
+    action: mrShowNode.dataset.mrAction,
+  });
+
+  new ShortcutsIssuable(true); // eslint-disable-line no-new
+  handleLocationHash();
+};

app/models/merge_request.rb

@@ -158,11 +158,13 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def rebase_in_progress?
+    strong_memoize(:rebase_in_progress) do
       # The source project can be deleted
-    return false unless source_project
+      next false unless source_project
 
       source_project.repository.rebase_in_progress?(id)
     end
+  end
 
   # Use this method whenever you need to make sure the head_pipeline is synced with the
   # branch head commit, for example checking if a merge request can be merged.

app/models/user.rb

@@ -551,7 +551,7 @@ class User < ActiveRecord::Base
     gpg_keys.each(&:update_invalid_gpg_signatures)
   end
 
-  # Returns the groups a user has access to
+  # Returns the groups a user has access to, either through a membership or a project authorization
   def authorized_groups
     union = Gitlab::SQL::Union
       .new([groups.select(:id), authorized_projects.select(:namespace_id)])
@@ -559,6 +559,11 @@ class User < ActiveRecord::Base
     Group.where("namespaces.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection
   end
 
+  # Returns the groups a user is a member of, either directly or through a parent group
+  def membership_groups
+    Gitlab::GroupHierarchy.new(groups).base_and_descendants
+  end
+
   # Returns a relation of groups the user has access to, including their parent
   # and child groups (recursively).
   def all_expanded_groups

app/views/ci/lints/show.html.haml

@@ -18,6 +18,8 @@
       .col-sm-12
         .pull-left.prepend-top-10
           = submit_tag('Validate', class: 'btn btn-success submit-yml')
+        .pull-right.prepend-top-10
+          = button_tag('Clear', type: 'button', class: 'btn btn-default clear-yml')
 
   .row.prepend-top-20
     .col-sm-12

changelogs/unreleased/bvl-fix-concurrent-fork-network-migrations.yml

+---
+title: Avoid running `PopulateForkNetworksRange`-migration multiple times
+merge_request: 16988
+author:
+type: fixed

changelogs/unreleased/feature-26598-clear-button-ci-lint.yml

+---
+title: Added clear button to ci lint editor
+merge_request:
+author: Michael Robinson

changelogs/unreleased/feature-oidc-groups-claim.yml

+---
+title: Add groups to OpenID Connect claims
+merge_request: 16929
+author: Hassan Zamani

config/initializers/doorkeeper_openid_connect.rb

@@ -31,6 +31,7 @@ Doorkeeper::OpenidConnect.configure do
       o.claim(:website)        { |user| user.full_website_url if user.website_url? }
       o.claim(:profile)        { |user| Gitlab::Routing.url_helpers.user_url user }
       o.claim(:picture)        { |user| user.avatar_url(only_path: false) }
+      o.claim(:groups)         { |user| user.membership_groups.map(&:full_path) }
     end
   end
 end

config/locales/doorkeeper.en.yml

@@ -68,7 +68,7 @@ en:
       read_user:
         Read-only access to the user's profile information, like username, public email and full name
       openid:
-        The ability to authenticate using GitLab, and read-only access to the user's profile information
+        The ability to authenticate using GitLab, and read-only access to the user's profile information and group memberships
       sudo:
         Access to the Sudo feature, to perform API actions as any user in the system (only available for admins)
     flash:

db/migrate/20170929131201_populate_fork_networks.rb

@@ -6,22 +6,8 @@ class PopulateForkNetworks < ActiveRecord::Migration
 
   DOWNTIME = false
 
-  MIGRATION = 'PopulateForkNetworksRange'.freeze
-  BATCH_SIZE = 100
-  DELAY_INTERVAL = 15.seconds
-
-  disable_ddl_transaction!
-
-  class ForkedProjectLink < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'forked_project_links'
-  end
-
   def up
-    say 'Populating the `fork_networks` based on existing `forked_project_links`'
-
-    queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+    say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller'
   end
 
   def down

db/post_migrate/20171124150326_reschedule_fork_network_creation.rb

@@ -3,22 +3,8 @@ class RescheduleForkNetworkCreation < ActiveRecord::Migration
 
   DOWNTIME = false
 
-  MIGRATION = 'PopulateForkNetworksRange'.freeze
-  BATCH_SIZE = 100
-  DELAY_INTERVAL = 15.seconds
-
-  disable_ddl_transaction!
-
-  class ForkedProjectLink < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'forked_project_links'
-  end
-
   def up
-    say 'Populating the `fork_networks` based on existing `forked_project_links`'
-
-    queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+    say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller'
   end
 
   def down

doc/administration/repository_storage_types.md

@@ -4,50 +4,63 @@
 
 ## Legacy Storage
 
-Legacy Storage is the storage behavior prior to version 10.0. For historical reasons, GitLab replicated the same
-mapping structure from the projects URLs:
+Legacy Storage is the storage behavior prior to version 10.0. For historical
+reasons, GitLab replicated the same mapping structure from the projects URLs:
 
- * Project's repository: `#{namespace}/#{project_name}.git`
- * Project's wiki: `#{namespace}/#{project_name}.wiki.git`
+* Project's repository: `#{namespace}/#{project_name}.git`
+* Project's wiki: `#{namespace}/#{project_name}.wiki.git`
 
-This structure made simple to migrate from existing solutions to GitLab and easy for Administrators to find where the
-repository is stored.
+This structure made it simple to migrate from existing solutions to GitLab and
+easy for Administrators to find where the repository is stored.
 
 On the other hand this has some drawbacks:
 
-Storage location will concentrate huge amount of top-level namespaces. The impact can be reduced by the introduction of [multiple storage paths][storage-paths].
+Storage location will concentrate huge amount of top-level namespaces. The
+impact can be reduced by the introduction of [multiple storage
+paths][storage-paths].
 
-Because Backups are a snapshot of the same URL mapping, if you try to recover a very old backup, you need to verify
-if any project has taken the place of an old removed project sharing the same URL. This means that `mygroup/myproject`
-from your backup may not be the same original project that is today in the same URL.
+Because backups are a snapshot of the same URL mapping, if you try to recover a
+very old backup, you need to verify whether any project has taken the place of
+an old removed or renamed project sharing the same URL. This means that
+`mygroup/myproject` from your backup may not be the same original project that
+is at that same URL today.
 
-Any change in the URL will need to be reflected on disk (when groups / users or projects are renamed). This can add a lot
-of load in big installations, and can be even worst if they are using any type of network based filesystem.
+Any change in the URL will need to be reflected on disk (when groups / users or
+projects are renamed). This can add a lot of load in big installations,
+especially if using any type of network based filesystem.
 
-Last, for GitLab Geo, this storage type means we have to synchronize the disk state, replicate renames in the correct
-order or we may end-up with wrong repository or missing data temporarily.
+For GitLab Geo in particular: Geo does work with legacy storage, but in some
+edge cases due to race conditions it can lead to errors when a project is
+renamed multiple times in short succession, or a project is deleted and
+recreated under the same name very quickly. We expect these race events to be
+rare, and we have not observed a race condition side-effect happening yet.
 
-This pattern also exists in other objects stored in GitLab, like issue Attachments, GitLab Pages artifacts,
-Docker Containers for the integrated Registry, etc.
+This pattern also exists in other objects stored in GitLab, like issue
+Attachments, GitLab Pages artifacts, Docker Containers for the integrated
+Registry, etc.
 
 ## Hashed Storage
 
-Hashed Storage is the new storage behavior we are rolling out with 10.0. It's not enabled by default yet, but we
-encourage everyone to try-it and take the time to fix any script you may have that depends on the old behavior.
+> **Warning:** Hashed storage is in **Beta**. For the latest updates, check the
+> associated [issue](https://gitlab.com/gitlab-com/infrastructure/issues/2821)
+> and please report any problems you encounter.
 
-Instead of coupling project URL and the folder structure where the repository will be stored on disk, we are coupling
-a hash, based on the project's ID.
+Hashed Storage is the new storage behavior we are rolling out with 10.0. Instead
+of coupling project URL and the folder structure where the repository will be
+stored on disk, we are coupling a hash, based on the project's ID. This makes
+the folder structure immutable, and therefore eliminates any requirement to
+synchronize state from URLs to disk structure. This means that renaming a group,
+user, or project will cost only the database transaction, and will take effect
+immediately.
 
-This makes the folder structure immutable, and therefore eliminates any requirement to synchronize state from URLs to
-disk structure. This means that renaming a group, user or project will cost only the database transaction, and will take
-effect immediately.
+The hash also helps to spread the repositories more evenly on the disk, so the
+top-level directory will contain less folders than the total amount of top-level
+namespaces.
 
-The hash also helps to spread the repositories more evenly on the disk, so the top-level directory will contain less
-folders than the total amount of top-level namespaces.
-
-Hash format is based on hexadecimal representation of SHA256: `SHA256(project.id)`.
-Top-level folder uses first 2 characters, followed by another folder with the next 2 characters. They are both stored in
-a special folder `@hashed`, to co-exist with existing Legacy projects:
+The hash format is based on the hexadecimal representation of SHA256:
+`SHA256(project.id)`. The top-level folder uses the first 2 characters, followed
+by another folder with the next 2 characters. They are both stored in a special
+`@hashed` folder, to be able to co-exist with existing Legacy Storage projects:
 
 ```ruby
 # Project's repository:
@@ -57,15 +70,13 @@ a special folder `@hashed`, to co-exist with existing Legacy projects:
 "@hashed/#{hash[0..1]}/#{hash[2..3]}/#{hash}.wiki.git"
 ```
 
-This new format also makes possible to restore backups with confidence, as when restoring a repository from the backup,
-you will never mistakenly restore a repository in the wrong project (considering the backup is made after the migration).
-
 ### How to migrate to Hashed Storage
 
-In GitLab, go to **Admin > Settings**, find the **Repository Storage** section and select
-"_Create new projects using hashed storage paths_".
+In GitLab, go to **Admin > Settings**, find the **Repository Storage** section
+and select "_Create new projects using hashed storage paths_".
 
-To migrate your existing projects to the new storage type, check the specific [rake tasks].
+To migrate your existing projects to the new storage type, check the specific
+[rake tasks].
 
 [ce-28283]: https://gitlab.com/gitlab-org/gitlab-ce/issues/28283
 [rake tasks]: raketasks/storage.md#migrate-existing-projects-to-hashed-storage
@@ -73,11 +84,13 @@ To migrate your existing projects to the new storage type, check the specific [r
 
 ### Hashed Storage coverage
 
-We are incrementally moving every storable object in GitLab to the Hashed Storage pattern. You can check the current
-coverage status below.
+We are incrementally moving every storable object in GitLab to the Hashed
+Storage pattern. You can check the current coverage status below (and also see
+the [issue](https://gitlab.com/gitlab-com/infrastructure/issues/2821)).
 
-Note that things stored in an S3 compatible endpoint will not have the downsides mentioned earlier, if they are not
-prefixed with `#{namespace}/#{project_name}`, which is true for CI Cache and LFS Objects.
+Note that things stored in an S3 compatible endpoint will not have the downsides
+mentioned earlier, if they are not prefixed with `#{namespace}/#{project_name}`,
+which is true for CI Cache and LFS Objects.
 
 | Storable Object | Legacy Storage | Hashed Storage | S3 Compatible | GitLab Version |
 | --------------- | -------------- | -------------- | ------------- | -------------- |

doc/development/background_migrations.md

@@ -94,6 +94,18 @@ jobs = [['BackgroundMigrationClassName', [1]],
 BackgroundMigrationWorker.bulk_perform_in(5.minutes, jobs)
 ```
 
+### Rescheduling background migrations
+
+If one of the background migrations contains a bug that is fixed in a patch
+release, the background migration needs to be rescheduled so the migration would
+be repeated on systems that already performed the initial migration.
+
+When you reschedule the background migration, make sure to turn the original
+scheduling into a no-op by clearing up the `#up` and `#down` methods of the
+migration performing the scheduling. Otherwise the background migration would be
+scheduled multiple times on systems that are upgrading multiple patch releases at
+once.
+
 ## Cleaning Up
 
 >**Note:**

doc/development/i18n/externalization.md

@@ -126,6 +126,9 @@ strings and remove any strings that aren't used anymore. You should check this
 file in. Once the changes are on master, they will be picked up by
 [Crowdin](http://translate.gitlab.com) and be presented for translation.
 
+If there are merge conflicts in the `gitlab.pot` file, you can delete the file
+and regenerate it using the same command. Confirm that you are not deleting any strings accidentally by looking over the diff.
+
 The command also updates the translation files for each language: `locale/*/gitlab.po`
 These changes can be discarded, the languange files will be updated by Crowdin
 automatically.

doc/install/installation.md

@@ -299,9 +299,9 @@ sudo usermod -aG redis git
 ### Clone the Source
 
     # Clone GitLab repository
-    sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ce.git -b 10-4-stable gitlab
+    sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ce.git -b 10-5-stable gitlab
 
-**Note:** You can change `10-4-stable` to `master` if you want the *bleeding edge* version, but never install master on a production server!
+**Note:** You can change `10-5-stable` to `master` if you want the *bleeding edge* version, but never install master on a production server!
 
 ### Configure It
 

doc/integration/openid_connect_provider.md

@@ -39,6 +39,7 @@ Currently the following user information is shared with clients:
 | `website`        | `string`  | URL for the user's website
 | `profile`        | `string`  | URL for the user's GitLab profile
 | `picture`        | `string`  | URL for the user's GitLab avatar
+| `groups`         | `array`   | Names of the groups the user is a member of
 
 [OpenID Connect]: http://openid.net/connect/ "OpenID Connect website"
 [doorkeeper-openid_connect]: https://github.com/doorkeeper-gem/doorkeeper-openid_connect "Doorkeeper::OpenidConnect website"

doc/user/project/issues/index.md

@@ -64,9 +64,7 @@ You can also [search and filter](../../search/index.md#issues-and-merge-requests
 
 ### Issues per group
 
-View all the issues in a group (that is, all the issues across all projects in that
-group) by navigating to **Group > Issues**. This view also has the open and closed
-issue tabs.
+View issues in all projects in the group, including all projects of all descendant subgroups of the group. Navigate to **Group > Issues** to view these issues. This view also has the open and closed issues tabs.
 
 ![Group Issues list view](img/group_issues_list_view.png)
 

doc/user/project/merge_requests/index.md

@@ -70,9 +70,9 @@ and you can use the tabs available to quickly filter by open and closed. You can
 
 ## Merge requests per group
 
-View all the merge requests in a group (that is, all the merge requests across all projects in that
-group) by navigating to **Group > Merge Requests**. This view also has the open, merged, and closed
-merge request tabs, from which you can [search and filter the results](../../search/index.md#issues-and-merge-requests-per-group).
+View merge requests in all projects in the group, including all projects of all descendant subgroups of the group. Navigate to **Group > Merge Requests** to view these merge requests. This view also has the open and closed merge requests tabs.
+
+You can [search and filter the results](../../search/index.md#issues-and-merge-requests-per-group) from here.
 
 ![Group Issues list view](img/group_merge_requests_list_view.png)
 
@@ -146,6 +146,19 @@ administrator to do so.
 
 ![Create new merge requests by email](img/create_from_email.png)
 
+## Find the merge request that introduced a change
+
+> **Note**: this feature was [implemented in GitLab 10.5](https://gitlab.com/gitlab-org/gitlab-ce/issues/2383).
+
+When viewing the commit details page, GitLab will link to the merge request (or
+merge requests, if it's in more than one) containing that commit.
+
+This only applies to commits that are in the most recent version of a merge
+request - if a commit was in a merge request, then rebased out of that merge
+request, they will not be linked.
+
+[Read more about merge request versions](versions.md)
+
 ## Revert changes
 
 GitLab implements Git's powerful feature to revert any commit with introducing
@@ -160,7 +173,7 @@ of merge request diff is created. When you visit a merge request that contains
 more than one pushes, you can select and compare the versions of those merge
 request diffs.
 
-[Read more about the merge requests versions.](versions.md)
+[Read more about merge request versions](versions.md)
 
 ## Work In Progress merge requests
 

doc/user/project/merge_requests/work_in_progress_merge_requests.md

@@ -7,7 +7,8 @@ have been marked a **Work In Progress**.
 ![Blocked Accept Button](img/wip_blocked_accept_button.png)
 
 To mark a merge request a Work In Progress, simply start its title with `[WIP]`
-or `WIP:`.
+or `WIP:`. As an alternative, you're also able to do it by sending a commit 
+with its title starting with `wip` or `WIP` to the merge request's source branch.
 
 ![Mark as WIP](img/wip_mark_as_wip.png)
 

doc/user/project/pages/getting_started_part_three.md

@@ -155,15 +155,40 @@ Certificates are NOT required to add to your custom
 (sub)domain on your GitLab Pages project, though they are
 highly recommendable.
 
-The importance of having any website securely served under HTTPS
-is explained on the introductory section of the blog post
-[Secure GitLab Pages with StartSSL](https://about.gitlab.com/2016/06/24/secure-gitlab-pages-with-startssl/#https-a-quick-overview).
+Let's start with an introduction to the importance of HTTPS.
+Alternatively, jump ahead to [adding certificates to your project](#adding-certificates-to-your-project).
 
-The reason why certificates are so important is that they encrypt
+#### Why should I care about HTTPS?
+
+This might be your first question. If our sites are hosted by GitLab Pages,
+they are static, hence we are not dealing with server-side scripts
+nor credit card transactions, then why do we need secure connections?
+
+Back in the 1990s, where HTTPS came out, [SSL](https://en.wikipedia.org/wiki/Transport_Layer_Security#SSL_1.0.2C_2.0_and_3.0) was considered a "special"
+security measure, necessary just for big companies, like banks and shoppings sites
+with financial transactions.
+Now we have a different picture. [According to Josh Aas](https://letsencrypt.org/2015/10/29/phishing-and-malware.html), Executive Director at [ISRG](https://en.wikipedia.org/wiki/Internet_Security_Research_Group):
+
+> _We’ve since come to realize that HTTPS is important for almost all websites. It’s important for any website that allows people to log in with a password, any website that [tracks its users](https://www.washingtonpost.com/news/the-switch/wp/2013/12/10/nsa-uses-google-cookies-to-pinpoint-targets-for-hacking/) in any way, any website that [doesn’t want its content altered](http://arstechnica.com/tech-policy/2014/09/why-comcasts-javascript-ad-injections-threaten-security-net-neutrality/), and for any site that offers content people might not want others to know they are consuming. We’ve also learned that any site not secured by HTTPS [can be used to attack other sites](http://krebsonsecurity.com/2015/04/dont-be-fodder-for-chinas-great-cannon/)._
+
+Therefore, the reason why certificates are so important is that they encrypt
 the connection between the **client** (you, me, your visitors)
 and the **server** (where you site lives), through a keychain of
 authentications and validations.
 
+How about taking Josh's advice and protecting our sites too? We will be
+well supported, and we'll contribute to a safer internet.
+
+#### Organizations supporting HTTPS
+
+There is a huge movement in favor of securing all the web. W3C fully
+[supports the cause](https://w3ctag.github.io/web-https/) and explains very well
+the reasons for that. Richard Barnes, a writer for Mozilla Security Blog,
+suggested that [Firefox would deprecate HTTP](https://blog.mozilla.org/security/2015/04/30/deprecating-non-secure-http/),
+and would no longer accept unsecured connections. Recently, Mozilla published a
+[communication](https://blog.mozilla.org/security/2016/03/29/march-2016-ca-communication/)
+reiterating the importance of HTTPS.
+
 ### Issuing Certificates
 
 GitLab Pages accepts [PEM](https://support.quovadisglobal.com/kb/a37/what-is-pem-format.aspx) certificates issued by

doc/user/project/pages/index.md

@@ -54,7 +54,6 @@ _Blog posts for securing GitLab Pages custom domains with SSL/TLS certificates:_
 
 - [CloudFlare](https://about.gitlab.com/2017/02/07/setting-up-gitlab-pages-with-cloudflare-certificates/)
 - [Let's Encrypt](https://about.gitlab.com/2016/04/11/tutorial-securing-your-gitlab-pages-with-tls-and-letsencrypt/) (outdated)
-- [StartSSL](https://about.gitlab.com/2016/06/24/secure-gitlab-pages-with-startssl/) (deprecated)
 
 ## Advanced use
 

lib/api/internal.rb

@@ -13,7 +13,7 @@ module API
       #   key_id - ssh key id for Git over SSH
       #   user_id - user id for Git over HTTP
       #   protocol - Git access protocol being used, e.g. HTTP or SSH
-      #   project - project path with namespace
+      #   project - project full_path (not path on disk)
       #   action - git action (git-upload-pack or git-receive-pack)
       #   changes - changes as "oldrev newrev ref", see Gitlab::ChangesList
       post "/allowed" do

lib/gitlab/background_migration/create_fork_network_memberships_range.rb

@@ -14,6 +14,14 @@ module Gitlab
       def perform(start_id, end_id)
         log("Creating memberships for forks: #{start_id} - #{end_id}")
 
+        insert_members(start_id, end_id)
+
+        if missing_members?(start_id, end_id)
+          BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id])
+        end
+      end
+
+      def insert_members(start_id, end_id)
         ActiveRecord::Base.connection.execute <<~INSERT_MEMBERS
           INSERT INTO fork_network_members (fork_network_id, project_id, forked_from_project_id)
 
@@ -33,10 +41,9 @@ module Gitlab
             WHERE existing_members.project_id = forked_project_links.forked_to_project_id
           )
         INSERT_MEMBERS
-
-        if missing_members?(start_id, end_id)
-          BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id])
-        end
+      rescue ActiveRecord::RecordNotUnique => e
+        # `fork_network_member` was created concurrently in another migration
+        log(e.message)
       end
 
       def missing_members?(start_id, end_id)

lib/gitlab/git/repository.rb

@@ -1614,9 +1614,7 @@ module Gitlab
 
       # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'.
       def branches_filter(filter: nil, sort_by: nil)
-        # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464
-        branches = Gitlab::GitalyClient.allow_n_plus_1_calls do
-          rugged.branches.each(filter).map do |rugged_ref|
+        branches = rugged.branches.each(filter).map do |rugged_ref|
           begin
             target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
             Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
@@ -1624,7 +1622,6 @@ module Gitlab
             # Omit invalid branch
           end
         end.compact
-        end
 
         sort_branches(branches, sort_by)
       end

lib/gitlab/shell.rb

@@ -294,7 +294,8 @@ module Gitlab
     #   add_namespace("/path/to/storage", "gitlab")
     #
     def add_namespace(storage, name)
-      Gitlab::GitalyClient.migrate(:add_namespace) do |enabled|
+      Gitlab::GitalyClient.migrate(:add_namespace,
+                                   status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
           gitaly_namespace_client(storage).add(name)
         else
@@ -315,7 +316,8 @@ module Gitlab
     #   rm_namespace("/path/to/storage", "gitlab")
     #
     def rm_namespace(storage, name)
-      Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled|
+      Gitlab::GitalyClient.migrate(:remove_namespace,
+                               status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
           gitaly_namespace_client(storage).remove(name)
         else
@@ -333,7 +335,8 @@ module Gitlab
     #   mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
     #
     def mv_namespace(storage, old_name, new_name)
-      Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled|
+      Gitlab::GitalyClient.migrate(:rename_namespace,
+                                   status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
           gitaly_namespace_client(storage).rename(old_name, new_name)
         else
@@ -368,7 +371,8 @@ module Gitlab
     #
     # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
     def exists?(storage, dir_name)
-      Gitlab::GitalyClient.migrate(:namespace_exists) do |enabled|
+      Gitlab::GitalyClient.migrate(:namespace_exists,
+                                   status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
           gitaly_namespace_client(storage).exists?(dir_name)
         else

qa/qa/page/main/login.rb

@@ -31,22 +31,29 @@ module QA
           end
         end
 
+        def set_initial_password_if_present
+          if page.has_content?('Change your password')
+            fill_in :user_password, with: Runtime::User.password
+            fill_in :user_password_confirmation, with: Runtime::User.password
+            click_button 'Change your password'
+          end
+        end
+
         def sign_in_using_ldap_credentials
+          using_wait_time 0 do
+            set_initial_password_if_present
+
             click_link 'LDAP'
 
             fill_in :username, with: Runtime::User.name
             fill_in :password, with: Runtime::User.password
-
             click_button 'Sign in'
           end
+        end
 
         def sign_in_using_credentials
           using_wait_time 0 do
-            if page.has_content?('Change your password')
-              fill_in :user_password, with: Runtime::User.password
-              fill_in :user_password_confirmation, with: Runtime::User.password
-              click_button 'Change your password'
-            end
+            set_initial_password_if_present
 
             click_link 'Standard' if page.has_content?('LDAP')
 

qa/qa/specs/features/api/users_spec.rb

@@ -14,7 +14,7 @@ module QA
       end
 
       scenario 'submit request with a valid user name' do
-        get request.url, { params: { username: 'root' } }
+        get request.url, { params: { username: Runtime::User.name } }
 
         expect_status(200)
         expect(json_body).to be_an Array
@@ -23,7 +23,7 @@ module QA
       end
 
       scenario 'submit request with an invalid user name' do
-        get request.url, { params: { username: 'invalid' } }
+        get request.url, { params: { username: SecureRandom.hex(10) } }
 
         expect_status(200)
         expect(json_body).to be_an Array

spec/controllers/profiles_controller_spec.rb

@@ -69,9 +69,8 @@ describe ProfilesController, :request_store do
 
   describe 'PUT update_username' do
     let(:namespace) { user.namespace }
-    let(:project) { create(:project_empty_repo, namespace: namespace) }
     let(:gitlab_shell) { Gitlab::Shell.new }
-    let(:new_username) { 'renamedtosomethingelse' }
+    let(:new_username) { generate(:username) }
 
     it 'allows username change' do
       sign_in(user)
@@ -85,7 +84,10 @@ describe ProfilesController, :request_store do
       expect(user.username).to eq(new_username)
     end
 
+    context 'with legacy storage' do
       it 'moves dependent projects to new namespace' do
+        project = create(:project_empty_repo, :legacy_storage, namespace: namespace)
+
         sign_in(user)
 
         put :update_username,
@@ -97,4 +99,24 @@ describe ProfilesController, :request_store do
         expect(gitlab_shell.exists?(project.repository_storage_path, "#{new_username}/#{project.path}.git")).to be_truthy
       end
     end
+
+    context 'with hashed storage' do
+      it 'keeps repository location unchanged on disk' do
+        project = create(:project_empty_repo, namespace: namespace)
+
+        before_disk_path = project.disk_path
+
+        sign_in(user)
+
+        put :update_username,
+          user: { username: new_username }
+
+        user.reload
+
+        expect(response.status).to eq(302)
+        expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_truthy
+        expect(before_disk_path).to eq(project.disk_path)
+      end
+    end
+  end
 end

spec/controllers/projects_controller_spec.rb

@@ -288,19 +288,26 @@ describe ProjectsController do
     render_views
 
     let(:admin) { create(:admin) }
-    let(:project) { create(:project, :repository) }
 
     before do
       sign_in(admin)
     end
 
+    shared_examples_for 'updating a project' do
       context 'when only renaming a project path' do
         it "sets the repository to the right path after a rename" do
+          original_repository_path = project.repository.path
+
           expect { update_project path: 'renamed_path' }
             .to change { project.reload.path }
-
           expect(project.path).to include 'renamed_path'
-        expect(assigns(:repository).path).to include project.path
+
+          if project.hashed_storage?(:repository)
+            expect(assigns(:repository).path).to eq(original_repository_path)
+          else
+            expect(assigns(:repository).path).to include(project.path)
+          end
+
           expect(response).to have_gitlab_http_status(302)
         end
       end
@@ -347,6 +354,19 @@ describe ProjectsController do
       end
     end
 
+    context 'hashed storage' do
+      let(:project) { create(:project, :repository) }
+
+      it_behaves_like 'updating a project'
+    end
+
+    context 'legacy storage' do
+      let(:project) { create(:project, :repository, :legacy_storage) }
+
+      it_behaves_like 'updating a project'
+    end
+  end
+
   describe '#transfer' do
     render_views
 

spec/factories/projects.rb

@@ -81,8 +81,10 @@ FactoryBot.define do
       archived true
     end
 
-    trait :hashed do
     storage_version Project::LATEST_STORAGE_VERSION
+
+    trait :legacy_storage do
+      storage_version nil
     end
 
     trait :access_requestable do

spec/features/ci_lint_spec.rb

@@ -3,16 +3,19 @@ require 'spec_helper'
 describe 'CI Lint', :js do
   before do
     sign_in(create(:user))
+
+    visit ci_lint_path
+    find('#ci-editor')
+    execute_script("ace.edit('ci-editor').setValue(#{yaml_content.to_json});")
+
+    # Ace editor updates a hidden textarea and it happens asynchronously
+    wait_for('YAML content') do
+      find('.ace_content').text.present?
+    end
   end
 
   describe 'YAML parsing' do
     before do
-      visit ci_lint_path
-      # Ace editor updates a hidden textarea and it happens asynchronously
-      # `sleep 0.1` is actually needed here because of this
-      find('#ci-editor')
-      execute_script("ace.edit('ci-editor').setValue(" + yaml_content.to_json + ");")
-      sleep 0.1
       click_on 'Validate'
     end
 
@@ -32,11 +35,10 @@ describe 'CI Lint', :js do
     end
 
     context 'YAML is incorrect' do
-      let(:yaml_content) { '' }
+      let(:yaml_content) { 'value: cannot have :' }
 
       it 'displays information about an error' do
         expect(page).to have_content('Status: syntax is incorrect')
-        expect(page).to have_content('Error: Please provide content of .gitlab-ci.yml')
       end
     end
 
@@ -48,4 +50,20 @@ describe 'CI Lint', :js do
       end
     end
   end
+
+  describe 'YAML clearing' do
+    before do
+      click_on 'Clear'
+    end
+
+    context 'YAML is present' do
+      let(:yaml_content) do
+        File.read(Rails.root.join('spec/support/gitlab_stubs/gitlab_ci.yml'))
+      end
+
+      it 'YAML content is cleared' do
+        expect(page).to have_field('content', with: '', visible: false, type: 'textarea')
+      end
+    end
+  end
 end

spec/features/projects/import_export/namespace_export_file_spec.rb

 require 'spec_helper'
 
-feature 'Import/Export - Namespace export file cleanup', :js do
+describe 'Import/Export - Namespace export file cleanup', :js do
   let(:export_path) { Dir.mktmpdir('namespace_export_file_spec') }
 
   before do
@@ -42,13 +42,13 @@ feature 'Import/Export - Namespace export file cleanup', :js do
   end
 
   describe 'legacy storage' do
-    let(:project) { create(:project) }
+    let(:project) { create(:project, :legacy_storage) }
 
     it_behaves_like 'handling project exports on namespace change'
   end
 
   describe 'hashed storage' do
-    let(:project) { create(:project, :hashed) }
+    let(:project) { create(:project) }
 
     it_behaves_like 'handling project exports on namespace change'
   end

spec/javascripts/gl_form_spec.js

-import Autosize from 'autosize';
+import autosize from 'autosize';
 import GLForm from '~/gl_form';
 import '~/lib/utils/text_utility';
 import '~/lib/utils/common_utils';
 
-window.autosize = Autosize;
-
 describe('GLForm', () => {
   describe('when instantiated', function () {
     beforeEach((done) => {
@@ -13,14 +11,12 @@ describe('GLForm', () => {
       spyOn($.prototype, 'off').and.returnValue(this.textarea);
       spyOn($.prototype, 'on').and.returnValue(this.textarea);
       spyOn($.prototype, 'css');
-      spyOn(window, 'autosize');
 
-      this.glForm = new GLForm(this.form);
+      this.glForm = new GLForm(this.form, false);
       setTimeout(() => {
         $.prototype.off.calls.reset();
         $.prototype.on.calls.reset();
         $.prototype.css.calls.reset();
-        window.autosize.calls.reset();
         done();
       });
     });
@@ -43,10 +39,6 @@ describe('GLForm', () => {
         expect($.prototype.on).toHaveBeenCalledWith('mouseup.autosize', jasmine.any(Function));
       });
 
-      it('should autosize the textarea', () => {
-        expect(window.autosize).toHaveBeenCalledWith(jasmine.any(Object));
-      });
-
       it('should set the resize css property to vertical', () => {
         expect($.prototype.css).toHaveBeenCalledWith('resize', 'vertical');
       });
@@ -74,7 +66,7 @@ describe('GLForm', () => {
           spyOn($.prototype, 'data');
           spyOn($.prototype, 'outerHeight').and.returnValue(200);
           spyOn(window, 'outerHeight').and.returnValue(400);
-          spyOn(window.autosize, 'destroy');
+          spyOn(autosize, 'destroy');
 
           this.glForm.destroyAutosize();
         });
@@ -88,7 +80,7 @@ describe('GLForm', () => {
         });
 
         it('should call autosize destroy', () => {
-          expect(window.autosize.destroy).toHaveBeenCalledWith(this.textarea);
+          expect(autosize.destroy).toHaveBeenCalledWith(this.textarea);
         });
 
         it('should set the data-height attribute', () => {
@@ -107,9 +99,9 @@ describe('GLForm', () => {
       it('should return undefined if the data-height equals the outerHeight', () => {
         spyOn($.prototype, 'outerHeight').and.returnValue(200);
         spyOn($.prototype, 'data').and.returnValue(200);
-        spyOn(window.autosize, 'destroy');
+        spyOn(autosize, 'destroy');
         expect(this.glForm.destroyAutosize()).toBeUndefined();
-        expect(window.autosize.destroy).not.toHaveBeenCalled();
+        expect(autosize.destroy).not.toHaveBeenCalled();
       });
     });
   });

spec/lib/backup/repository_spec.rb

@@ -33,6 +33,17 @@ describe Backup::Repository do
         allow(Gitlab::Popen).to receive(:popen).and_return(['error', 1])
       end
 
+      context 'hashed storage' do
+        it 'shows the appropriate error' do
+          described_class.new.restore
+
+          expect(progress).to have_received(:puts).with("Ignoring error on #{project.full_path} (#{project.disk_path}) - error")
+        end
+      end
+
+      context 'legacy storage' do
+        let!(:project) { create(:project, :legacy_storage) }
+
         it 'shows the appropriate error' do
           described_class.new.restore
 
@@ -40,6 +51,7 @@ describe Backup::Repository do
         end
       end
     end
+  end
 
   describe '#empty_repo?' do
     context 'for a wiki' do

spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb

@@ -23,8 +23,8 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq do
     let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) }
     let!(:user1) { create(:user, :with_avatar) }
     let!(:user2) { create(:user, :with_avatar) }
-    let!(:project1) { create(:project, :with_avatar) }
-    let!(:project2) { create(:project, :with_avatar) }
+    let!(:project1) { create(:project, :legacy_storage, :with_avatar) }
+    let!(:project2) { create(:project, :legacy_storage, :with_avatar) }
 
     before do
       UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload
@@ -48,7 +48,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq do
 
     it 'adds untracked files to the uploads table' do
       expect do
-        subject.perform(1, untracked_files_for_uploads.last.id)
+        subject.perform(1, untracked_files_for_uploads.reorder(:id).last.id)
       end.to change { uploads.count }.from(4).to(8)
 
       expect(user2.uploads.count).to eq(1)
@@ -213,13 +213,13 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq do
     end
 
     context 'for a project avatar file path' do
-      let(:model) { create(:project, :with_avatar) }
+      let(:model) { create(:project, :legacy_storage, :with_avatar) }
 
       it_behaves_like 'non_markdown_file'
     end
 
     context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
-      let(:model) { create(:project) }
+      let(:model) { create(:project, :legacy_storage) }
 
       before do
         # Upload the file
@@ -304,7 +304,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do
 
     context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
       it 'returns the file path relative to the project directory in uploads' do
-        project = create(:project)
+        project = create(:project, :legacy_storage)
         random_hex = SecureRandom.hex
 
         assert_upload_path("/#{project.full_path}/#{random_hex}/Some file.jpg", "#{random_hex}/Some file.jpg")
@@ -357,7 +357,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do
 
     context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
       it 'returns FileUploader as a string' do
-        project = create(:project)
+        project = create(:project, :legacy_storage)
 
         assert_uploader("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", 'FileUploader')
       end
@@ -409,7 +409,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do
 
     context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
       it 'returns Project as a string' do
-        project = create(:project)
+        project = create(:project, :legacy_storage)
 
         assert_model_type("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", 'Project')
       end
@@ -461,7 +461,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do
 
     context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
       it 'returns the ID as a string' do
-        project = create(:project)
+        project = create(:project, :legacy_storage)
 
         assert_model_id("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", project.id)
       end
@@ -483,7 +483,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do
     end
 
     context 'for a project avatar file path' do
-      let(:project) { create(:project, avatar: uploaded_file) }
+      let(:project) { create(:project, :legacy_storage, avatar: uploaded_file) }
       let(:untracked_file) { described_class.create!(path: project.uploads.first.path) }
 
       it 'returns the file size' do
@@ -496,7 +496,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do
     end
 
     context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
-      let(:project) { create(:project) }
+      let(:project) { create(:project, :legacy_storage) }
       let(:untracked_file) { create_untracked_file("/#{project.full_path}/#{project.uploads.first.path}") }
 
       before do

spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb

@@ -77,7 +77,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
     context 'when files were uploaded before and after hashed storage was enabled' do
       let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) }
       let!(:user) { create(:user, :with_avatar) }
-      let!(:project1) { create(:project, :with_avatar) }
+      let!(:project1) { create(:project, :with_avatar, :legacy_storage) }
       let(:project2) { create(:project) } # instantiate after enabling hashed_storage
 
       before do
@@ -149,7 +149,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
     context 'when files were uploaded before and after hashed storage was enabled' do
       let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) }
       let!(:user) { create(:user, :with_avatar) }
-      let!(:project1) { create(:project, :with_avatar) }
+      let!(:project1) { create(:project, :with_avatar, :legacy_storage) }
       let(:project2) { create(:project) } # instantiate after enabling hashed_storage
 
       before do

spec/lib/gitlab/bare_repository_import/importer_spec.rb

@@ -8,11 +8,15 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
   subject(:importer) { described_class.new(admin, bare_repository) }
 
   before do
+    @rainbow = Rainbow.enabled
+    Rainbow.enabled = false
+
     allow(described_class).to receive(:log)
   end
 
   after do
     FileUtils.rm_rf(base_dir)
+    Rainbow.enabled = @rainbow
   end
 
   shared_examples 'importing a repository' do
@@ -148,7 +152,7 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
       # This is a quick way to get a valid repository instead of copying an
       # existing one. Since it's not persisted, the importer will try to
       # create the project.
-      project = build(:project, :repository)
+      project = build(:project, :legacy_storage, :repository)
       original_commit_count = project.repository.commit_count
 
       bare_repo = Gitlab::BareRepositoryImport::Repository.new(project.repository_storage_path, project.repository.path)

spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb

@@ -94,7 +94,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
   describe '#move_repositories' do
     let(:namespace) { create(:group, name: 'hello-group') }
     it 'moves a project for a namespace' do
-      create(:project, :repository, namespace: namespace, path: 'hello-project')
+      create(:project, :repository, :legacy_storage, namespace: namespace, path: 'hello-project')
       expected_path = File.join(TestEnv.repos_path, 'bye-group', 'hello-project.git')
 
       subject.move_repositories(namespace, 'hello-group', 'bye-group')
@@ -104,7 +104,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
 
     it 'moves a namespace in a subdirectory correctly' do
       child_namespace = create(:group, name: 'sub-group', parent: namespace)
-      create(:project, :repository, namespace: child_namespace, path: 'hello-project')
+      create(:project, :repository, :legacy_storage, namespace: child_namespace, path: 'hello-project')
 
       expected_path = File.join(TestEnv.repos_path, 'hello-group', 'renamed-sub-group', 'hello-project.git')
 
@@ -115,7 +115,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
 
     it 'moves a parent namespace with subdirectories' do
       child_namespace = create(:group, name: 'sub-group', parent: namespace)
-      create(:project, :repository, namespace: child_namespace, path: 'hello-project')
+      create(:project, :repository, :legacy_storage, namespace: child_namespace, path: 'hello-project')
       expected_path = File.join(TestEnv.repos_path, 'renamed-group', 'sub-group', 'hello-project.git')
 
       subject.move_repositories(child_namespace, 'hello-group', 'renamed-group')
@@ -166,7 +166,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
 
   describe '#rename_namespace_dependencies' do
     it "moves the the repository for a project in the namespace" do
-      create(:project, :repository, namespace: namespace, path: "the-path-project")
+      create(:project, :repository, :legacy_storage, namespace: namespace, path: "the-path-project")
       expected_repo = File.join(TestEnv.repos_path, "the-path0", "the-path-project.git")
 
       subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0')
@@ -187,7 +187,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
     end
 
     it 'invalidates the markdown cache of related projects' do
-      project = create(:project, namespace: namespace, path: "the-path-project")
+      project = create(:project, :legacy_storage, namespace: namespace, path: "the-path-project")
 
       expect(subject).to receive(:remove_cached_html_for_projects).with([project.id])
 
@@ -243,7 +243,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
 
   describe '#revert_renames', :redis do
     it 'renames the routes back to the previous values' do
-      project = create(:project, :repository, path: 'a-project', namespace: namespace)
+      project = create(:project, :legacy_storage, :repository, path: 'a-project', namespace: namespace)
       subject.rename_namespace(namespace)
 
       expect(subject).to receive(:perform_rename)
@@ -261,7 +261,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
     end
 
     it 'moves the repositories back to their original place' do
-      project = create(:project, :repository, path: 'a-project', namespace: namespace)
+      project = create(:project, :repository, :legacy_storage, path: 'a-project', namespace: namespace)
       project.create_repository
       subject.rename_namespace(namespace)
 

spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb

@@ -5,6 +5,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :de
   let(:subject) { described_class.new(['the-path'], migration) }
   let(:project) do
     create(:project,
+           :legacy_storage,
            path: 'the-path',
            namespace: create(:namespace, path: 'known-parent' ))
   end
@@ -17,7 +18,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :de
   describe '#projects_for_paths' do
     it 'searches using nested paths' do
       namespace = create(:namespace, path: 'hello')
-      project = create(:project, path: 'THE-path', namespace: namespace)
+      project = create(:project, :legacy_storage, path: 'THE-path', namespace: namespace)
 
       result_ids = described_class.new(['Hello/the-path'], migration)
                      .projects_for_paths.map(&:id)
@@ -26,8 +27,8 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :de
     end
 
     it 'includes the correct projects' do
-      project = create(:project, path: 'THE-path')
-      _other_project = create(:project)
+      project = create(:project, :legacy_storage, path: 'THE-path')
+      _other_project = create(:project, :legacy_storage)
 
       result_ids = subject.projects_for_paths.map(&:id)
 
@@ -36,7 +37,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :de
   end
 
   describe '#rename_projects' do
-    let!(:projects) { create_list(:project, 2, path: 'the-path') }
+    let!(:projects) { create_list(:project, 2, :legacy_storage, path: 'the-path') }
 
     it 'renames each project' do
       expect(subject).to receive(:rename_project).twice
@@ -120,7 +121,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :de
 
   describe '#move_repository' do
     let(:known_parent) { create(:namespace, path: 'known-parent') }
-    let(:project) { create(:project, :repository, path: 'the-path', namespace: known_parent) }
+    let(:project) { create(:project, :repository, :legacy_storage, path: 'the-path', namespace: known_parent) }
 
     it 'moves the repository for a project' do
       expected_path = File.join(TestEnv.repos_path, 'known-parent', 'new-repo.git')

spec/lib/gitlab/email/attachment_uploader_spec.rb

@@ -2,7 +2,7 @@ require "spec_helper"
 
 describe Gitlab::Email::AttachmentUploader do
   describe "#execute" do
-    let(:project) { build(:project) }
+    let(:project) { create(:project) }
     let(:message_raw) { fixture_file("emails/attachment.eml") }
     let(:message) { Mail::Message.new(message_raw) }
 

spec/lib/gitlab/gfm/uploads_rewriter_spec.rb

@@ -39,8 +39,8 @@ describe Gitlab::Gfm::UploadsRewriter do
       it 'copies files' do
         expect(new_files).to all(exist)
         expect(old_paths).not_to match_array new_paths
-        expect(old_paths).to all(include(old_project.full_path))
-        expect(new_paths).to all(include(new_project.full_path))
+        expect(old_paths).to all(include(old_project.disk_path))
+        expect(new_paths).to all(include(new_project.disk_path))
       end
 
       it 'does not remove old files' do

spec/lib/gitlab/import_export/uploads_restorer_spec.rb

@@ -16,7 +16,7 @@ describe Gitlab::ImportExport::UploadsRestorer do
     end
 
     describe 'legacy storage' do
-      let(:project) { create(:project) }
+      let(:project) { create(:project, :legacy_storage) }
 
       subject(:restorer) { described_class.new(project: project, shared: shared) }
 
@@ -34,7 +34,7 @@ describe Gitlab::ImportExport::UploadsRestorer do
     end
 
     describe 'hashed storage' do
-      let(:project) { create(:project, :hashed) }
+      let(:project) { create(:project) }
 
       subject(:restorer) { described_class.new(project: project, shared: shared) }
 

spec/lib/gitlab/import_export/uploads_saver_spec.rb

@@ -15,7 +15,7 @@ describe Gitlab::ImportExport::UploadsSaver do
     end
 
     describe 'legacy storage' do
-      let(:project) { create(:project) }
+      let(:project) { create(:project, :legacy_storage) }
 
       subject(:saver) { described_class.new(shared: shared, project: project) }
 
@@ -37,7 +37,7 @@ describe Gitlab::ImportExport::UploadsSaver do
     end
 
     describe 'hashed storage' do
-      let(:project) { create(:project, :hashed) }
+      let(:project) { create(:project) }
 
       subject(:saver) { described_class.new(shared: shared, project: project) }
 

spec/lib/gitlab/repo_path_spec.rb

@@ -6,11 +6,11 @@ describe ::Gitlab::RepoPath do
 
     context 'a repository storage path' do
       it 'parses a full repository path' do
-        expect(described_class.parse(project.repository.path)).to eq([project, false, nil])
+        expect(described_class.parse(project.repository.full_path)).to eq([project, false, nil])
       end
 
       it 'parses a full wiki path' do
-        expect(described_class.parse(project.wiki.repository.path)).to eq([project, true, nil])
+        expect(described_class.parse(project.wiki.repository.full_path)).to eq([project, true, nil])
       end
     end
 

spec/lib/gitlab/shell_spec.rb

@@ -443,7 +443,7 @@ describe Gitlab::Shell do
     end
 
     describe '#remove_repository' do
-      let!(:project) { create(:project, :repository) }
+      let!(:project) { create(:project, :repository, :legacy_storage) }
       let(:disk_path) { "#{project.disk_path}.git" }
 
       it 'returns true when the command succeeds' do

spec/lib/gitlab/workhorse_spec.rb

@@ -324,7 +324,7 @@ describe Gitlab::Workhorse do
       it 'includes a Repository param' do
         repo_param = {
           storage_name: 'default',
-          relative_path: project.full_path + '.git',
+          relative_path: project.disk_path + '.git',
           gl_repository: "project-#{project.id}"
         }
 

spec/migrations/migrate_process_commit_worker_jobs_spec.rb

@@ -4,7 +4,7 @@ require 'spec_helper'
 require Rails.root.join('db', 'migrate', '20161124141322_migrate_process_commit_worker_jobs.rb')
 
 describe MigrateProcessCommitWorkerJobs do
-  let(:project) { create(:project, :repository) }
+  let(:project) { create(:project, :legacy_storage, :repository) }
   let(:user) { create(:user) }
   let(:commit) { project.commit.raw.rugged_commit }
 

spec/migrations/turn_nested_groups_into_regular_groups_for_mysql_spec.rb

@@ -4,7 +4,7 @@ require Rails.root.join('db', 'migrate', '20170503140202_turn_nested_groups_into
 describe TurnNestedGroupsIntoRegularGroupsForMysql do
   let!(:parent_group) { create(:group) }
   let!(:child_group) { create(:group, parent: parent_group) }
-  let!(:project) { create(:project, :empty_repo, namespace: child_group) }
+  let!(:project) { create(:project, :legacy_storage, :empty_repo, namespace: child_group) }
   let!(:member) { create(:user) }
   let(:migration) { described_class.new }
 

spec/models/namespace_spec.rb

@@ -168,19 +168,7 @@ describe Namespace do
   end
 
   describe '#move_dir', :request_store do
-    let(:namespace) { create(:namespace) }
-    let!(:project) { create(:project_empty_repo, namespace: namespace) }
-
-    it "raises error when directory exists" do
-      expect { namespace.move_dir }.to raise_error("namespace directory cannot be moved")
-    end
-
-    it "moves dir if path changed" do
-      namespace.update_attributes(path: namespace.full_path + '_new')
-
-      expect(gitlab_shell.exists?(project.repository_storage_path, "#{namespace.path}/#{project.path}.git")).to be_truthy
-    end
-
+    shared_examples "namespace restrictions" do
       context "when any project has container images" do
         let(:container_repository) { create(:container_repository) }
 
@@ -198,17 +186,34 @@ describe Namespace do
           expect { namespace.move_dir }.to raise_error(/Namespace cannot be moved/)
         end
       end
+    end
+
+    context 'legacy storage' do
+      let(:namespace) { create(:namespace) }
+      let!(:project) { create(:project_empty_repo, :legacy_storage, namespace: namespace) }
+
+      it_behaves_like 'namespace restrictions'
+
+      it "raises error when directory exists" do
+        expect { namespace.move_dir }.to raise_error("namespace directory cannot be moved")
+      end
+
+      it "moves dir if path changed" do
+        namespace.update_attributes(path: namespace.full_path + '_new')
+
+        expect(gitlab_shell.exists?(project.repository_storage_path, "#{namespace.path}/#{project.path}.git")).to be_truthy
+      end
 
       context 'with subgroups' do
         let(:parent) { create(:group, name: 'parent', path: 'parent') }
         let(:child) { create(:group, name: 'child', path: 'child', parent: parent) }
-      let!(:project) { create(:project_empty_repo, path: 'the-project', namespace: child, skip_disk_validation: true) }
+        let!(:project) { create(:project_empty_repo, :legacy_storage, path: 'the-project', namespace: child, skip_disk_validation: true) }
         let(:uploads_dir) { FileUploader.root }
         let(:pages_dir) { File.join(TestEnv.pages_path) }
 
         before do
-        FileUtils.mkdir_p(File.join(uploads_dir, 'parent', 'child', 'the-project'))
-        FileUtils.mkdir_p(File.join(pages_dir, 'parent', 'child', 'the-project'))
+          FileUtils.mkdir_p(File.join(uploads_dir, project.full_path))
+          FileUtils.mkdir_p(File.join(pages_dir, project.full_path))
         end
 
         context 'renaming child' do
@@ -239,13 +244,29 @@ describe Namespace do
           end
         end
       end
+    end
+
+    context 'hashed storage' do
+      let(:namespace) { create(:namespace) }
+      let!(:project) { create(:project_empty_repo, namespace: namespace) }
+
+      it_behaves_like 'namespace restrictions'
+
+      it "repository directory remains unchanged if path changed" do
+        before_disk_path = project.disk_path
+        namespace.update_attributes(path: namespace.full_path + '_new')
+
+        expect(before_disk_path).to eq(project.disk_path)
+        expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_truthy
+      end
+    end
 
     it 'updates project full path in .git/config for each project inside namespace' do
       parent = create(:group, name: 'mygroup', path: 'mygroup')
       subgroup = create(:group, name: 'mysubgroup', path: 'mysubgroup', parent: parent)
-      project_in_parent_group = create(:project, :repository, namespace: parent, name: 'foo1')
-      hashed_project_in_subgroup = create(:project, :repository, :hashed, namespace: subgroup, name: 'foo2')
-      legacy_project_in_subgroup = create(:project, :repository, namespace: subgroup, name: 'foo3')
+      project_in_parent_group = create(:project, :legacy_storage, :repository, namespace: parent, name: 'foo1')
+      hashed_project_in_subgroup = create(:project, :repository, namespace: subgroup, name: 'foo2')
+      legacy_project_in_subgroup = create(:project, :legacy_storage, :repository, namespace: subgroup, name: 'foo3')
 
       parent.update(path: 'mygroup_new')
 
@@ -260,12 +281,14 @@ describe Namespace do
   end
 
   describe '#rm_dir', 'callback' do
-    let!(:project) { create(:project_empty_repo, namespace: namespace) }
     let(:repository_storage_path) { Gitlab.config.repositories.storages.default['path'] }
     let(:path_in_dir) { File.join(repository_storage_path, namespace.full_path) }
     let(:deleted_path) { namespace.full_path.gsub(namespace.path, "#{namespace.full_path}+#{namespace.id}+deleted") }
     let(:deleted_path_in_dir) { File.join(repository_storage_path, deleted_path) }
 
+    context 'legacy storage' do
+      let!(:project) { create(:project_empty_repo, :legacy_storage, namespace: namespace) }
+
       it 'renames its dirs when deleted' do
         allow(GitlabShellWorker).to receive(:perform_in)
 
@@ -283,7 +306,7 @@ describe Namespace do
       context 'in sub-groups' do
         let(:parent) { create(:group, path: 'parent') }
         let(:child) { create(:group, parent: parent, path: 'child') }
-      let!(:project) { create(:project_empty_repo, namespace: child) }
+        let!(:project) { create(:project_empty_repo, :legacy_storage, namespace: child) }
         let(:path_in_dir) { File.join(repository_storage_path, 'parent', 'child') }
         let(:deleted_path) { File.join('parent', "child+#{child.id}+deleted") }
         let(:deleted_path_in_dir) { File.join(repository_storage_path, deleted_path) }
@@ -310,6 +333,27 @@ describe Namespace do
       end
     end
 
+    context 'hashed storage' do
+      let!(:project) { create(:project_empty_repo, namespace: namespace) }
+
+      it 'has no repositories base directories to remove' do
+        allow(GitlabShellWorker).to receive(:perform_in)
+
+        expect(File.exist?(path_in_dir)).to be(false)
+
+        namespace.destroy
+
+        expect(File.exist?(deleted_path_in_dir)).to be(false)
+      end
+
+      it 'removes the exports folder' do
+        expect(namespace).to receive(:remove_exports!)
+
+        namespace.destroy
+      end
+    end
+  end
+
   describe '.find_by_path_or_name' do
     before do
       @namespace = create(:namespace, name: 'WoW', path: 'woW')
@@ -567,8 +611,8 @@ describe Namespace do
   end
 
   describe '#remove_exports' do
-    let(:legacy_project) { create(:project, :with_export, namespace: namespace) }
-    let(:hashed_project) { create(:project, :with_export, :hashed, namespace: namespace) }
+    let(:legacy_project) { create(:project, :with_export, :legacy_storage, namespace: namespace) }
+    let(:hashed_project) { create(:project, :with_export, namespace: namespace) }
     let(:export_path) { Dir.mktmpdir('namespace_remove_exports_spec') }
     let(:legacy_export) { legacy_project.export_project_path }
     let(:hashed_export) { hashed_project.export_project_path }

spec/models/project_spec.rb

@@ -2504,6 +2504,7 @@ describe Project do
   end
 
   describe '#remove_exports' do
+    let(:legacy_project) { create(:project, :legacy_storage, :with_export) }
     let(:project) { create(:project, :with_export) }
 
     it 'removes the exports directory for the project' do
@@ -2516,15 +2517,29 @@ describe Project do
       expect(File.exist?(project.export_path)).to be_falsy
     end
 
-    it 'is a no-op when there is no namespace' do
+    it 'is a no-op on legacy projects when there is no namespace' do
+      export_path = legacy_project.export_path
+
+      legacy_project.update_column(:namespace_id, nil)
+
+      expect(FileUtils).not_to receive(:rm_rf).with(export_path)
+
+      legacy_project.remove_exports
+
+      expect(File.exist?(export_path)).to be_truthy
+    end
+
+    it 'runs on hashed storage projects when there is no namespace' do
       export_path = project.export_path
+
       project.update_column(:namespace_id, nil)
 
-      expect(FileUtils).not_to receive(:rm_rf).with(export_path)
+      allow(FileUtils).to receive(:rm_rf).and_call_original
+      expect(FileUtils).to receive(:rm_rf).with(export_path).and_call_original
 
       project.remove_exports
 
-      expect(File.exist?(export_path)).to be_truthy
+      expect(File.exist?(export_path)).to be_falsy
     end
 
     it 'is run when the project is destroyed' do
@@ -2545,7 +2560,7 @@ describe Project do
   end
 
   context 'legacy storage' do
-    let(:project) { create(:project, :repository) }
+    let(:project) { create(:project, :repository, :legacy_storage) }
     let(:gitlab_shell) { Gitlab::Shell.new }
     let(:project_storage) { project.send(:storage) }
 
@@ -2719,6 +2734,8 @@ describe Project do
     let(:project) { create(:project, :repository, skip_disk_validation: true) }
     let(:gitlab_shell) { Gitlab::Shell.new }
     let(:hash) { Digest::SHA2.hexdigest(project.id.to_s) }
+    let(:hashed_prefix) { File.join('@hashed', hash[0..1], hash[2..3]) }
+    let(:hashed_path) { File.join(hashed_prefix, hash) }
 
     before do
       stub_application_setting(hashed_storage_enabled: true)
@@ -2744,14 +2761,12 @@ describe Project do
 
     describe '#base_dir' do
       it 'returns base_dir based on hash of project id' do
-        expect(project.base_dir).to eq("@hashed/#{hash[0..1]}/#{hash[2..3]}")
+        expect(project.base_dir).to eq(hashed_prefix)
       end
     end
 
     describe '#disk_path' do
       it 'returns disk_path based on hash of project id' do
-        hashed_path = "@hashed/#{hash[0..1]}/#{hash[2..3]}/#{hash}"
-
         expect(project.disk_path).to eq(hashed_path)
       end
     end
@@ -2760,7 +2775,7 @@ describe Project do
       it 'delegates to gitlab_shell to ensure namespace is created' do
         allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
 
-        expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, "@hashed/#{hash[0..1]}/#{hash[2..3]}")
+        expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, hashed_prefix)
 
         project.ensure_storage_path_exists
       end

spec/models/user_spec.rb

@@ -1586,14 +1586,37 @@ describe User do
   describe '#authorized_groups' do
     let!(:user) { create(:user) }
     let!(:private_group) { create(:group) }
+    let!(:child_group) { create(:group, parent: private_group) }
+
+    let!(:project_group) { create(:group) }
+    let!(:project) { create(:project, group: project_group) }
 
     before do
       private_group.add_user(user, Gitlab::Access::MASTER)
+      project.add_master(user)
     end
 
     subject { user.authorized_groups }
 
-    it { is_expected.to eq([private_group]) }
+    it { is_expected.to contain_exactly private_group, project_group }
+  end
+
+  describe '#membership_groups' do
+    let!(:user) { create(:user) }
+    let!(:parent_group) { create(:group) }
+    let!(:child_group) { create(:group, parent: parent_group) }
+
+    before do
+      parent_group.add_user(user, Gitlab::Access::MASTER)
+    end
+
+    subject { user.membership_groups }
+
+    if Group.supports_nested_groups?
+      it { is_expected.to contain_exactly parent_group, child_group }
+    else
+      it { is_expected.to contain_exactly parent_group }
+    end
   end
 
   describe '#authorized_projects', :delete do

spec/requests/api/internal_spec.rb

@@ -366,20 +366,9 @@ describe API::Internal do
           end
         end
 
-        context 'project as /namespace/project' do
-          it do
-            push(key, project_with_repo_path('/' + project.full_path))
-
-            expect(response).to have_gitlab_http_status(200)
-            expect(json_response["status"]).to be_truthy
-            expect(json_response["repository_path"]).to eq(project.repository.path_to_repo)
-            expect(json_response["gl_repository"]).to eq("project-#{project.id}")
-          end
-        end
-
         context 'project as namespace/project' do
           it do
-            push(key, project_with_repo_path(project.full_path))
+            push(key, project)
 
             expect(response).to have_gitlab_http_status(200)
             expect(json_response["status"]).to be_truthy
@@ -496,8 +485,10 @@ describe API::Internal do
     end
 
     context 'project does not exist' do
-      it do
-        pull(key, project_with_repo_path('gitlab/notexist'))
+      it 'returns a 200 response with status: false' do
+        project.destroy
+
+        pull(key, project)
 
         expect(response).to have_gitlab_http_status(200)
         expect(json_response["status"]).to be_falsey
@@ -569,6 +560,7 @@ describe API::Internal do
     end
 
     context 'the project path was changed' do
+      let(:project) { create(:project, :repository, :legacy_storage) }
       let!(:old_path_to_repo) { project.repository.path_to_repo }
       let!(:repository) { project.repository }
 
@@ -858,9 +850,14 @@ describe API::Internal do
     end
   end
 
-  def project_with_repo_path(path)
-    double().tap do |fake_project|
-      allow(fake_project).to receive_message_chain('repository.path_to_repo' => path)
+  def gl_repository_for(project_or_wiki)
+    case project_or_wiki
+    when ProjectWiki
+      project_or_wiki.project.gl_repository(is_wiki: true)
+    when Project
+      project_or_wiki.gl_repository(is_wiki: false)
+    else
+      nil
     end
   end
 
@@ -868,18 +865,8 @@ describe API::Internal do
     post(
       api("/internal/allowed"),
       key_id: key.id,
-      project: project.repository.path_to_repo,
-      action: 'git-upload-pack',
-      secret_token: secret_token,
-      protocol: protocol
-    )
-  end
-
-  def pull_with_path(key, path_to_repo, protocol = 'ssh')
-    post(
-      api("/internal/allowed"),
-      key_id: key.id,
-      project: path_to_repo,
+      project: project.full_path,
+      gl_repository: gl_repository_for(project),
       action: 'git-upload-pack',
       secret_token: secret_token,
       protocol: protocol
@@ -891,20 +878,8 @@ describe API::Internal do
       api("/internal/allowed"),
       changes: 'd14d6c0abdd253381df51a723d58691b2ee1ab08 570e7b2abdd848b95f2f578043fc23bd6f6fd24d refs/heads/master',
       key_id: key.id,
-      project: project.repository.path_to_repo,
-      action: 'git-receive-pack',
-      secret_token: secret_token,
-      protocol: protocol,
-      env: env
-    )
-  end
-
-  def push_with_path(key, path_to_repo, protocol = 'ssh', env: nil)
-    post(
-      api("/internal/allowed"),
-      changes: 'd14d6c0abdd253381df51a723d58691b2ee1ab08 570e7b2abdd848b95f2f578043fc23bd6f6fd24d refs/heads/master',
-      key_id: key.id,
-      project: path_to_repo,
+      project: project.full_path,
+      gl_repository: gl_repository_for(project),
       action: 'git-receive-pack',
       secret_token: secret_token,
       protocol: protocol,
@@ -917,7 +892,8 @@ describe API::Internal do
       api("/internal/allowed"),
       ref: 'master',
       key_id: key.id,
-      project: project.repository.path_to_repo,
+      project: project.full_path,
+      gl_repository: gl_repository_for(project),
       action: 'git-upload-archive',
       secret_token: secret_token,
       protocol: 'ssh'
@@ -929,7 +905,7 @@ describe API::Internal do
       api("/internal/lfs_authenticate"),
       key_id: key_id,
       secret_token: secret_token,
-      project: project.repository.path_to_repo
+      project: project.full_path
     )
   end
 end

spec/requests/api/projects_spec.rb

@@ -460,7 +460,7 @@ describe API::Projects do
       expect(response).to have_gitlab_http_status(201)
 
       project.each_pair do |k, v|
-        next if %i[has_external_issue_tracker issues_enabled merge_requests_enabled wiki_enabled].include?(k)
+        next if %i[has_external_issue_tracker issues_enabled merge_requests_enabled wiki_enabled storage_version].include?(k)
 
         expect(json_response[k.to_s]).to eq(v)
       end
@@ -622,12 +622,8 @@ describe API::Projects do
   end
 
   describe 'POST /projects/user/:id' do
-    before do
-      expect(project).to be_persisted
-    end
-
     it 'creates new project without path but with name and return 201' do
-      expect { post api("/projects/user/#{user.id}", admin), name: 'Foo Project' }.to change {Project.count}.by(1)
+      expect { post api("/projects/user/#{user.id}", admin), name: 'Foo Project' }.to change { Project.count }.by(1)
       expect(response).to have_gitlab_http_status(201)
 
       project = Project.last
@@ -666,8 +662,9 @@ describe API::Projects do
       post api("/projects/user/#{user.id}", admin), project
 
       expect(response).to have_gitlab_http_status(201)
+
       project.each_pair do |k, v|
-        next if %i[has_external_issue_tracker path].include?(k)
+        next if %i[has_external_issue_tracker path storage_version].include?(k)
 
         expect(json_response[k.to_s]).to eq(v)
       end

spec/requests/api/v3/projects_spec.rb

@@ -401,7 +401,7 @@ describe API::V3::Projects do
       post v3_api('/projects', user), project
 
       project.each_pair do |k, v|
-        next if %i[has_external_issue_tracker issues_enabled merge_requests_enabled wiki_enabled].include?(k)
+        next if %i[storage_version has_external_issue_tracker issues_enabled merge_requests_enabled wiki_enabled].include?(k)
 
         expect(json_response[k.to_s]).to eq(v)
       end
@@ -545,7 +545,7 @@ describe API::V3::Projects do
 
       expect(response).to have_gitlab_http_status(201)
       project.each_pair do |k, v|
-        next if %i[has_external_issue_tracker path].include?(k)
+        next if %i[storage_version has_external_issue_tracker path].include?(k)
 
         expect(json_response[k.to_s]).to eq(v)
       end

spec/requests/git_http_spec.rb

@@ -163,7 +163,7 @@ describe 'Git HTTP requests' do
             download(path) do |response|
               json_body = ActiveSupport::JSON.decode(response.body)
 
-              expect(json_body['RepoPath']).to include(wiki.repository.full_path)
+              expect(json_body['RepoPath']).to include(wiki.repository.disk_path)
             end
           end
         end

spec/requests/openid_connect_spec.rb

@@ -65,10 +65,20 @@ describe 'OpenID Connect requests' do
         )
       end
 
-      let(:public_email) { build :email, email: 'public@example.com' }
-      let(:private_email) { build :email, email: 'private@example.com' }
+      let!(:public_email) { build :email, email: 'public@example.com' }
+      let!(:private_email) { build :email, email: 'private@example.com' }
 
-      it 'includes all user information' do
+      let!(:group1) { create :group, path: 'group1' }
+      let!(:group2) { create :group, path: 'group2' }
+      let!(:group3) { create :group, path: 'group3', parent: group2 }
+      let!(:group4) { create :group, path: 'group4', parent: group3 }
+
+      before do
+        group1.add_user(user, GroupMember::OWNER)
+        group3.add_user(user, Gitlab::Access::DEVELOPER)
+      end
+
+      it 'includes all user information and group memberships' do
         request_user_info
 
         expect(json_response).to eq({
@@ -79,7 +89,13 @@ describe 'OpenID Connect requests' do
           'email_verified' => true,
           'website'        => 'https://example.com',
           'profile'        => 'http://localhost/alice',
-          'picture'        => "http://localhost/uploads/-/system/user/avatar/#{user.id}/dk.png"
+          'picture'        => "http://localhost/uploads/-/system/user/avatar/#{user.id}/dk.png",
+          'groups'         =>
+            if Group.supports_nested_groups?
+              ['group1', 'group2/group3', 'group2/group3/group4']
+            else
+              ['group1', 'group2/group3']
+            end
         })
       end
     end

spec/services/groups/destroy_service_spec.rb

@@ -6,7 +6,7 @@ describe Groups::DestroyService do
   let!(:user)         { create(:user) }
   let!(:group)        { create(:group) }
   let!(:nested_group) { create(:group, parent: group) }
-  let!(:project)      { create(:project, namespace: group) }
+  let!(:project)      { create(:project, :legacy_storage, namespace: group) }
   let!(:notification_setting) { create(:notification_setting, source: group)}
   let(:gitlab_shell) { Gitlab::Shell.new }
   let(:remove_path)  { group.path + "+#{group.id}+deleted" }
@@ -141,7 +141,7 @@ describe Groups::DestroyService do
     end
 
     context 'legacy storage' do
-      let!(:project) { create(:project, :empty_repo, namespace: group) }
+      let!(:project) { create(:project, :legacy_storage, :empty_repo, namespace: group) }
 
       it 'removes repository' do
         expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey
@@ -149,7 +149,7 @@ describe Groups::DestroyService do
     end
 
     context 'hashed storage' do
-      let!(:project) { create(:project, :hashed, :empty_repo, namespace: group) }
+      let!(:project) { create(:project, :empty_repo, namespace: group) }
 
       it 'removes repository' do
         expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey

spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 describe Projects::HashedStorage::MigrateAttachmentsService do
   subject(:service) { described_class.new(project) }
-  let(:project) { create(:project) }
+  let(:project) { create(:project, :legacy_storage) }
   let(:legacy_storage) { Storage::LegacyProject.new(project) }
   let(:hashed_storage) { Storage::HashedProject.new(project) }
 

spec/services/projects/hashed_storage/migrate_repository_service_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 describe Projects::HashedStorage::MigrateRepositoryService do
   let(:gitlab_shell) { Gitlab::Shell.new }
-  let(:project) { create(:project, :repository, :wiki_repo) }
+  let(:project) { create(:project, :legacy_storage, :repository, :wiki_repo) }
   let(:service) { described_class.new(project) }
   let(:legacy_storage) { Storage::LegacyProject.new(project) }
   let(:hashed_storage) { Storage::HashedProject.new(project) }

spec/services/projects/hashed_storage_migration_service_spec.rb

 require 'spec_helper'
 
 describe Projects::HashedStorageMigrationService do
-  let(:project) { create(:project, :empty_repo, :wiki_repo) }
+  let(:project) { create(:project, :empty_repo, :wiki_repo, :legacy_storage) }
   subject(:service) { described_class.new(project) }
 
   describe '#execute' do

spec/services/projects/transfer_service_spec.rb

@@ -4,7 +4,7 @@ describe Projects::TransferService do
   let(:gitlab_shell) { Gitlab::Shell.new }
   let(:user) { create(:user) }
   let(:group) { create(:group) }
-  let(:project) { create(:project, :repository, namespace: user.namespace) }
+  let(:project) { create(:project, :repository, :legacy_storage, namespace: user.namespace) }
 
   context 'namespace -> namespace' do
     before do
@@ -214,7 +214,7 @@ describe Projects::TransferService do
   end
 
   context 'when hashed storage in use' do
-    let(:hashed_project) { create(:project, :repository, :hashed, namespace: user.namespace) }
+    let(:hashed_project) { create(:project, :repository, namespace: user.namespace) }
 
     before do
       group.add_owner(user)

spec/services/projects/update_service_spec.rb

@@ -150,6 +150,8 @@ describe Projects::UpdateService do
       let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] }
 
       context 'with legacy storage' do
+        let(:project) { create(:project, :legacy_storage, :repository, creator: user, namespace: user.namespace) }
+
         before do
           gitlab_shell.add_repository(repository_storage, "#{user.namespace.full_path}/existing")
         end

spec/services/users/destroy_service_spec.rb

@@ -173,7 +173,7 @@ describe Users::DestroyService do
       end
 
       context 'legacy storage' do
-        let!(:project) { create(:project, :empty_repo, namespace: user.namespace) }
+        let!(:project) { create(:project, :empty_repo, :legacy_storage, namespace: user.namespace) }
 
         it 'removes repository' do
           expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey
@@ -181,7 +181,7 @@ describe Users::DestroyService do
       end
 
       context 'hashed storage' do
-        let!(:project) { create(:project, :empty_repo, :hashed, namespace: user.namespace) }
+        let!(:project) { create(:project, :empty_repo, namespace: user.namespace) }
 
         it 'removes repository' do
           expect(gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey

spec/support/migrations_helpers.rb

@@ -25,14 +25,19 @@ module MigrationsHelpers
     clear_schema_cache!
 
     # Reset column information for the most offending classes **after** we
-    # migrated the schema up, otherwise, column information could be outdated
-    ActiveRecord::Base.descendants.each { |klass| klass.reset_column_information }
+    # migrated the schema up, otherwise, column information could be
+    # outdated. We have a separate method for this so we can override it in EE.
+    ActiveRecord::Base.descendants.each(&method(:reset_column_information))
 
     # Without that, we get errors because of missing attributes, e.g.
     # super: no superclass method `elasticsearch_indexing' for #<ApplicationSetting:0x00007f85628508d8>
     ApplicationSetting.define_attribute_methods
   end
 
+  def reset_column_information(klass)
+    klass.reset_column_information
+  end
+
   def previous_migration
     migrations.each_cons(2) do |previous, migration|
       break previous if migration.name == described_class.name

spec/uploaders/file_uploader_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 describe FileUploader do
   let(:group) { create(:group, name: 'awesome') }
-  let(:project) { create(:project, namespace: group, name: 'project') }
+  let(:project) { create(:project, :legacy_storage, namespace: group, name: 'project') }
   let(:uploader) { described_class.new(project) }
   let(:upload)  { double(model: project, path: 'secret/foo.jpg') }
 
@@ -16,12 +16,12 @@ describe FileUploader do
 
   shared_examples 'uses hashed storage' do
     context 'when rolled out attachments' do
+      let(:project) { build_stubbed(:project, namespace: group, name: 'project') }
+
       before do
         allow(project).to receive(:disk_path).and_return('ca/fe/fe/ed')
       end
 
-      let(:project) { build_stubbed(:project, :hashed, namespace: group, name: 'project') }
-
       it_behaves_like 'builds correct paths',
                       store_dir: %r{ca/fe/fe/ed/\h+},
                       absolute_path: %r{#{described_class.root}/ca/fe/fe/ed/secret/foo.jpg}

spec/workers/repository_fork_worker_spec.rb

@@ -68,7 +68,7 @@ describe RepositoryForkWorker do
     end
 
     it "handles bad fork" do
-      error_message = "Unable to fork project #{fork_project.id} for repository #{project.full_path} -> #{fork_project.full_path}"
+      error_message = "Unable to fork project #{fork_project.id} for repository #{project.disk_path} -> #{fork_project.disk_path}"
 
       expect_fork_repository.and_return(false)
 

spec/workers/storage_migrator_worker_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 describe StorageMigratorWorker do
   subject(:worker) { described_class.new }
-  let(:projects) { create_list(:project, 2) }
+  let(:projects) { create_list(:project, 2, :legacy_storage) }
 
   describe '#perform' do
     let(:ids) { projects.map(&:id) }

vendor/gitignore/Android.gitignore

@@ -54,3 +54,10 @@ google-services.json
 freeline.py
 freeline/
 freeline_project_description.json
+
+# fastlane
+fastlane/report.xml
+fastlane/Preview.html
+fastlane/screenshots
+fastlane/test_output
+fastlane/readme.md

vendor/gitignore/Dart.gitignore

 # See https://www.dartlang.org/tools/private-files.html
 
 # Files and directories created by pub
+.dart_tool/
 .packages
 .pub/
 build/