Automatic merge of gitlab-org/gitlab-ce master

GITLAB_WORKHORSE_VERSION

-8.3.1
+8.3.3

app/assets/javascripts/issue.js

@@ -16,7 +16,9 @@ export default class Issue {
     Issue.createMrDropdownWrap = document.querySelector('.create-mr-dropdown-wrap');
 
     Issue.initMergeRequests();
-    Issue.initRelatedBranches();
+    if (document.querySelector('#related-branches')) {
+      Issue.initRelatedBranches();
+    }
 
     this.closeButtons = $('a.btn-close');
     this.reopenButtons = $('a.btn-reopen');

app/assets/javascripts/pdf/index.vue

@@ -28,7 +28,7 @@ export default {
   },
   watch: { pdf: 'load' },
   mounted() {
-    pdfjsLib.PDFJS.workerSrc = workerSrc;
+    pdfjsLib.GlobalWorkerOptions.workerSrc = workerSrc;
     if (this.hasPDF) this.load();
   },
   methods: {

app/controllers/projects/graphs_controller.rb

@@ -46,12 +46,8 @@ class Projects::GraphsController < Projects::ApplicationController
 
   def get_languages
     @languages =
-      if @project.repository_languages.present?
-        @project.repository_languages.map do |lang|
-          { value: lang.share, label: lang.name, color: lang.color, highlight: lang.color }
-        end
-      else
-        @project.repository.languages
+      ::Projects::RepositoryLanguagesService.new(@project, current_user).execute.map do |lang|
+        { value: lang.share, label: lang.name, color: lang.color, highlight: lang.color }
       end
   end
 

app/controllers/projects/issues_controller.rb

@@ -39,6 +39,7 @@ class Projects::IssuesController < Projects::ApplicationController
   before_action :authorize_create_merge_request_from!, only: [:create_merge_request]
 
   before_action :authorize_import_issues!, only: [:import_csv]
+  before_action :authorize_download_code!, only: [:related_branches]
 
   before_action :set_suggested_issues_feature_flags, only: [:new]
 

app/policies/project_policy.rb

@@ -177,7 +177,6 @@ class ProjectPolicy < BasePolicy
     enable :read_cycle_analytics
     enable :award_emoji
     enable :read_pages_content
-    enable :read_release
   end
 
   # These abilities are not allowed to admins that are not members of the project,
@@ -204,6 +203,7 @@ class ProjectPolicy < BasePolicy
     enable :read_deployment
     enable :read_merge_request
     enable :read_sentry_issue
+    enable :read_release
   end
 
   # We define `:public_user_access` separately because there are cases in gitlab-ee

app/services/projects/detect_repository_languages_service.rb

@@ -2,7 +2,7 @@
 
 module Projects
   class DetectRepositoryLanguagesService < BaseService
-    attr_reader :detected_repository_languages, :programming_languages
+    attr_reader :programming_languages
 
     # rubocop: disable CodeReuse/ActiveRecord
     def execute
@@ -25,6 +25,8 @@ module Projects
           RepositoryLanguage.table_name,
           detection.insertions(matching_programming_languages)
         )
+
+        set_detected_repository_languages
       end
 
       project.repository_languages.reload
@@ -56,5 +58,11 @@ module Projects
       retry
     end
     # rubocop: enable CodeReuse/ActiveRecord
+
+    def set_detected_repository_languages
+      return if project.detected_repository_languages?
+
+      project.update_column(:detected_repository_languages, true)
+    end
   end
 end

app/services/projects/repository_languages_service.rb

+# frozen_string_literal: true
+
+module Projects
+  class RepositoryLanguagesService < BaseService
+    def execute
+      perform_language_detection unless project.detected_repository_languages?
+      persisted_repository_languages
+    end
+
+    private
+
+    def perform_language_detection
+      if persisted_repository_languages.blank?
+        ::DetectRepositoryLanguagesWorker.perform_async(project.id, current_user.id)
+      else
+        project.update_column(:detected_repository_languages, true)
+      end
+    end
+
+    def persisted_repository_languages
+      project.repository_languages
+    end
+  end
+end

app/views/projects/issues/show.html.haml

@@ -90,8 +90,9 @@
     #merge-requests{ data: { url: referenced_merge_requests_project_issue_path(@project, @issue) } }
       // This element is filled in using JavaScript.
 
-    #related-branches{ data: { url: related_branches_project_issue_path(@project, @issue) } }
-      // This element is filled in using JavaScript.
+    - if can?(current_user, :download_code, @project)
+      #related-branches{ data: { url: related_branches_project_issue_path(@project, @issue) } }
+        // This element is filled in using JavaScript.
 
   .content-block.emoji-block.emoji-block-sticky
     .row

app/views/projects/merge_requests/conflicts/_submit_form.html.haml

@@ -6,7 +6,7 @@
   .form-group.row
     .col-md-4
       %h4= _('Resolve conflicts on source branch')
-      .resolve-info
+      .resolve-info{ "v-pre": true }
         = translation.html_safe
     .col-md-8
       %label.label-bold{ "for" => "commit-message" }

changelogs/unreleased/disallow-guests-to-access-releases.yml

+---
+title: Disallow guest users from accessing Releases
+merge_request:
+author:
+type: security

changelogs/unreleased/security-55503-fix-pdf-js-vulnerability.yml

+---
+title: Fix PDF.js vulnerability
+merge_request:
+author:
+type: security

changelogs/unreleased/security-56224.yml

+---
+title: Hide "related branches" when user does not have permission
+merge_request:
+author:
+type: security

changelogs/unreleased/security-56927-xss-resolve-conflicts-branch-name.yml

+---
+title: Fix XSS in resolve conflicts form
+merge_request:
+author:
+type: security

changelogs/unreleased/security-exif-migration.yml

+---
+title: Added rake task for removing EXIF data from existing uploads.
+merge_request:
+author:
+type: security

changelogs/unreleased/security-id-potential-denial-languages.yml

+---
+title: Return cached languages if they've been detected before
+merge_request:
+author:
+type: security

changelogs/unreleased/use-untrusted-regexp.yml

+---
+title: Use UntrustedRegexp for matching refs policy
+merge_request:
+author:
+type: security

db/migrate/20190312071108_add_detected_repository_languages_to_projects.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddDetectedRepositoryLanguagesToProjects < ActiveRecord::Migration[5.0]
+  DOWNTIME = false
+
+  def change
+    add_column :projects, :detected_repository_languages, :boolean
+  end
+end

doc/administration/raketasks/uploads/sanitize.md

+# Uploads Sanitize tasks
+
+## Requirements
+
+You need `exiftool` installed on your system. If you installed GitLab:
+
+-   Using the Omnibus package, you're all set.
+-   From source, make sure `exiftool` is installed:
+
+    ```sh
+    # Debian/Ubuntu
+    sudo apt-get install libimage-exiftool-perl
+
+    # RHEL/CentOS
+    sudo yum install perl-Image-ExifTool
+    ```
+
+## Remove EXIF data from existing uploads
+
+Since 11.9 EXIF data are automatically stripped from JPG or TIFF image uploads.
+Because EXIF data may contain sensitive information (e.g. GPS location), you
+can remove EXIF data also from existing images which were uploaded before
+with the following command:
+
+```bash
+sudo RAILS_ENV=production -u git -H bundle exec rake gitlab:uploads:sanitize:remove_exif
+```
+
+This command by default runs in dry mode and it doesn't remove EXIF data. It can be used for
+checking if (and how many) images should be sanitized.
+
+The rake task accepts following parameters.
+
+Parameter | Type | Description
+--------- | ---- | -----------
+`start_id` | integer | Only uploads with equal or greater ID will be processed
+`stop_id` | integer | Only uploads with equal or smaller ID will be processed
+`dry_run` | boolean | Do not remove EXIF data, only check if EXIF data are present or not, default: true
+`sleep_time` | float | Pause for number of seconds after processing each image, default: 0.3 seconds
+
+If you have too many uploads, you can speed up sanitization by setting
+`sleep_time` to a lower value or by running multiple rake tasks in parallel,
+each with a separate range of upload IDs (by setting `start_id` and `stop_id`).
+
+To run the command without dry mode and remove EXIF data from all uploads, you can use:
+
+```bash
+sudo RAILS_ENV=production -u git -H bundle exec rake gitlab:uploads:sanitize:remove_exif[,,false,] 2>&1 | tee exif.log
+```
+
+To run the command without dry mode on uploads with ID between 100 and 5000 and pause for 0.1 second, you can use:
+
+```bash
+sudo RAILS_ENV=production -u git -H bundle exec rake gitlab:uploads:sanitize:remove_exif[100,5000,false,0.1] 2>&1 | tee exif.log
+```
+
+Because the output of commands will be probably long, the output is written also into exif.log file.
+
+If sanitization fails for an upload, an error message should be in the output of the rake task (typical reasons may
+be that the file is missing in the storage or it's not a valid image). Please
+[report](https://gitlab.com/gitlab-org/gitlab-ce/issues/new) any issues at `gitlab.com` and use
+prefix 'EXIF' in issue title with the error output and (if possible) the image.

doc/ci/yaml/README.md

@@ -351,6 +351,19 @@ job:
     - branches
 ```
 
+Pattern matching is case-sensitive by default. Use `i` flag modifier, like
+`/pattern/i` to make a pattern case-insensitive:
+
+```yaml
+job:
+  # use regexp
+  only:
+    - /^issue-.*$/i
+  # use special keyword
+  except:
+    - branches
+```
+
 In this example, `job` will run only for refs that are tagged, or if a build is
 explicitly requested via an API trigger or a [Pipeline Schedule][schedules]:
 

doc/raketasks/README.md

@@ -15,3 +15,4 @@ comments: false
 - [Import](import.md) of git repositories in bulk
 - [Rebuild authorized_keys file](http://docs.gitlab.com/ce/raketasks/maintenance.html#rebuild-authorized_keys-file) task for administrators
 - [Migrate Uploads](../administration/raketasks/uploads/migrate.md)
+- [Sanitize Uploads](../administration/raketasks/uploads/sanitize.md)

lib/api/projects.rb

@@ -373,11 +373,9 @@ module API
 
       desc 'Get languages in project repository'
       get ':id/languages' do
-        if user_project.repository_languages.present?
-          user_project.repository_languages.map { |l| [l.name, l.share] }.to_h
-        else
-          user_project.repository.languages.map { |language| language.values_at(:label, :value) }.to_h
-        end
+        ::Projects::RepositoryLanguagesService
+          .new(user_project, current_user)
+          .execute.map { |lang| [lang.name, lang.share] }.to_h
       end
 
       desc 'Remove a project'

lib/gitlab/ci/build/policy/refs.rb

@@ -35,8 +35,8 @@ module Gitlab
             # patterns can be matched only when branch or tag is used
             # the pattern matching does not work for merge requests pipelines
             if pipeline.branch? || pipeline.tag?
-              if pattern.first == "/" && pattern.last == "/"
-                Regexp.new(pattern[1...-1]) =~ pipeline.ref
+              if regexp = Gitlab::UntrustedRegexp::RubySyntax.fabricate(pattern)
+                regexp.match?(pipeline.ref)
               else
                 pattern == pipeline.ref
               end

lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb

@@ -13,13 +13,13 @@ module Gitlab
             def initialize(regexp)
               @value = regexp
 
-              unless Gitlab::UntrustedRegexp.valid?(@value)
+              unless Gitlab::UntrustedRegexp::RubySyntax.valid?(@value)
                 raise Lexer::SyntaxError, 'Invalid regular expression!'
               end
             end
 
             def evaluate(variables = {})
-              Gitlab::UntrustedRegexp.fabricate(@value)
+              Gitlab::UntrustedRegexp::RubySyntax.fabricate!(@value)
             rescue RegexpError
               raise Expression::RuntimeError, 'Invalid regular expression!'
             end

lib/gitlab/config/entry/legacy_validation_helpers.rb

@@ -45,17 +45,15 @@ module Gitlab
         end
 
         def validate_regexp(value)
-          !value.nil? && Regexp.new(value.to_s) && true
-        rescue RegexpError, TypeError
-          false
+          Gitlab::UntrustedRegexp::RubySyntax.valid?(value)
         end
 
         def validate_string_or_regexp(value)
           return true if value.is_a?(Symbol)
           return false unless value.is_a?(String)
 
-          if value.first == '/' && value.last == '/'
-            validate_regexp(value[1...-1])
+          if Gitlab::UntrustedRegexp::RubySyntax.matches_syntax?(value)
+            validate_regexp(value)
           else
             true
           end

lib/gitlab/config/entry/validators.rb

@@ -120,17 +120,13 @@ module Gitlab
 
           private
 
-          def look_like_regexp?(value)
-            value.is_a?(String) && value.start_with?('/') &&
-              value.end_with?('/')
+          def matches_syntax?(value)
+            Gitlab::UntrustedRegexp::RubySyntax.matches_syntax?(value)
           end
 
           def validate_regexp(value)
-            look_like_regexp?(value) &&
-              Regexp.new(value.to_s[1...-1]) &&
-              true
-          rescue RegexpError
-            false
+            matches_syntax?(value) &&
+              Gitlab::UntrustedRegexp::RubySyntax.valid?(value)
           end
         end
 
@@ -149,7 +145,7 @@ module Gitlab
 
           def validate_string_or_regexp(value)
             return false unless value.is_a?(String)
-            return validate_regexp(value) if look_like_regexp?(value)
+            return validate_regexp(value) if matches_syntax?(value)
 
             true
           end

lib/gitlab/import_export/import_export.yml

@@ -129,6 +129,7 @@ excluded_attributes:
     - :mirror_last_update_at
     - :mirror_last_successful_update_at
     - :bfg_object_map
+    - :detected_repository_languages
     - :tag_list
   namespaces:
     - :runners_token

lib/gitlab/sanitizers/exif.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Sanitizers
+    class Exif
+      # these tags are not removed from the image
+      WHITELISTED_TAGS = %w(
+        ResolutionUnit
+        XResolution
+        YResolution
+        YCbCrSubSampling
+        YCbCrPositioning
+        BitsPerSample
+        ImageHeight
+        ImageWidth
+        ImageSize
+        Copyright
+        CopyrightNotice
+        Orientation
+      ).freeze
+
+      # these tags are common in exiftool output, these
+      # do not contain any sensitive information, but
+      # we don't need to preserve them when removing
+      # exif tags
+      IGNORED_TAGS = %w(
+        ColorComponents
+        EncodingProcess
+        ExifByteOrder
+        ExifToolVersion
+        JFIFVersion
+        Directory
+        FileAccessDate
+        FileInodeChangeDate
+        FileModifyDate
+        FileName
+        FilePermissions
+        FileSize
+        SourceFile
+        Megapixels
+        FileType
+        FileTypeExtension
+        MIMEType
+      ).freeze
+
+      ALLOWED_TAGS = WHITELISTED_TAGS + IGNORED_TAGS
+      EXCLUDE_PARAMS = WHITELISTED_TAGS.map { |tag| "-#{tag}" }
+
+      attr_reader :logger
+
+      def initialize(logger: Rails.logger)
+        @logger = logger
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def batch_clean(start_id: nil, stop_id: nil, dry_run: true, sleep_time: nil)
+        relation = Upload.where('lower(path) like ? or lower(path) like ? or lower(path) like ?',
+                                '%.jpg', '%.jpeg', '%.tiff')
+
+        logger.info "running in dry run mode, no images will be rewritten" if dry_run
+
+        find_params = {
+          start: start_id.present? ? start_id.to_i : nil,
+          finish: stop_id.present? ? stop_id.to_i : Upload.last&.id
+        }
+
+        relation.find_each(find_params) do |upload|
+          clean(upload.build_uploader, dry_run: dry_run)
+          sleep sleep_time if sleep_time
+        rescue => err
+          logger.error "failed to sanitize #{upload_ref(upload)}: #{err.message}"
+          logger.debug err.backtrace.join("\n ")
+        end
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      def clean(uploader, dry_run: true)
+        Dir.mktmpdir('gitlab-exif') do |tmpdir|
+          src_path = fetch_upload_to_file(uploader, tmpdir)
+
+          to_remove = extra_tags(src_path)
+
+          if to_remove.empty?
+            logger.info "#{upload_ref(uploader.upload)}: only whitelisted tags present, skipping"
+            break
+          end
+
+          logger.info "#{upload_ref(uploader.upload)}: found exif tags to remove: #{to_remove}"
+
+          break if dry_run
+
+          remove_and_store(tmpdir, src_path, uploader)
+        end
+      end
+
+      def extra_tags(path)
+        exif_tags(path).keys - ALLOWED_TAGS
+      end
+
+      private
+
+      def remove_and_store(tmpdir, src_path, uploader)
+        exec_remove_exif!(src_path)
+        logger.info "#{upload_ref(uploader.upload)}: exif removed, storing"
+        File.open(src_path, 'r') { |f| uploader.store!(f) }
+      end
+
+      def exec_remove_exif!(path)
+        # IPTC and XMP-iptcExt groups may keep copyright information so
+        # we always preserve them
+        cmd = ["exiftool", "-all=", "-tagsFromFile", "@", *EXCLUDE_PARAMS, "--IPTC:all", "--XMP-iptcExt:all", path]
+        output, status = Gitlab::Popen.popen(cmd)
+
+        if status != 0
+          raise "exiftool return code is #{status}: #{output}"
+        end
+
+        if File.size(path) == 0
+          raise "size of file is 0"
+        end
+
+        # exiftool creates backup of the original file in filename_original
+        old_path = "#{path}_original"
+        if File.size(path) == File.size(old_path)
+          raise "size of sanitized file is same as original size"
+        end
+      end
+
+      def fetch_upload_to_file(uploader, dir)
+        # upload is stored into the file with the original name - this filename
+        # is used by carrierwave when storing the file back to the storage
+        filename = File.join(dir, uploader.filename)
+
+        File.open(filename, 'w') do |file|
+          file.binmode
+          file.write uploader.read
+        end
+
+        filename
+      end
+
+      def upload_ref(upload)
+        "#{upload.id}:#{upload.path}"
+      end
+
+      def exif_tags(path)
+        cmd = ["exiftool", "-all", "-j", "-sort", "--IPTC:all", "--XMP-iptcExt:all", path]
+        output, status = Gitlab::Popen.popen(cmd)
+
+        raise "failed to get exif tags: #{output}" if status != 0
+
+        JSON.parse(output).first
+      end
+    end
+  end
+end

lib/gitlab/untrusted_regexp.rb

@@ -35,6 +35,10 @@ module Gitlab
       matches
     end
 
+    def match?(text)
+      text.present? && scan(text).present?
+    end
+
     def replace(text, rewrite)
       RE2.Replace(text, regexp, rewrite)
     end
@@ -43,37 +47,6 @@ module Gitlab
       self.source == other.source
     end
 
-    # Handles regular expressions with the preferred RE2 library where possible
-    # via UntustedRegex. Falls back to Ruby's built-in regular expression library
-    # when the syntax would be invalid in RE2.
-    #
-    # One difference between these is `(?m)` multi-line mode. Ruby regex enables
-    # this by default, but also handles `^` and `$` differently.
-    # See: https://www.regular-expressions.info/modifiers.html
-    def self.with_fallback(pattern, multiline: false)
-      UntrustedRegexp.new(pattern, multiline: multiline)
-    rescue RegexpError
-      Regexp.new(pattern)
-    end
-
-    def self.valid?(pattern)
-      !!self.fabricate(pattern)
-    rescue RegexpError
-      false
-    end
-
-    def self.fabricate(pattern)
-      matches = pattern.match(%r{^/(?<regexp>.+)/(?<flags>[ismU]*)$})
-
-      raise RegexpError, 'Invalid regular expression!' if matches.nil?
-
-      expression = matches[:regexp]
-      flags = matches[:flags]
-      expression.prepend("(?#{flags})") if flags.present?
-
-      self.new(expression, multiline: false)
-    end
-
     private
 
     attr_reader :regexp

lib/gitlab/untrusted_regexp/ruby_syntax.rb

+# frozen_string_literal: true
+
+module Gitlab
+  class UntrustedRegexp
+    # This class implements support for Ruby syntax of regexps
+    # and converts that to RE2 representation:
+    # /<regexp>/<flags>
+    class RubySyntax
+      PATTERN = %r{^/(?<regexp>.+)/(?<flags>[ismU]*)$}.freeze
+
+      # Checks if pattern matches a regexp pattern
+      # but does not enforce it's validity
+      def self.matches_syntax?(pattern)
+        pattern.is_a?(String) && pattern.match(PATTERN).present?
+      end
+
+      # The regexp can match the pattern `/.../`, but may not be fabricatable:
+      # it can be invalid or incomplete: `/match ( string/`
+      def self.valid?(pattern)
+        !!self.fabricate(pattern)
+      end
+
+      def self.fabricate(pattern)
+        self.fabricate!(pattern)
+      rescue RegexpError
+        nil
+      end
+
+      def self.fabricate!(pattern)
+        raise RegexpError, 'Pattern is not string!' unless pattern.is_a?(String)
+
+        matches = pattern.match(PATTERN)
+        raise RegexpError, 'Invalid regular expression!' if matches.nil?
+
+        expression = matches[:regexp]
+        flags = matches[:flags]
+        expression.prepend("(?#{flags})") if flags.present?
+
+        UntrustedRegexp.new(expression, multiline: false)
+      end
+    end
+  end
+end

lib/tasks/gitlab/uploads/sanitize.rake

+namespace :gitlab do
+  namespace :uploads do
+    namespace :sanitize do
+      desc 'GitLab | Uploads | Remove EXIF from images.'
+      task :remove_exif, [:start_id, :stop_id, :dry_run, :sleep_time] => :environment do |task, args|
+        args.with_defaults(dry_run: 'true')
+        args.with_defaults(sleep_time: 0.3)
+
+        logger = Logger.new(STDOUT)
+
+        sanitizer = Gitlab::Sanitizers::Exif.new(logger: logger)
+        sanitizer.batch_clean(start_id: args.start_id, stop_id: args.stop_id,
+                              dry_run: args.dry_run != 'false',
+                              sleep_time: args.sleep_time.to_f)
+      end
+    end
+  end
+end

spec/controllers/projects/graphs_controller_spec.rb

@@ -27,6 +27,7 @@ describe Projects::GraphsController do
 
   describe 'charts' do
     context 'when languages were previously detected' do
+      let(:project) { create(:project, :repository, detected_repository_languages: true) }
       let!(:repository_language) { create(:repository_language, project: project) }
 
       it 'sets the languages properly' do

spec/features/issues/user_creates_branch_and_merge_request_spec.rb

 require 'rails_helper'
 
 describe 'User creates branch and merge request on issue page', :js do
+  let(:membership_level) { :developer }
   let(:user) { create(:user) }
   let!(:project) { create(:project, :repository) }
   let(:issue) { create(:issue, project: project, title: 'Cherry-Coloured Funk') }
@@ -17,7 +18,7 @@ describe 'User creates branch and merge request on issue page', :js do
 
   context 'when signed in' do
     before do
-      project.add_developer(user)
+      project.add_user(user, membership_level)
 
       sign_in(user)
     end
@@ -167,6 +168,39 @@ describe 'User creates branch and merge request on issue page', :js do
         expect(page).not_to have_css('.create-mr-dropdown-wrap')
       end
     end
+
+    context 'when related branch exists' do
+      let!(:project) { create(:project, :repository, :private) }
+      let(:branch_name) { "#{issue.iid}-foo" }
+
+      before do
+        project.repository.create_branch(branch_name, 'master')
+
+        visit project_issue_path(project, issue)
+      end
+
+      context 'when user is developer' do
+        it 'shows related branches' do
+          expect(page).to have_css('#related-branches')
+
+          wait_for_requests
+
+          expect(page).to have_content(branch_name)
+        end
+      end
+
+      context 'when user is guest' do
+        let(:membership_level) { :guest }
+
+        it 'does not show related branches' do
+          expect(page).not_to have_css('#related-branches')
+
+          wait_for_requests
+
+          expect(page).not_to have_content(branch_name)
+        end
+      end
+    end
   end
 
   private

spec/features/merge_request/user_resolves_conflicts_spec.rb

@@ -164,6 +164,21 @@ describe 'Merge request > User resolves conflicts', :js do
         expect(page).to have_content('Gregor Samsa woke from troubled dreams')
       end
     end
+
+    context "with malicious branch name" do
+      let(:bad_branch_name) { "malicious-branch-{{toString.constructor('alert(/xss/)')()}}" }
+      let(:branch) { project.repository.create_branch(bad_branch_name, 'conflict-resolvable') }
+      let(:merge_request) { create_merge_request(branch.name) }
+
+      before do
+        visit project_merge_request_path(project, merge_request)
+        click_link('conflicts', href: %r{/conflicts\Z})
+      end
+
+      it "renders bad name without xss issues" do
+        expect(find('.resolve-conflicts-form .resolve-info')).to have_content(bad_branch_name)
+      end
+    end
   end
 
   UNRESOLVABLE_CONFLICTS = {

spec/features/projects/graph_spec.rb

@@ -6,6 +6,8 @@ describe 'Project Graph', :js do
   let(:branch_name) { 'master' }
 
   before do
+    ::Projects::DetectRepositoryLanguagesService.new(project, user).execute
+
     project.add_maintainer(user)
 
     sign_in(user)

spec/javascripts/pdf/index_spec.js

 import Vue from 'vue';
-import { PDFJS } from 'vendor/pdf';
+import { GlobalWorkerOptions } from 'vendor/pdf';
 import workerSrc from 'vendor/pdf.worker.min';
 
 import PDFLab from '~/pdf/index.vue';
 import pdf from '../fixtures/blob/pdf/test.pdf';
 
-PDFJS.workerSrc = workerSrc;
+GlobalWorkerOptions.workerSrc = workerSrc;
 const Component = Vue.extend(PDFLab);
 
 describe('PDF component', () => {

spec/javascripts/pdf/page_spec.js

@@ -12,7 +12,7 @@ describe('Page component', () => {
   let testPage;
 
   beforeEach(done => {
-    pdfjsLib.PDFJS.workerSrc = workerSrc;
+    pdfjsLib.GlobalWorkerOptions.workerSrc = workerSrc;
     pdfjsLib
       .getDocument(testPDF)
       .then(pdf => pdf.getPage(1))

spec/lib/gitlab/ci/build/policy/refs_spec.rb

@@ -92,10 +92,23 @@ describe Gitlab::Ci::Build::Policy::Refs do
           .to be_satisfied_by(pipeline)
       end
 
+      it 'is satisfied when case-insensitive regexp matches pipeline ref' do
+        expect(described_class.new(['/DOCS-.*/i']))
+          .to be_satisfied_by(pipeline)
+      end
+
       it 'is not satisfied when regexp does not match pipeline ref' do
         expect(described_class.new(['/fix-.*/']))
           .not_to be_satisfied_by(pipeline)
       end
     end
+
+    context 'malicious regexp' do
+      let(:pipeline) { build_stubbed(:ci_pipeline, ref: malicious_text) }
+
+      subject { described_class.new([malicious_regexp_ruby]) }
+
+      include_examples 'malicious regexp'
+    end
   end
 end

spec/lib/gitlab/ci/pipeline/expression/lexeme/pattern_spec.rb

@@ -85,7 +85,7 @@ describe Gitlab::Ci::Pipeline::Expression::Lexeme::Pattern do
     end
 
     it 'raises error if evaluated regexp is not valid' do
-      allow(Gitlab::UntrustedRegexp).to receive(:valid?).and_return(true)
+      allow(Gitlab::UntrustedRegexp::RubySyntax).to receive(:valid?).and_return(true)
 
       regexp = described_class.new('/invalid ( .*/')
 

spec/lib/gitlab/ci/trace/stream_spec.rb

@@ -414,7 +414,7 @@ describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
 
       context 'malicious regexp' do
         let(:data) { malicious_text }
-        let(:regex) { malicious_regexp }
+        let(:regex) { malicious_regexp_re2 }
 
         include_examples 'malicious regexp'
       end

spec/lib/gitlab/route_map_spec.rb

@@ -60,7 +60,7 @@ describe Gitlab::RouteMap do
 
       subject do
         map = described_class.new(<<-"MAP".strip_heredoc)
-        - source: '#{malicious_regexp}'
+        - source: '#{malicious_regexp_re2}'
           public: '/'
         MAP
 

spec/lib/gitlab/sanitizers/exif_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Sanitizers::Exif do
+  let(:sanitizer) { described_class.new }
+
+  describe '#batch_clean' do
+    context 'with image uploads' do
+      let!(:uploads) { create_list(:upload, 3, :with_file, :issuable_upload) }
+
+      it 'processes all uploads if range ID is not set' do
+        expect(sanitizer).to receive(:clean).exactly(3).times
+
+        sanitizer.batch_clean
+      end
+
+      it 'processes only uploads in the selected range' do
+        expect(sanitizer).to receive(:clean).once
+
+        sanitizer.batch_clean(start_id: uploads[1].id, stop_id: uploads[1].id)
+      end
+
+      it 'pauses if sleep_time is set' do
+        expect(sanitizer).to receive(:sleep).exactly(3).times.with(1.second)
+        expect(sanitizer).to receive(:clean).exactly(3).times
+
+        sanitizer.batch_clean(sleep_time: 1)
+      end
+    end
+
+    it 'filters only jpg/tiff images' do
+      create(:upload, path: 'filename.jpg')
+      create(:upload, path: 'filename.jpeg')
+      create(:upload, path: 'filename.JPG')
+      create(:upload, path: 'filename.tiff')
+      create(:upload, path: 'filename.TIFF')
+      create(:upload, path: 'filename.png')
+      create(:upload, path: 'filename.txt')
+
+      expect(sanitizer).to receive(:clean).exactly(5).times
+      sanitizer.batch_clean
+    end
+  end
+
+  describe '#clean' do
+    let(:uploader) { create(:upload, :with_file, :issuable_upload).build_uploader }
+
+    context "no dry run" do
+      it "removes exif from the image" do
+        uploader.store!(fixture_file_upload('spec/fixtures/rails_sample.jpg'))
+
+        original_upload = uploader.upload
+        expected_args = ["exiftool", "-all=", "-tagsFromFile", "@", *Gitlab::Sanitizers::Exif::EXCLUDE_PARAMS, "--IPTC:all", "--XMP-iptcExt:all", kind_of(String)]
+
+        expect(sanitizer).to receive(:extra_tags).and_return(["", 0])
+        expect(sanitizer).to receive(:exec_remove_exif!).once.and_call_original
+        expect(uploader).to receive(:store!).and_call_original
+        expect(Gitlab::Popen).to receive(:popen).with(expected_args) do |args|
+          File.write("#{args.last}_original", "foo") if args.last.start_with?(Dir.tmpdir)
+
+          [expected_args, 0]
+        end
+
+        sanitizer.clean(uploader, dry_run: false)
+
+        expect(uploader.upload.id).not_to eq(original_upload.id)
+        expect(uploader.upload.path).to eq(original_upload.path)
+      end
+
+      it "ignores image without exif" do
+        expected_args = ["exiftool", "-all", "-j", "-sort", "--IPTC:all", "--XMP-iptcExt:all", kind_of(String)]
+
+        expect(Gitlab::Popen).to receive(:popen).with(expected_args).and_return(["[{}]", 0])
+        expect(sanitizer).not_to receive(:exec_remove_exif!)
+        expect(uploader).not_to receive(:store!)
+
+        sanitizer.clean(uploader, dry_run: false)
+      end
+
+      it "raises an error if the exiftool fails with an error" do
+        expect(Gitlab::Popen).to receive(:popen).and_return(["error", 1])
+
+        expect { sanitizer.clean(uploader, dry_run: false) }.to raise_exception(RuntimeError, "failed to get exif tags: error")
+      end
+    end
+
+    context "dry run" do
+      it "doesn't change the image" do
+        expect(sanitizer).to receive(:extra_tags).and_return({ 'foo' => 'bar' })
+        expect(sanitizer).not_to receive(:exec_remove_exif!)
+        expect(uploader).not_to receive(:store!)
+
+        sanitizer.clean(uploader, dry_run: true)
+      end
+    end
+  end
+
+  describe "#extra_tags" do
+    it "returns a list of keys for exif file" do
+      tags = '[{
+                "DigitalSourceType": "some source",
+                "ImageHeight": 654
+              }]'
+
+      expect(Gitlab::Popen).to receive(:popen).and_return([tags, 0])
+
+      expect(sanitizer.extra_tags('filename')).not_to be_empty
+    end
+
+    it "returns an empty list for file with only whitelisted and ignored tags" do
+      tags = '[{
+                "ImageHeight": 654,
+                "Megapixels": 0.641
+              }]'
+
+      expect(Gitlab::Popen).to receive(:popen).and_return([tags, 0])
+
+      expect(sanitizer.extra_tags('some file')).to be_empty
+    end
+  end
+end

spec/lib/gitlab/untrusted_regexp/ruby_syntax_spec.rb

+require 'fast_spec_helper'
+require 'support/shared_examples/malicious_regexp_shared_examples'
+
+describe Gitlab::UntrustedRegexp::RubySyntax do
+  describe '.matches_syntax?' do
+    it 'returns true if regexp is valid' do
+      expect(described_class.matches_syntax?('/some .* thing/'))
+        .to be true
+    end
+
+    it 'returns true if regexp is invalid, but resembles regexp' do
+      expect(described_class.matches_syntax?('/some ( thing/'))
+        .to be true
+    end
+  end
+
+  describe '.valid?' do
+    it 'returns true if regexp is valid' do
+      expect(described_class.valid?('/some .* thing/'))
+        .to be true
+    end
+
+    it 'returns false if regexp is invalid' do
+      expect(described_class.valid?('/some ( thing/'))
+        .to be false
+    end
+  end
+
+  describe '.fabricate' do
+    context 'when regexp is valid' do
+      it 'fabricates regexp without flags' do
+        expect(described_class.fabricate('/some .* thing/')).not_to be_nil
+      end
+    end
+
+    context 'when regexp is a raw pattern' do
+      it 'returns error' do
+        expect(described_class.fabricate('some .* thing')).to be_nil
+      end
+    end
+  end
+
+  describe '.fabricate!' do
+    context 'when regexp is using /regexp/ scheme with flags' do
+      it 'fabricates regexp with a single flag' do
+        regexp = described_class.fabricate!('/something/i')
+
+        expect(regexp).to eq Gitlab::UntrustedRegexp.new('(?i)something')
+        expect(regexp.scan('SOMETHING')).to be_one
+      end
+
+      it 'fabricates regexp with multiple flags' do
+        regexp = described_class.fabricate!('/something/im')
+
+        expect(regexp).to eq Gitlab::UntrustedRegexp.new('(?im)something')
+      end
+
+      it 'fabricates regexp without flags' do
+        regexp = described_class.fabricate!('/something/')
+
+        expect(regexp).to eq Gitlab::UntrustedRegexp.new('something')
+      end
+    end
+
+    context 'when regexp is a raw pattern' do
+      it 'raises an error' do
+        expect { described_class.fabricate!('some .* thing') }
+          .to raise_error(RegexpError)
+      end
+    end
+  end
+end

spec/lib/gitlab/untrusted_regexp_spec.rb

@@ -2,48 +2,6 @@ require 'fast_spec_helper'
 require 'support/shared_examples/malicious_regexp_shared_examples'
 
 describe Gitlab::UntrustedRegexp do
-  describe '.valid?' do
-    it 'returns true if regexp is valid' do
-      expect(described_class.valid?('/some ( thing/'))
-        .to be false
-    end
-
-    it 'returns true if regexp is invalid' do
-      expect(described_class.valid?('/some .* thing/'))
-        .to be true
-    end
-  end
-
-  describe '.fabricate' do
-    context 'when regexp is using /regexp/ scheme with flags' do
-      it 'fabricates regexp with a single flag' do
-        regexp = described_class.fabricate('/something/i')
-
-        expect(regexp).to eq described_class.new('(?i)something')
-        expect(regexp.scan('SOMETHING')).to be_one
-      end
-
-      it 'fabricates regexp with multiple flags' do
-        regexp = described_class.fabricate('/something/im')
-
-        expect(regexp).to eq described_class.new('(?im)something')
-      end
-
-      it 'fabricates regexp without flags' do
-        regexp = described_class.fabricate('/something/')
-
-        expect(regexp).to eq described_class.new('something')
-      end
-    end
-
-    context 'when regexp is a raw pattern' do
-      it 'raises an error' do
-        expect { described_class.fabricate('some .* thing') }
-          .to raise_error(RegexpError)
-      end
-    end
-  end
-
   describe '#initialize' do
     subject { described_class.new(pattern) }
 
@@ -92,11 +50,41 @@ describe Gitlab::UntrustedRegexp do
     end
   end
 
+  describe '#match?' do
+    subject { described_class.new(regexp).match?(text) }
+
+    context 'malicious regexp' do
+      let(:text) { malicious_text }
+      let(:regexp) { malicious_regexp_re2 }
+
+      include_examples 'malicious regexp'
+    end
+
+    context 'matching regexp' do
+      let(:regexp) { 'foo' }
+      let(:text) { 'foo' }
+
+      it 'returns an array of nil matches' do
+        is_expected.to eq(true)
+      end
+    end
+
+    context 'non-matching regexp' do
+      let(:regexp) { 'boo' }
+      let(:text) { 'foo' }
+
+      it 'returns an array of nil matches' do
+        is_expected.to eq(false)
+      end
+    end
+  end
+
   describe '#scan' do
     subject { described_class.new(regexp).scan(text) }
+
     context 'malicious regexp' do
       let(:text) { malicious_text }
-      let(:regexp) { malicious_regexp }
+      let(:regexp) { malicious_regexp_re2 }
 
       include_examples 'malicious regexp'
     end

spec/policies/project_policy_spec.rb

@@ -2,6 +2,96 @@ require 'spec_helper'
 
 describe ProjectPolicy do
   include_context 'ProjectPolicy context'
+  set(:guest) { create(:user) }
+  set(:reporter) { create(:user) }
+  set(:developer) { create(:user) }
+  set(:maintainer) { create(:user) }
+  set(:owner) { create(:user) }
+  set(:admin) { create(:admin) }
+  let(:project) { create(:project, :public, namespace: owner.namespace) }
+
+  let(:base_guest_permissions) do
+    %i[
+      read_project read_board read_list read_wiki read_issue
+      read_project_for_iids read_issue_iid read_label
+      read_milestone read_project_snippet read_project_member read_note
+      create_project create_issue create_note upload_file create_merge_request_in
+      award_emoji
+    ]
+  end
+
+  let(:base_reporter_permissions) do
+    %i[
+      download_code fork_project create_project_snippet update_issue
+      admin_issue admin_label admin_list read_commit_status read_build
+      read_container_image read_pipeline read_environment read_deployment
+      read_merge_request download_wiki_code read_sentry_issue read_release
+    ]
+  end
+
+  let(:team_member_reporter_permissions) do
+    %i[build_download_code build_read_container_image]
+  end
+
+  let(:developer_permissions) do
+    %i[
+      admin_milestone admin_merge_request update_merge_request create_commit_status
+      update_commit_status create_build update_build create_pipeline
+      update_pipeline create_merge_request_from create_wiki push_code
+      resolve_note create_container_image update_container_image
+      create_environment create_deployment create_release update_release
+    ]
+  end
+
+  let(:base_maintainer_permissions) do
+    %i[
+      push_to_delete_protected_branch update_project_snippet update_environment
+      update_deployment admin_project_snippet admin_project_member admin_note admin_wiki admin_project
+      admin_commit_status admin_build admin_container_image
+      admin_pipeline admin_environment admin_deployment destroy_release add_cluster
+      daily_statistics
+    ]
+  end
+
+  let(:public_permissions) do
+    %i[
+      download_code fork_project read_commit_status read_pipeline
+      read_container_image build_download_code build_read_container_image
+      download_wiki_code read_release
+    ]
+  end
+
+  let(:owner_permissions) do
+    %i[
+      change_namespace change_visibility_level rename_project remove_project
+      archive_project remove_fork_project destroy_merge_request destroy_issue
+      set_issue_iid set_issue_created_at set_note_created_at
+    ]
+  end
+
+  # Used in EE specs
+  let(:additional_guest_permissions) { [] }
+  let(:additional_reporter_permissions) { [] }
+  let(:additional_maintainer_permissions) { [] }
+
+  let(:guest_permissions) { base_guest_permissions + additional_guest_permissions }
+  let(:reporter_permissions) { base_reporter_permissions + additional_reporter_permissions }
+  let(:maintainer_permissions) { base_maintainer_permissions + additional_maintainer_permissions }
+
+  before do
+    project.add_guest(guest)
+    project.add_maintainer(maintainer)
+    project.add_developer(developer)
+    project.add_reporter(reporter)
+  end
+
+  def expect_allowed(*permissions)
+    permissions.each { |p| is_expected.to be_allowed(p) }
+  end
+
+  def expect_disallowed(*permissions)
+    permissions.each { |p| is_expected.not_to be_allowed(p) }
+  end
 
   it 'does not include the read_issue permission when the issue author is not a member of the private project' do
     project = create(:project, :private)

spec/requests/api/projects_spec.rb

@@ -13,12 +13,18 @@ shared_examples 'languages and percentages JSON response' do
     )
   end
 
-  it 'returns expected language values' do
-    get api("/projects/#{project.id}/languages", user)
+  context "when the languages haven't been detected yet" do
+    it 'returns expected language values' do
+      get api("/projects/#{project.id}/languages", user)
+
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(json_response).to eq({})
 
-    expect(response).to have_gitlab_http_status(:ok)
-    expect(json_response).to eq(expected_languages)
-    expect(json_response.count).to be > 1
+      get api("/projects/#{project.id}/languages", user)
+
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(JSON.parse(response.body)).to eq(expected_languages)
+    end
   end
 
   context 'when the languages were detected before' do

spec/requests/api/releases_spec.rb

@@ -4,12 +4,14 @@ describe API::Releases do
   let(:project) { create(:project, :repository, :private) }
   let(:maintainer) { create(:user) }
   let(:reporter) { create(:user) }
+  let(:guest) { create(:user) }
   let(:non_project_member) { create(:user) }
   let(:commit) { create(:commit, project: project) }
 
   before do
     project.add_maintainer(maintainer)
     project.add_reporter(reporter)
+    project.add_guest(guest)
 
     project.repository.add_tag(maintainer, 'v0.1', commit.id)
     project.repository.add_tag(maintainer, 'v0.2', commit.id)
@@ -66,6 +68,24 @@ describe API::Releases do
       end
     end
 
+    context 'when user is a guest' do
+      it 'responds 403 Forbidden' do
+        get api("/projects/#{project.id}/releases", guest)
+
+        expect(response).to have_gitlab_http_status(:forbidden)
+      end
+
+      context 'when project is public' do
+        let(:project) { create(:project, :repository, :public) }
+
+        it 'responds 200 OK' do
+          get api("/projects/#{project.id}/releases", guest)
+
+          expect(response).to have_gitlab_http_status(:ok)
+        end
+      end
+    end
+
     context 'when user is not a project member' do
       it 'cannot find the project' do
         get api("/projects/#{project.id}/releases", non_project_member)
@@ -189,6 +209,24 @@ describe API::Releases do
           end
         end
       end
+
+      context 'when user is a guest' do
+        it 'responds 403 Forbidden' do
+          get api("/projects/#{project.id}/releases/v0.1", guest)
+
+          expect(response).to have_gitlab_http_status(:forbidden)
+        end
+
+        context 'when project is public' do
+          let(:project) { create(:project, :repository, :public) }
+
+          it 'responds 200 OK' do
+            get api("/projects/#{project.id}/releases/v0.1", guest)
+
+            expect(response).to have_gitlab_http_status(:ok)
+          end
+        end
+      end
     end
 
     context 'when specified tag is not found in the project' do

spec/services/projects/detect_repository_languages_service_spec.rb

@@ -19,6 +19,10 @@ describe Projects::DetectRepositoryLanguagesService, :clean_gitlab_redis_shared_
 
         expect(names).to eq(%w[Ruby JavaScript HTML CoffeeScript])
       end
+
+      it 'updates detected_repository_languages flag' do
+        expect { subject.execute }.to change(project, :detected_repository_languages).to(true)
+      end
     end
 
     context 'with a previous detection' do
@@ -36,6 +40,12 @@ describe Projects::DetectRepositoryLanguagesService, :clean_gitlab_redis_shared_
 
         expect(repository_languages).to eq(%w[Ruby D])
       end
+
+      it "doesn't touch detected_repository_languages flag" do
+        expect(project).not_to receive(:update_column).with(:detected_repository_languages, true)
+
+        subject.execute
+      end
     end
 
     context 'when no repository exists' do

spec/services/projects/repository_languages_service_spec.rb

+require 'spec_helper'
+
+describe Projects::RepositoryLanguagesService do
+  let(:service) { described_class.new(project, project.owner) }
+
+  context 'when detected_repository_languages flag is set' do
+    let(:project) { create(:project) }
+
+    context 'when a project is without detected programming languages' do
+      it 'schedules a worker and returns the empty result' do
+        expect(::DetectRepositoryLanguagesWorker).to receive(:perform_async).with(project.id, project.owner.id)
+        expect(service.execute).to eq([])
+      end
+    end
+
+    context 'when a project is with detected programming languages' do
+      let!(:repository_language) { create(:repository_language, project: project) }
+
+      it 'does not schedule a worker and returns the detected languages' do
+        expect(::DetectRepositoryLanguagesWorker).not_to receive(:perform_async).with(project.id, project.owner.id)
+
+        languages = service.execute
+
+        expect(languages.size).to eq(1)
+        expect(languages.last.attributes.values).to eq(
+          [project.id, repository_language.programming_language_id, repository_language.share]
+        )
+      end
+
+      it 'sets detected_repository_languages flag' do
+        expect { service.execute }.to change(project, :detected_repository_languages).from(nil).to(true)
+      end
+    end
+  end
+
+  context 'when detected_repository_languages flag is not set' do
+    let!(:repository_language) { create(:repository_language, project: project) }
+    let(:project) { create(:project, detected_repository_languages: true) }
+    let(:languages) { service.execute }
+
+    it 'returns repository languages' do
+      expect(languages.size).to eq(1)
+      expect(languages.last.attributes.values).to eq(
+        [project.id, repository_language.programming_language_id, repository_language.share]
+      )
+    end
+  end
+end

spec/support/shared_examples/malicious_regexp_shared_examples.rb

@@ -2,7 +2,8 @@ require 'timeout'
 
 shared_examples 'malicious regexp' do
   let(:malicious_text) { 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!' }
-  let(:malicious_regexp) { '(?i)^(([a-z])+.)+[A-Z]([a-z])+$' }
+  let(:malicious_regexp_re2) { '(?i)^(([a-z])+.)+[A-Z]([a-z])+$' }
+  let(:malicious_regexp_ruby) { '/^(([a-z])+.)+[A-Z]([a-z])+$/i' }
 
   it 'takes under a second' do
     expect { Timeout.timeout(1) { subject } }.not_to raise_error