Merge branch 'import-timeout' of https://dev.gitlab.org/dzaporozhets/gitlabhq...

Merge branch 'import-timeout' of https://dev.gitlab.org/dzaporozhets/gitlabhq into dzaporozhets/gitlabhq-import-timeout
Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>

Conflicts:
	CHANGELOG
	db/schema.rb

CHANGELOG

@@ -17,6 +17,7 @@ v 6.7.0
   - Add GFM autocompletion for MergeRequests (Robert Speicher)
   - Add webhook when a new tag is pushed (Jeroen van Baarsen)
   - Add button for toggling inline comments in diff view
+  - Add retry feature for repository import
 
 v 6.6.2
   - Fix 500 error on branch/tag create or remove via UI

app/controllers/projects_controller.rb

@@ -5,7 +5,7 @@ class ProjectsController < ApplicationController
 
   # Authorize
   before_filter :authorize_read_project!, except: [:index, :new, :create]
-  before_filter :authorize_admin_project!, only: [:edit, :update, :destroy, :transfer, :archive, :unarchive]
+  before_filter :authorize_admin_project!, only: [:edit, :update, :destroy, :transfer, :archive, :unarchive, :retry_import]
   before_filter :require_non_empty_project, only: [:blob, :tree, :graph]
 
   layout 'navless', only: [:new, :create, :fork]
@@ -21,16 +21,9 @@ class ProjectsController < ApplicationController
 
   def create
     @project = ::Projects::CreateService.new(current_user, params[:project]).execute
+    flash[:notice] = 'Project was successfully created.' if @project.saved?
 
     respond_to do |format|
-      flash[:notice] = 'Project was successfully created.' if @project.saved?
-      format.html do
-        if @project.saved?
-          redirect_to @project
-        else
-          render "new"
-        end
-      end
       format.js
     end
   end
@@ -55,6 +48,11 @@ class ProjectsController < ApplicationController
   end
 
   def show
+    if @project.import_in_progress?
+      redirect_to import_project_path(@project)
+      return
+    end
+
     return authenticate_user! unless @project.public? || current_user
 
     limit = (params[:limit] || 20).to_i
@@ -67,9 +65,7 @@ class ProjectsController < ApplicationController
         if @project.empty_repo?
           render "projects/empty", layout: user_layout
         else
-          if current_user
-            @last_push = current_user.recent_push(@project.id)
-          end
+          @last_push = current_user.recent_push(@project.id) if current_user
           render :show, layout: user_layout
         end
       end
@@ -77,6 +73,28 @@ class ProjectsController < ApplicationController
     end
   end
 
+  def import
+    if project.import_finished?
+      redirect_to @project
+      return
+    end
+  end
+
+  def retry_import
+    unless @project.import_failed?
+      redirect_to import_project_path(@project)
+    end
+
+    @project.import_url = params[:project][:import_url]
+
+    if @project.save
+      @project.reload
+      @project.import_retry
+    end
+
+    redirect_to import_project_path(@project)
+  end
+
   def destroy
     return access_denied! unless can?(current_user, :remove_project, project)
 

app/models/project.rb

@@ -28,7 +28,6 @@ class Project < ActiveRecord::Base
   include Gitlab::VisibilityLevel
   extend Enumerize
 
-  default_value_for :imported, false
   default_value_for :archived, false
 
   ActsAsTaggableOn.strict_case_match = true
@@ -59,13 +58,10 @@ class Project < ActiveRecord::Base
   has_one :gemnasium_service, dependent: :destroy
   has_one :forked_project_link, dependent: :destroy, foreign_key: "forked_to_project_id"
   has_one :forked_from_project, through: :forked_project_link
-
   # Merge Requests for target project should be removed with it
   has_many :merge_requests,     dependent: :destroy, foreign_key: "target_project_id"
-
   # Merge requests from source project should be kept when source project was removed
   has_many :fork_merge_requests, foreign_key: "source_project_id", class_name: MergeRequest
-
   has_many :issues, -> { order "state DESC, created_at DESC" }, dependent: :destroy
   has_many :services,           dependent: :destroy
   has_many :events,             dependent: :destroy
@@ -74,10 +70,8 @@ class Project < ActiveRecord::Base
   has_many :snippets,           dependent: :destroy, class_name: "ProjectSnippet"
   has_many :hooks,              dependent: :destroy, class_name: "ProjectHook"
   has_many :protected_branches, dependent: :destroy
-
   has_many :users_projects, dependent: :destroy
   has_many :users, through: :users_projects
-
   has_many :deploy_keys_projects, dependent: :destroy
   has_many :deploy_keys, through: :deploy_keys_projects
 
@@ -97,15 +91,12 @@ class Project < ActiveRecord::Base
   validates :issues_enabled, :wall_enabled, :merge_requests_enabled,
             :wiki_enabled, inclusion: { in: [true, false] }
   validates :issues_tracker_id, length: { maximum: 255 }, allow_blank: true
-
   validates :namespace, presence: true
   validates_uniqueness_of :name, scope: :namespace_id
   validates_uniqueness_of :path, scope: :namespace_id
-
   validates :import_url,
     format: { with: URI::regexp(%w(git http https)), message: "should be a valid url" },
     if: :import?
-
   validate :check_limit, on: :create
 
   # Scopes
@@ -118,14 +109,36 @@ class Project < ActiveRecord::Base
   scope :sorted_by_activity, -> { reorder("projects.last_activity_at DESC") }
   scope :personal, ->(user) { where(namespace_id: user.namespace_id) }
   scope :joined, ->(user) { where("namespace_id != ?", user.namespace_id) }
-
   scope :public_only, -> { where(visibility_level: Project::PUBLIC) }
   scope :public_and_internal_only, -> { where(visibility_level: Project.public_and_internal_levels) }
-
   scope :non_archived, -> { where(archived: false) }
 
   enumerize :issues_tracker, in: (Gitlab.config.issues_tracker.keys).append(:gitlab), default: :gitlab
 
+  state_machine :import_status, initial: :none do
+    event :import_start do
+      transition :none => :started
+    end
+
+    event :import_finish do
+      transition :started => :finished
+    end
+
+    event :import_fail do
+      transition :started => :failed
+    end
+
+    event :import_retry do
+      transition :failed => :started
+    end
+
+    state :started
+    state :finished
+    state :failed
+
+    after_transition any => :started, :do => :add_import_job
+  end
+
   class << self
     def public_and_internal_levels
       [Project::PUBLIC, Project::INTERNAL]
@@ -202,12 +215,28 @@ class Project < ActiveRecord::Base
     id && persisted?
   end
 
+  def add_import_job
+    RepositoryImportWorker.perform_in(2.seconds, id)
+  end
+
   def import?
     import_url.present?
   end
 
   def imported?
-    imported
+    import_finished?
+  end
+
+  def import_in_progress?
+    import? && import_status == 'started'
+  end
+
+  def import_failed?
+    import_status == 'failed'
+  end
+
+  def import_finished?
+    import_status == 'finished'
   end
 
   def check_limit

app/observers/project_observer.rb

 class ProjectObserver < BaseObserver
   def after_create(project)
-    project.update_column(:last_activity_at, project.created_at)
-
-    return true if project.forked?
-
-    if project.import?
-      RepositoryImportWorker.perform_in(5.seconds, project.id)
-    else
-      GitlabShellWorker.perform_async(
-        :add_repository,
-        project.path_with_namespace
-      )
-
-      log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
-    end
-
-    if project.wiki_enabled?
-      begin
-        # force the creation of a wiki,
-        GollumWiki.new(project, project.owner).wiki
-      rescue GollumWiki::CouldNotCreateWikiError => ex
-        # Prevent project observer crash
-        # if failed to create wiki
-        nil
-      end
-    end
+    log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
   end
 
   def after_update(project)

app/services/projects/create_service.rb

@@ -58,6 +58,29 @@ module Projects
             user: current_user
           )
         end
+
+        @project.update_column(:last_activity_at, @project.created_at)
+
+        if @project.import?
+          @project.import_start
+        else
+          GitlabShellWorker.perform_async(
+            :add_repository,
+            @project.path_with_namespace
+          )
+
+        end
+
+        if @project.wiki_enabled?
+          begin
+            # force the creation of a wiki,
+            GollumWiki.new(@project, @project.owner).wiki
+          rescue GollumWiki::CouldNotCreateWikiError => ex
+            # Prevent project observer crash
+            # if failed to create wiki
+            nil
+          end
+        end
       end
 
       @project

app/views/projects/create.js.haml

 - if @project.saved?
-  :plain
-    location.href = "#{project_path(@project)}";
+  - if @project.import?
+    :plain
+      location.href = "#{import_project_path(@project)}";
+  - else
+    :plain
+      location.href = "#{project_path(@project)}";
 - else
   :plain
     $(".project-edit-errors").html("#{escape_javascript(render('errors'))}");

app/views/projects/empty.html.haml

 = render "home_panel"
 
-- if @project.import? && !@project.imported
-  .save-project-loader
-    %center
-      %h2
-        %i.icon-spinner.icon-spin
-        Importing repository.
-      %p.monospace git clone --bare #{@project.import_url}
-      %p Please wait while we import the repository for you. Refresh at will.
-      :javascript
-        new ProjectImport();
+%div.git-empty
+  %fieldset
+    %legend Git global setup:
+    %pre.dark
+      :preserve
+        git config --global user.name "#{git_user_name}"
+        git config --global user.email "#{git_user_email}"
 
-- else
-  %div.git-empty
-    %fieldset
-      %legend Git global setup:
-      %pre.dark
-        :preserve
-          git config --global user.name "#{git_user_name}"
-          git config --global user.email "#{git_user_email}"
+  %fieldset
+    %legend Create Repository
+    %pre.dark
+      :preserve
+        mkdir #{@project.path}
+        cd #{@project.path}
+        git init
+        touch README
+        git add README
+        git commit -m 'first commit'
+        git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
+        git push -u origin master
 
-    %fieldset
-      %legend Create Repository
-      %pre.dark
-        :preserve
-          mkdir #{@project.path}
-          cd #{@project.path}
-          git init
-          touch README
-          git add README
-          git commit -m 'first commit'
-          git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
-          git push -u origin master
+  %fieldset
+    %legend Existing Git Repo?
+    %pre.dark
+      :preserve
+        cd existing_git_repo
+        git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
+        git push -u origin master
 
-    %fieldset
-      %legend Existing Git Repo?
-      %pre.dark
-        :preserve
-          cd existing_git_repo
-          git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
-          git push -u origin master
-
-  - if can? current_user, :remove_project, @project
-    .prepend-top-20
-      = link_to 'Remove project', @project, data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-remove pull-right"
+- if can? current_user, :remove_project, @project
+  .prepend-top-20
+    = link_to 'Remove project', @project, data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-remove pull-right"

app/views/projects/import.html.haml

+- if @project.import_in_progress?
+  .save-project-loader
+    %center
+      %h2
+        %i.icon-spinner.icon-spin
+        Import in progress.
+      %p.monospace git clone --bare #{@project.import_url}
+      %p Please wait while we import the repository for you. Refresh at will.
+      :javascript
+        new ProjectImport();
+
+- elsif @project.import_failed?
+  .save-project-loader
+    %center
+      %h2
+        Import failed. Retry?
+  %hr
+  - if can?(current_user, :admin_project, @project)
+    = form_for @project, url: retry_import_project_path(@project), method: :put, html: { class: 'form-horizontal' } do |f|
+      .form-group.import-url-data
+        = f.label :import_url, class: 'control-label' do
+          %span Import existing repo
+        .col-sm-10
+          = f.text_field :import_url, class: 'form-control', placeholder: 'https://github.com/randx/six.git'
+          .bs-callout.bs-callout-info
+            This url must be publicly accessible or you can add a username and password like this: https://username:password@gitlab.com/company/project.git.
+            %br
+            The import will time out after 2 minutes. For big repositories, use a clone/push combination.
+      .form-actions
+        = f.submit 'Retry import', class: "btn btn-create", tabindex: 4

app/views/projects/new.html.haml

@@ -23,6 +23,7 @@
           .col-sm-2
           .col-sm-10
             = link_to "#", class: 'js-toggle-button' do
+              %i.icon-edit
               %span Customize repository name?
         .js-toggle-content.hide
           .form-group
@@ -46,8 +47,10 @@
               %span Import existing repo
             .col-sm-10
               = f.text_field :import_url, class: 'form-control', placeholder: 'https://github.com/randx/six.git'
-              .light
-                URL must be cloneable
+              .bs-callout.bs-callout-info
+                This url must be publicly accessible or you can add a username and password like this: https://username:password@gitlab.com/company/project.git.
+                %br
+                The import will time out after 2 minutes. For big repositories, use a clone/push combination.
       %hr
 
       .form-group

app/workers/repository_import_worker.rb

@@ -11,11 +11,11 @@ class RepositoryImportWorker
                                project.import_url)
 
     if result
-      project.imported = true
+      project.import_finish
       project.save
       project.satellite.create unless project.satellite.exists?
     else
-      project.imported = false
+      project.import_fail
     end
   end
 end

config/routes.rb

@@ -179,6 +179,8 @@ Gitlab::Application.routes.draw do
       post :archive
       post :unarchive
       get :autocomplete_sources
+      get :import
+      put :retry_import
     end
 
     scope module: :projects do

db/migrate/20140312145357_add_import_status_to_project.rb

+class AddImportStatusToProject < ActiveRecord::Migration
+  def change
+    add_column :projects, :import_status, :string
+  end
+end

db/migrate/20140313092127_migrate_already_imported_projects.rb

+class MigrateAlreadyImportedProjects < ActiveRecord::Migration
+  def up
+    Project.where(imported: true).update_all(import_status: "finished")
+    Project.where(imported: false).update_all(import_status: "none")
+    remove_column :projects, :imported
+  end
+
+  def down
+    add_column :projects, :imported, :boolean, default: false
+    Project.where(import_status: 'finished').update_all(imported: true)
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20140305193308) do
+ActiveRecord::Schema.define(version: 20140313092127) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -214,10 +214,10 @@ ActiveRecord::Schema.define(version: 20140305193308) do
     t.string   "issues_tracker_id"
     t.boolean  "snippets_enabled",       default: true,     null: false
     t.datetime "last_activity_at"
-    t.boolean  "imported",               default: false,    null: false
     t.string   "import_url"
     t.integer  "visibility_level",       default: 0,        null: false
     t.boolean  "archived",               default: false,    null: false
+    t.string   "import_status"
   end
 
   add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree