Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
ef9774da
Commit
ef9774da
authored
May 06, 2019
by
GitLab Bot
Browse files
Options
Browse Files
Download
Plain Diff
Automatic merge of gitlab-org/gitlab-ce master
parents
257798f0
b80893c2
Changes
11
Hide whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
169 additions
and
1 deletion
+169
-1
changelogs/unreleased/issue_57906_fix_github_import.yml
changelogs/unreleased/issue_57906_fix_github_import.yml
+5
-0
db/migrate/20190506135337_add_temporary_indexes_to_state_id.rb
...grate/20190506135337_add_temporary_indexes_to_state_id.rb
+34
-0
db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb
...90506135400_schedule_sync_issuables_state_id_where_nil.rb
+63
-0
db/schema.rb
db/schema.rb
+1
-1
lib/gitlab/bitbucket_server_import/importer.rb
lib/gitlab/bitbucket_server_import/importer.rb
+1
-0
lib/gitlab/github_import/importer/issue_importer.rb
lib/gitlab/github_import/importer/issue_importer.rb
+1
-0
lib/gitlab/github_import/importer/pull_request_importer.rb
lib/gitlab/github_import/importer/pull_request_importer.rb
+1
-0
spec/lib/gitlab/bitbucket_server_import/importer_spec.rb
spec/lib/gitlab/bitbucket_server_import/importer_spec.rb
+1
-0
spec/lib/gitlab/github_import/importer/issue_importer_spec.rb
.../lib/gitlab/github_import/importer/issue_importer_spec.rb
+2
-0
spec/lib/gitlab/github_import/importer/pull_request_importer_spec.rb
...tlab/github_import/importer/pull_request_importer_spec.rb
+3
-0
spec/migrations/schedule_sync_issuables_state_id_where_nil_spec.rb
...ations/schedule_sync_issuables_state_id_where_nil_spec.rb
+57
-0
No files found.
changelogs/unreleased/issue_57906_fix_github_import.yml
0 → 100644
View file @
ef9774da
---
title
:
Fix issuables state_id nil when importing projects from GitHub
merge_request
:
28027
author
:
type
:
fixed
db/migrate/20190506135337_add_temporary_indexes_to_state_id.rb
0 → 100644
View file @
ef9774da
# frozen_string_literal: true
# This migration adds temporary indexes to state_id column of issues
# and merge_requests tables. It will be used only to peform the scheduling
# for populating state_id in a post migrate and will be removed after it.
# Check: ScheduleSyncIssuablesStateIdWhereNil.
class
AddTemporaryIndexesToStateId
<
ActiveRecord
::
Migration
[
5.1
]
include
Gitlab
::
Database
::
MigrationHelpers
DOWNTIME
=
false
disable_ddl_transaction!
def
up
%w(issues merge_requests)
.
each
do
|
table
|
add_concurrent_index
(
table
,
'id'
,
name:
index_name_for
(
table
),
where:
"state_id IS NULL"
)
end
end
def
down
remove_concurrent_index_by_name
(
:issues
,
index_name_for
(
"issues"
))
remove_concurrent_index_by_name
(
:merge_requests
,
index_name_for
(
"merge_requests"
))
end
def
index_name_for
(
table
)
"idx_on_
#{
table
}
_where_state_id_is_null"
end
end
db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb
0 → 100644
View file @
ef9774da
# frozen_string_literal: true
class
ScheduleSyncIssuablesStateIdWhereNil
<
ActiveRecord
::
Migration
[
5.1
]
# Issues and MergeRequests imported by GitHub are being created with
# state_id = null, this fixes them.
#
# Part of a bigger plan: https://gitlab.com/gitlab-org/gitlab-ce/issues/51789
include
Gitlab
::
Database
::
MigrationHelpers
DOWNTIME
=
false
# 2019-05-02 gitlab.com issuable numbers
# issues with state_id nil: ~40000
# merge requests with state_id nil: ~200000
#
# Using 5000 as batch size and 120 seconds interval will create:
# ~8 jobs for issues - taking ~16 minutes
# ~40 jobs for merge requests - taking ~1.34 hours
#
BATCH_SIZE
=
5000
DELAY_INTERVAL
=
120
.
seconds
.
to_i
ISSUES_MIGRATION
=
'SyncIssuesStateId'
.
freeze
MERGE_REQUESTS_MIGRATION
=
'SyncMergeRequestsStateId'
.
freeze
disable_ddl_transaction!
class
Issue
<
ActiveRecord
::
Base
include
EachBatch
self
.
table_name
=
'issues'
end
class
MergeRequest
<
ActiveRecord
::
Base
include
EachBatch
self
.
table_name
=
'merge_requests'
end
def
up
queue_background_migration_jobs_by_range_at_intervals
(
Issue
.
where
(
state_id:
nil
),
ISSUES_MIGRATION
,
DELAY_INTERVAL
,
batch_size:
BATCH_SIZE
)
queue_background_migration_jobs_by_range_at_intervals
(
MergeRequest
.
where
(
state_id:
nil
),
MERGE_REQUESTS_MIGRATION
,
DELAY_INTERVAL
,
batch_size:
BATCH_SIZE
)
# Remove temporary indexes added on "AddTemporaryIndexesToStateId"
remove_concurrent_index_by_name
(
:issues
,
"idx_on_issues_where_state_id_is_null"
)
remove_concurrent_index_by_name
(
:merge_requests
,
"idx_on_merge_requests_where_state_id_is_null"
)
end
def
down
# No op
end
end
db/schema.rb
View file @
ef9774da
...
...
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord
::
Schema
.
define
(
version:
20190
426180107
)
do
ActiveRecord
::
Schema
.
define
(
version:
20190
506135400
)
do
# These are extensions that must be enabled in order to support this database
enable_extension
"plpgsql"
...
...
lib/gitlab/bitbucket_server_import/importer.rb
View file @
ef9774da
...
...
@@ -201,6 +201,7 @@ module Gitlab
target_branch:
Gitlab
::
Git
.
ref_name
(
pull_request
.
target_branch_name
),
target_branch_sha:
pull_request
.
target_branch_sha
,
state:
pull_request
.
state
,
state_id:
MergeRequest
.
available_states
[
pull_request
.
state
],
author_id:
author_id
,
assignee_id:
nil
,
created_at:
pull_request
.
created_at
,
...
...
lib/gitlab/github_import/importer/issue_importer.rb
View file @
ef9774da
...
...
@@ -53,6 +53,7 @@ module Gitlab
description:
description
,
milestone_id:
milestone_finder
.
id_for
(
issue
),
state:
issue
.
state
,
state_id:
::
Issue
.
available_states
[
issue
.
state
],
created_at:
issue
.
created_at
,
updated_at:
issue
.
updated_at
}
...
...
lib/gitlab/github_import/importer/pull_request_importer.rb
View file @
ef9774da
...
...
@@ -55,6 +55,7 @@ module Gitlab
source_branch:
pull_request
.
formatted_source_branch
,
target_branch:
pull_request
.
target_branch
,
state:
pull_request
.
state
,
state_id:
::
MergeRequest
.
available_states
[
pull_request
.
state
],
milestone_id:
milestone_finder
.
id_for
(
pull_request
),
author_id:
author_id
,
assignee_id:
user_finder
.
assignee_id_for
(
pull_request
),
...
...
spec/lib/gitlab/bitbucket_server_import/importer_spec.rb
View file @
ef9774da
...
...
@@ -105,6 +105,7 @@ describe Gitlab::BitbucketServerImport::Importer do
expect
(
merge_request
.
metrics
.
merged_by
).
to
eq
(
project
.
owner
)
expect
(
merge_request
.
metrics
.
merged_at
).
to
eq
(
@merge_event
.
merge_timestamp
)
expect
(
merge_request
.
merge_commit_sha
).
to
eq
(
'12345678'
)
expect
(
merge_request
.
state_id
).
to
eq
(
3
)
end
it
'imports comments'
do
...
...
spec/lib/gitlab/github_import/importer/issue_importer_spec.rb
View file @
ef9774da
...
...
@@ -98,6 +98,7 @@ describe Gitlab::GithubImport::Importer::IssueImporter, :clean_gitlab_redis_cach
description:
'This is my issue'
,
milestone_id:
milestone
.
id
,
state: :opened
,
state_id:
1
,
created_at:
created_at
,
updated_at:
updated_at
},
...
...
@@ -127,6 +128,7 @@ describe Gitlab::GithubImport::Importer::IssueImporter, :clean_gitlab_redis_cach
description:
"*Created by: alice*
\n\n
This is my issue"
,
milestone_id:
milestone
.
id
,
state: :opened
,
state_id:
1
,
created_at:
created_at
,
updated_at:
updated_at
},
...
...
spec/lib/gitlab/github_import/importer/pull_request_importer_spec.rb
View file @
ef9774da
...
...
@@ -93,6 +93,7 @@ describe Gitlab::GithubImport::Importer::PullRequestImporter, :clean_gitlab_redi
source_branch:
'github/fork/alice/feature'
,
target_branch:
'master'
,
state: :merged
,
state_id:
3
,
milestone_id:
milestone
.
id
,
author_id:
user
.
id
,
assignee_id:
user
.
id
,
...
...
@@ -138,6 +139,7 @@ describe Gitlab::GithubImport::Importer::PullRequestImporter, :clean_gitlab_redi
source_branch:
'github/fork/alice/feature'
,
target_branch:
'master'
,
state: :merged
,
state_id:
3
,
milestone_id:
milestone
.
id
,
author_id:
project
.
creator_id
,
assignee_id:
user
.
id
,
...
...
@@ -184,6 +186,7 @@ describe Gitlab::GithubImport::Importer::PullRequestImporter, :clean_gitlab_redi
source_branch:
'master-42'
,
target_branch:
'master'
,
state: :merged
,
state_id:
3
,
milestone_id:
milestone
.
id
,
author_id:
user
.
id
,
assignee_id:
user
.
id
,
...
...
spec/migrations/schedule_sync_issuables_state_id_where_nil_spec.rb
0 → 100644
View file @
ef9774da
# frozen_string_literal: true
require
'spec_helper'
require
Rails
.
root
.
join
(
'db'
,
'post_migrate'
,
'20190506135400_schedule_sync_issuables_state_id_where_nil'
)
describe
ScheduleSyncIssuablesStateIdWhereNil
,
:migration
,
:sidekiq
do
let
(
:namespaces
)
{
table
(
:namespaces
)
}
let
(
:projects
)
{
table
(
:projects
)
}
let
(
:merge_requests
)
{
table
(
:merge_requests
)
}
let
(
:issues
)
{
table
(
:issues
)
}
let
(
:migration
)
{
described_class
.
new
}
let
(
:group
)
{
namespaces
.
create!
(
name:
'gitlab'
,
path:
'gitlab'
)
}
let
(
:project
)
{
projects
.
create!
(
namespace_id:
group
.
id
)
}
shared_examples
'scheduling migrations'
do
before
do
Sidekiq
::
Worker
.
clear_all
stub_const
(
"
#{
described_class
.
name
}
::BATCH_SIZE"
,
2
)
end
it
'correctly schedules issuable sync background migration'
do
Sidekiq
::
Testing
.
fake!
do
Timecop
.
freeze
do
migrate!
expect
(
migration
).
to
be_scheduled_delayed_migration
(
120
.
seconds
,
resource_1
.
id
,
resource_3
.
id
)
expect
(
migration
).
to
be_scheduled_delayed_migration
(
240
.
seconds
,
resource_5
.
id
,
resource_5
.
id
)
expect
(
BackgroundMigrationWorker
.
jobs
.
size
).
to
eq
(
2
)
end
end
end
end
describe
'#up'
do
context
'issues'
do
it_behaves_like
'scheduling migrations'
do
let
(
:migration
)
{
described_class
::
ISSUES_MIGRATION
}
let!
(
:resource_1
)
{
issues
.
create!
(
description:
'first'
,
state:
'opened'
,
state_id:
nil
)
}
let!
(
:resource_2
)
{
issues
.
create!
(
description:
'second'
,
state:
'closed'
,
state_id:
2
)
}
let!
(
:resource_3
)
{
issues
.
create!
(
description:
'third'
,
state:
'closed'
,
state_id:
nil
)
}
let!
(
:resource_4
)
{
issues
.
create!
(
description:
'fourth'
,
state:
'closed'
,
state_id:
2
)
}
let!
(
:resource_5
)
{
issues
.
create!
(
description:
'fifth'
,
state:
'closed'
,
state_id:
nil
)
}
end
end
context
'merge requests'
do
it_behaves_like
'scheduling migrations'
do
let
(
:migration
)
{
described_class
::
MERGE_REQUESTS_MIGRATION
}
let!
(
:resource_1
)
{
merge_requests
.
create!
(
state:
'opened'
,
state_id:
nil
,
target_project_id:
project
.
id
,
target_branch:
'feature1'
,
source_branch:
'master'
)
}
let!
(
:resource_2
)
{
merge_requests
.
create!
(
state:
'closed'
,
state_id:
2
,
target_project_id:
project
.
id
,
target_branch:
'feature2'
,
source_branch:
'master'
)
}
let!
(
:resource_3
)
{
merge_requests
.
create!
(
state:
'merged'
,
state_id:
nil
,
target_project_id:
project
.
id
,
target_branch:
'feature3'
,
source_branch:
'master'
)
}
let!
(
:resource_4
)
{
merge_requests
.
create!
(
state:
'locked'
,
state_id:
3
,
target_project_id:
project
.
id
,
target_branch:
'feature4'
,
source_branch:
'master'
)
}
let!
(
:resource_5
)
{
merge_requests
.
create!
(
state:
'locked'
,
state_id:
nil
,
target_project_id:
project
.
id
,
target_branch:
'feature4'
,
source_branch:
'master'
)
}
end
end
end
end
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment