Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
729de4f1
Commit
729de4f1
authored
Jul 23, 2018
by
Andreas Brandl
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add migration to cleanup internal_ids.
See
https://gitlab.com/gitlab-org/gitlab-ce/issues/49446
.
parent
fb98496f
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
171 additions
and
0 deletions
+171
-0
changelogs/unreleased/ab-49446-internal-ids-inconsistency.yml
...gelogs/unreleased/ab-49446-internal-ids-inconsistency.yml
+5
-0
db/post_migrate/20180723130817_delete_inconsistent_internal_id_records.rb
...20180723130817_delete_inconsistent_internal_id_records.rb
+47
-0
spec/migrations/delete_inconsistent_internal_id_records_spec.rb
...igrations/delete_inconsistent_internal_id_records_spec.rb
+119
-0
No files found.
changelogs/unreleased/ab-49446-internal-ids-inconsistency.yml
0 → 100644
View file @
729de4f1
---
title
:
Add migration to cleanup internal_ids inconsistency.
merge_request
:
20926
author
:
type
:
fixed
db/post_migrate/20180723130817_delete_inconsistent_internal_id_records.rb
0 → 100644
View file @
729de4f1
# frozen_string_literal: true
class
DeleteInconsistentInternalIdRecords
<
ActiveRecord
::
Migration
include
Gitlab
::
Database
::
MigrationHelpers
DOWNTIME
=
false
disable_ddl_transaction!
# This migration cleans up any inconsistent records in internal_ids.
#
# That is, it deletes records that track a `last_value` that is
# smaller than the maximum internal id (usually `iid`) found in
# the corresponding model records.
def
up
disable_statement_timeout
do
delete_internal_id_records
(
'issues'
,
'project_id'
)
delete_internal_id_records
(
'merge_requests'
,
'project_id'
,
'target_project_id'
)
delete_internal_id_records
(
'deployments'
,
'project_id'
)
delete_internal_id_records
(
'milestones'
,
'project_id'
)
delete_internal_id_records
(
'milestones'
,
'namespace_id'
,
'group_id'
)
delete_internal_id_records
(
'ci_pipelines'
,
'project_id'
)
end
end
class
InternalId
<
ActiveRecord
::
Base
self
.
table_name
=
'internal_ids'
enum
usage:
{
issues:
0
,
merge_requests:
1
,
deployments:
2
,
milestones:
3
,
epics:
4
,
ci_pipelines:
5
}
end
private
def
delete_internal_id_records
(
base_table
,
scope_column_name
,
base_scope_column_name
=
scope_column_name
)
sql
=
<<~
SQL
SELECT id FROM ( -- workaround for MySQL
SELECT internal_ids.id FROM (
SELECT
#{
base_scope_column_name
}
AS
#{
scope_column_name
}
, max(iid) as maximum_iid from
#{
base_table
}
GROUP BY
#{
scope_column_name
}
) maxima JOIN internal_ids USING (
#{
scope_column_name
}
)
WHERE internal_ids.usage=
#{
InternalId
.
usages
.
fetch
(
base_table
)
}
AND maxima.maximum_iid > internal_ids.last_value
) internal_ids
SQL
InternalId
.
where
(
"id IN (
#{
sql
}
)"
).
tap
do
|
ids
|
# rubocop:disable GitlabSecurity/SqlInjection
say
"Deleting internal_id records for
#{
base_table
}
:
#{
ids
.
pluck
(
:project_id
,
:last_value
)
}
"
unless
ids
.
empty?
end
.
delete_all
end
end
spec/migrations/delete_inconsistent_internal_id_records_spec.rb
0 → 100644
View file @
729de4f1
# frozen_string_literal: true
# rubocop:disable RSpec/FactoriesInMigrationSpecs
require
'spec_helper'
require
Rails
.
root
.
join
(
'db'
,
'post_migrate'
,
'20180723130817_delete_inconsistent_internal_id_records.rb'
)
describe
DeleteInconsistentInternalIdRecords
,
:migration
do
let!
(
:project1
)
{
create
(
:project
)
}
let!
(
:project2
)
{
create
(
:project
)
}
let!
(
:project3
)
{
create
(
:project
)
}
let
(
:internal_id_query
)
{
->
(
project
)
{
InternalId
.
where
(
usage:
InternalId
.
usages
[
scope
.
to_s
.
tableize
],
project:
project
)
}
}
let
(
:create_models
)
do
3
.
times
{
create
(
scope
,
project:
project1
)
}
3
.
times
{
create
(
scope
,
project:
project2
)
}
3
.
times
{
create
(
scope
,
project:
project3
)
}
end
shared_examples_for
'deleting inconsistent internal_id records'
do
before
do
create_models
internal_id_query
.
call
(
project1
).
first
.
tap
do
|
iid
|
iid
.
last_value
=
iid
.
last_value
-
2
# This is an inconsistent record
iid
.
save!
end
internal_id_query
.
call
(
project3
).
first
.
tap
do
|
iid
|
iid
.
last_value
=
iid
.
last_value
+
2
# This is a consistent record
iid
.
save!
end
end
it
"deletes inconsistent issues"
do
expect
{
migrate!
}.
to
change
{
internal_id_query
.
call
(
project1
).
size
}.
from
(
1
).
to
(
0
)
end
it
"retains consistent issues"
do
expect
{
migrate!
}.
not_to
change
{
internal_id_query
.
call
(
project2
).
size
}
end
it
"retains consistent records, especially those with a greater last_value"
do
expect
{
migrate!
}.
not_to
change
{
internal_id_query
.
call
(
project3
).
size
}
end
end
context
'for issues'
do
let
(
:scope
)
{
:issue
}
it_behaves_like
'deleting inconsistent internal_id records'
end
context
'for merge_requests'
do
let
(
:scope
)
{
:merge_request
}
let
(
:create_models
)
do
3
.
times
{
|
i
|
create
(
scope
,
target_project:
project1
,
source_project:
project1
,
source_branch:
i
.
to_s
)
}
3
.
times
{
|
i
|
create
(
scope
,
target_project:
project2
,
source_project:
project2
,
source_branch:
i
.
to_s
)
}
3
.
times
{
|
i
|
create
(
scope
,
target_project:
project3
,
source_project:
project3
,
source_branch:
i
.
to_s
)
}
end
it_behaves_like
'deleting inconsistent internal_id records'
end
context
'for deployments'
do
let
(
:scope
)
{
:deployment
}
it_behaves_like
'deleting inconsistent internal_id records'
end
context
'for milestones (by project)'
do
let
(
:scope
)
{
:milestone
}
it_behaves_like
'deleting inconsistent internal_id records'
end
context
'for ci_pipelines'
do
let
(
:scope
)
{
:ci_pipeline
}
it_behaves_like
'deleting inconsistent internal_id records'
end
context
'for milestones (by group)'
do
# milestones (by group) is a little different than all of the other models
let!
(
:group1
)
{
create
(
:group
)
}
let!
(
:group2
)
{
create
(
:group
)
}
let!
(
:group3
)
{
create
(
:group
)
}
let
(
:internal_id_query
)
{
->
(
group
)
{
InternalId
.
where
(
usage:
InternalId
.
usages
[
'milestones'
],
namespace:
group
)
}
}
before
do
3
.
times
{
create
(
:milestone
,
group:
group1
)
}
3
.
times
{
create
(
:milestone
,
group:
group2
)
}
3
.
times
{
create
(
:milestone
,
group:
group3
)
}
internal_id_query
.
call
(
group1
).
first
.
tap
do
|
iid
|
iid
.
last_value
=
iid
.
last_value
-
2
# This is an inconsistent record
iid
.
save!
end
internal_id_query
.
call
(
group3
).
first
.
tap
do
|
iid
|
iid
.
last_value
=
iid
.
last_value
+
2
# This is a consistent record
iid
.
save!
end
end
it
"deletes inconsistent issues"
do
expect
{
migrate!
}.
to
change
{
internal_id_query
.
call
(
group1
).
size
}.
from
(
1
).
to
(
0
)
end
it
"retains consistent issues"
do
expect
{
migrate!
}.
not_to
change
{
internal_id_query
.
call
(
group2
).
size
}
end
it
"retains consistent records, especially those with a greater last_value"
do
expect
{
migrate!
}.
not_to
change
{
internal_id_query
.
call
(
group3
).
size
}
end
end
end
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment