Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
92b7d1bb
Commit
92b7d1bb
authored
Nov 09, 2017
by
Valery Sizov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
GEO: Address review comments
parent
5754557d
Changes
10
Hide whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
44 additions
and
10 deletions
+44
-10
app/models/geo/file_registry.rb
app/models/geo/file_registry.rb
+1
-1
app/models/geo/project_registry.rb
app/models/geo/project_registry.rb
+1
-1
app/services/geo/base_sync_service.rb
app/services/geo/base_sync_service.rb
+6
-1
app/services/geo/repository_sync_service.rb
app/services/geo/repository_sync_service.rb
+2
-1
app/services/geo/wiki_sync_service.rb
app/services/geo/wiki_sync_service.rb
+2
-1
app/workers/geo/file_download_dispatch_worker.rb
app/workers/geo/file_download_dispatch_worker.rb
+1
-1
app/workers/geo/repository_sync_worker.rb
app/workers/geo/repository_sync_worker.rb
+1
-1
db/geo/migrate/20171101105200_add_retry_count_fields_to_registries.rb
...te/20171101105200_add_retry_count_fields_to_registries.rb
+25
-1
db/geo/schema.rb
db/geo/schema.rb
+3
-0
spec/services/geo/base_sync_service_spec.rb
spec/services/geo/base_sync_service_spec.rb
+2
-2
No files found.
app/models/geo/file_registry.rb
View file @
92b7d1bb
class
Geo::FileRegistry
<
Geo
::
BaseRegistry
class
Geo::FileRegistry
<
Geo
::
BaseRegistry
scope
:failed
,
->
{
where
(
success:
false
)
}
scope
:failed
,
->
{
where
(
success:
false
)
}
scope
:synced
,
->
{
where
(
success:
true
)
}
scope
:synced
,
->
{
where
(
success:
true
)
}
scope
:
to_be_retried
,
->
{
where
(
'retry_at is NULL OR retry_at < ?'
,
Time
.
now
)
}
scope
:
retry_due
,
->
{
where
(
'retry_at is NULL OR retry_at < ?'
,
Time
.
now
)
}
scope
:lfs_objects
,
->
{
where
(
file_type: :lfs
)
}
scope
:lfs_objects
,
->
{
where
(
file_type: :lfs
)
}
scope
:attachments
,
->
{
where
(
file_type:
Geo
::
FileService
::
DEFAULT_OBJECT_TYPES
)
}
scope
:attachments
,
->
{
where
(
file_type:
Geo
::
FileService
::
DEFAULT_OBJECT_TYPES
)
}
end
end
app/models/geo/project_registry.rb
View file @
92b7d1bb
...
@@ -15,7 +15,7 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
...
@@ -15,7 +15,7 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
where
(
repository_sync_failed
.
or
(
wiki_sync_failed
))
where
(
repository_sync_failed
.
or
(
wiki_sync_failed
))
end
end
def
self
.
to_be_retried
def
self
.
retry_due
where
(
where
(
arel_table
[
:repository_retry_at
].
lt
(
Time
.
now
)
arel_table
[
:repository_retry_at
].
lt
(
Time
.
now
)
.
or
(
arel_table
[
:wiki_retry_at
].
lt
(
Time
.
now
))
.
or
(
arel_table
[
:wiki_retry_at
].
lt
(
Time
.
now
))
...
...
app/services/geo/base_sync_service.rb
View file @
92b7d1bb
...
@@ -172,7 +172,12 @@ module Geo
...
@@ -172,7 +172,12 @@ module Geo
end
end
def
disk_path_temp
def
disk_path_temp
"
#{
repository
.
disk_path
}
_temp"
unless
@disk_path_temp
random_string
=
(
0
...
6
).
map
{
(
'a'
..
'z'
).
to_a
.
sample
}.
join
@disk_path_temp
=
"
#{
repository
.
disk_path
}
_
#{
random_string
}
"
end
@disk_path_temp
end
end
def
build_temporary_repository
def
build_temporary_repository
...
...
app/services/geo/repository_sync_service.rb
View file @
92b7d1bb
...
@@ -17,7 +17,6 @@ module Geo
...
@@ -17,7 +17,6 @@ module Geo
if
redownload
if
redownload
log_info
(
'Redownloading repository'
)
log_info
(
'Redownloading repository'
)
clean_up_temporary_repository
fetch_geo_mirror
(
build_temporary_repository
)
fetch_geo_mirror
(
build_temporary_repository
)
set_temp_repository_as_main
set_temp_repository_as_main
else
else
...
@@ -37,6 +36,8 @@ module Geo
...
@@ -37,6 +36,8 @@ module Geo
log_error
(
'Invalid repository'
,
e
)
log_error
(
'Invalid repository'
,
e
)
registry
.
update
(
force_to_redownload_repository:
true
)
registry
.
update
(
force_to_redownload_repository:
true
)
expire_repository_caches
expire_repository_caches
ensure
clean_up_temporary_repository
if
redownload
end
end
def
expire_repository_caches
def
expire_repository_caches
...
...
app/services/geo/wiki_sync_service.rb
View file @
92b7d1bb
...
@@ -16,7 +16,6 @@ module Geo
...
@@ -16,7 +16,6 @@ module Geo
if
redownload
if
redownload
log_info
(
'Redownloading wiki'
)
log_info
(
'Redownloading wiki'
)
clean_up_temporary_repository
fetch_geo_mirror
(
build_temporary_repository
)
fetch_geo_mirror
(
build_temporary_repository
)
set_temp_repository_as_main
set_temp_repository_as_main
else
else
...
@@ -37,6 +36,8 @@ module Geo
...
@@ -37,6 +36,8 @@ module Geo
rescue
Gitlab
::
Git
::
Repository
::
NoRepository
=>
e
rescue
Gitlab
::
Git
::
Repository
::
NoRepository
=>
e
log_error
(
'Invalid wiki'
,
e
)
log_error
(
'Invalid wiki'
,
e
)
registry
.
update
(
force_to_redownload_wiki:
true
)
registry
.
update
(
force_to_redownload_wiki:
true
)
ensure
clean_up_temporary_repository
if
redownload
end
end
def
ssh_url_to_wiki
def
ssh_url_to_wiki
...
...
app/workers/geo/file_download_dispatch_worker.rb
View file @
92b7d1bb
...
@@ -33,7 +33,7 @@ module Geo
...
@@ -33,7 +33,7 @@ module Geo
def
find_failed_objects
(
batch_size
:)
def
find_failed_objects
(
batch_size
:)
Geo
::
FileRegistry
Geo
::
FileRegistry
.
failed
.
failed
.
to_be_retried
.
retry_due
.
limit
(
batch_size
)
.
limit
(
batch_size
)
.
pluck
(
:file_id
,
:file_type
)
.
pluck
(
:file_id
,
:file_type
)
end
end
...
...
app/workers/geo/repository_sync_worker.rb
View file @
92b7d1bb
...
@@ -33,7 +33,7 @@ module Geo
...
@@ -33,7 +33,7 @@ module Geo
def
find_project_ids_updated_recently
(
batch_size
:)
def
find_project_ids_updated_recently
(
batch_size
:)
current_node
.
project_registries
current_node
.
project_registries
.
dirty
.
dirty
.
to_be_retried
.
retry_due
.
order
(
Gitlab
::
Database
.
nulls_first_order
(
:last_repository_synced_at
,
:desc
))
.
order
(
Gitlab
::
Database
.
nulls_first_order
(
:last_repository_synced_at
,
:desc
))
.
limit
(
batch_size
)
.
limit
(
batch_size
)
.
pluck
(
:project_id
)
.
pluck
(
:project_id
)
...
...
db/geo/migrate/20171101105200_add_retry_count_fields_to_registries.rb
View file @
92b7d1bb
class
AddRetryCountFieldsToRegistries
<
ActiveRecord
::
Migration
class
AddRetryCountFieldsToRegistries
<
ActiveRecord
::
Migration
def
change
include
Gitlab
::
Database
::
MigrationHelpers
DOWNTIME
=
false
disable_ddl_transaction!
def
up
add_column
:file_registry
,
:retry_count
,
:integer
add_column
:file_registry
,
:retry_count
,
:integer
add_column
:file_registry
,
:retry_at
,
:datetime
add_column
:file_registry
,
:retry_at
,
:datetime
...
@@ -10,5 +16,23 @@ class AddRetryCountFieldsToRegistries < ActiveRecord::Migration
...
@@ -10,5 +16,23 @@ class AddRetryCountFieldsToRegistries < ActiveRecord::Migration
add_column
:project_registry
,
:wiki_retry_count
,
:integer
add_column
:project_registry
,
:wiki_retry_count
,
:integer
add_column
:project_registry
,
:wiki_retry_at
,
:datetime
add_column
:project_registry
,
:wiki_retry_at
,
:datetime
add_column
:project_registry
,
:force_to_redownload_wiki
,
:boolean
add_column
:project_registry
,
:force_to_redownload_wiki
,
:boolean
# Indecies
add_concurrent_index
:file_registry
,
:retry_at
add_concurrent_index
:project_registry
,
:repository_retry_at
add_concurrent_index
:project_registry
,
:wiki_retry_at
end
def
down
remove_column
:file_registry
,
:retry_count
,
:integer
remove_column
:file_registry
,
:retry_at
,
:datetime
remove_column
:project_registry
,
:repository_retry_count
,
:integer
remove_column
:project_registry
,
:repository_retry_at
,
:datetime
remove_column
:project_registry
,
:force_to_redownload_repository
,
:boolean
remove_column
:project_registry
,
:wiki_retry_count
,
:integer
remove_column
:project_registry
,
:wiki_retry_at
,
:datetime
remove_column
:project_registry
,
:force_to_redownload_wiki
,
:boolean
end
end
end
end
db/geo/schema.rb
View file @
92b7d1bb
...
@@ -32,6 +32,7 @@ ActiveRecord::Schema.define(version: 20171101105200) do
...
@@ -32,6 +32,7 @@ ActiveRecord::Schema.define(version: 20171101105200) do
add_index
"file_registry"
,
[
"file_type"
,
"file_id"
],
name:
"index_file_registry_on_file_type_and_file_id"
,
unique:
true
,
using: :btree
add_index
"file_registry"
,
[
"file_type"
,
"file_id"
],
name:
"index_file_registry_on_file_type_and_file_id"
,
unique:
true
,
using: :btree
add_index
"file_registry"
,
[
"file_type"
],
name:
"index_file_registry_on_file_type"
,
using: :btree
add_index
"file_registry"
,
[
"file_type"
],
name:
"index_file_registry_on_file_type"
,
using: :btree
add_index
"file_registry"
,
[
"retry_at"
],
name:
"index_file_registry_on_retry_at"
,
using: :btree
add_index
"file_registry"
,
[
"success"
],
name:
"index_file_registry_on_success"
,
using: :btree
add_index
"file_registry"
,
[
"success"
],
name:
"index_file_registry_on_success"
,
using: :btree
create_table
"project_registry"
,
force: :cascade
do
|
t
|
create_table
"project_registry"
,
force: :cascade
do
|
t
|
...
@@ -54,7 +55,9 @@ ActiveRecord::Schema.define(version: 20171101105200) do
...
@@ -54,7 +55,9 @@ ActiveRecord::Schema.define(version: 20171101105200) do
add_index
"project_registry"
,
[
"last_repository_successful_sync_at"
],
name:
"index_project_registry_on_last_repository_successful_sync_at"
,
using: :btree
add_index
"project_registry"
,
[
"last_repository_successful_sync_at"
],
name:
"index_project_registry_on_last_repository_successful_sync_at"
,
using: :btree
add_index
"project_registry"
,
[
"last_repository_synced_at"
],
name:
"index_project_registry_on_last_repository_synced_at"
,
using: :btree
add_index
"project_registry"
,
[
"last_repository_synced_at"
],
name:
"index_project_registry_on_last_repository_synced_at"
,
using: :btree
add_index
"project_registry"
,
[
"project_id"
],
name:
"index_project_registry_on_project_id"
,
unique:
true
,
using: :btree
add_index
"project_registry"
,
[
"project_id"
],
name:
"index_project_registry_on_project_id"
,
unique:
true
,
using: :btree
add_index
"project_registry"
,
[
"repository_retry_at"
],
name:
"index_project_registry_on_repository_retry_at"
,
using: :btree
add_index
"project_registry"
,
[
"resync_repository"
],
name:
"index_project_registry_on_resync_repository"
,
using: :btree
add_index
"project_registry"
,
[
"resync_repository"
],
name:
"index_project_registry_on_resync_repository"
,
using: :btree
add_index
"project_registry"
,
[
"resync_wiki"
],
name:
"index_project_registry_on_resync_wiki"
,
using: :btree
add_index
"project_registry"
,
[
"resync_wiki"
],
name:
"index_project_registry_on_resync_wiki"
,
using: :btree
add_index
"project_registry"
,
[
"wiki_retry_at"
],
name:
"index_project_registry_on_wiki_retry_at"
,
using: :btree
end
end
spec/services/geo/base_sync_service_spec.rb
View file @
92b7d1bb
...
@@ -9,10 +9,10 @@ describe Geo::BaseSyncService do
...
@@ -9,10 +9,10 @@ describe Geo::BaseSyncService do
describe
'#lease_key'
do
describe
'#lease_key'
do
it
'returns a key in the correct pattern'
do
it
'returns a key in the correct pattern'
do
described_class
.
type
=
:test
allow
(
described_class
).
to
receive
(
:type
)
{
:wiki
}
allow
(
project
).
to
receive
(
:id
)
{
999
}
allow
(
project
).
to
receive
(
:id
)
{
999
}
expect
(
subject
.
lease_key
).
to
eq
(
'geo_sync_service:
test
:999'
)
expect
(
subject
.
lease_key
).
to
eq
(
'geo_sync_service:
wiki
:999'
)
end
end
end
end
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment