Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
e7021c96
Commit
e7021c96
authored
May 29, 2020
by
Douglas Barbosa Alexandre
Committed by
Mike Kozono
Jun 04, 2020
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Default FDW to false on Geo::LfsObjectRegistryFinder
parent
59d52d9a
Changes
3
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
189 additions
and
505 deletions
+189
-505
ee/app/finders/geo/lfs_object_registry_finder.rb
ee/app/finders/geo/lfs_object_registry_finder.rb
+17
-50
ee/spec/finders/geo/lfs_object_registry_finder_spec.rb
ee/spec/finders/geo/lfs_object_registry_finder_spec.rb
+160
-430
ee/spec/models/geo_node_status_spec.rb
ee/spec/models/geo_node_status_spec.rb
+12
-25
No files found.
ee/app/finders/geo/lfs_object_registry_finder.rb
View file @
e7021c96
...
...
@@ -2,10 +2,8 @@
module
Geo
class
LfsObjectRegistryFinder
<
FileRegistryFinder
# Counts all existing registries independent
# of any change on filters / selective sync
def
count_registry
Geo
::
LfsObjectRegistry
.
count
syncable
.
count
end
def
count_syncable
...
...
@@ -13,22 +11,19 @@ module Geo
end
def
count_synced
lfs_objects
.
synced
.
count
syncable
.
synced
.
count
end
def
count_failed
lfs_objects
.
failed
.
count
syncable
.
failed
.
count
end
def
count_synced_missing_on_primary
lfs_objects
.
synced
.
missing_on_primary
.
count
syncable
.
synced
.
missing_on_primary
.
count
end
def
syncable
return
lfs_objects
if
selective_sync?
return
LfsObject
.
with_files_stored_locally
if
local_storage_only?
LfsObject
Geo
::
LfsObjectRegistry
end
# Returns untracked IDs as well as tracked IDs that are unused.
...
...
@@ -49,14 +44,8 @@ module Geo
#
# @return [Array] the first element is an Array of untracked IDs, and the second element is an Array of tracked IDs that are unused
def
find_registry_differences
(
range
)
source_ids
=
lfs_objects
(
fdw:
false
)
.
id_in
(
range
)
.
pluck_primary_key
tracked_ids
=
Geo
::
LfsObjectRegistry
.
pluck_model_ids_in_range
(
range
)
source_ids
=
lfs_objects
..
id_in
(
range
).
pluck_primary_key
tracked_ids
=
syncable
.
pluck_model_ids_in_range
(
range
)
untracked_ids
=
source_ids
-
tracked_ids
unused_tracked_ids
=
tracked_ids
-
source_ids
...
...
@@ -82,30 +71,12 @@ module Geo
# @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord
def
find_never_synced_registries
(
batch_size
:,
except_ids:
[])
Geo
::
LfsObjectRegistry
syncable
.
never
.
model_id_not_in
(
except_ids
)
.
limit
(
batch_size
)
end
# rubocop:enable CodeReuse/ActiveRecord
# Deprecated in favor of the process using
# #find_registry_differences and #find_never_synced_registries
#
# Find limited amount of non replicated lfs objects.
#
# You can pass a list with `except_ids:` so you can exclude items you
# already scheduled but haven't finished and aren't persisted to the database yet
#
# @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord
def
find_unsynced
(
batch_size
:,
except_ids:
[])
lfs_objects
.
missing_registry
.
id_not_in
(
except_ids
)
.
limit
(
batch_size
)
end
alias_method
:find_unsynced
,
:find_never_synced_registries
# rubocop:enable CodeReuse/ActiveRecord
# rubocop:disable CodeReuse/ActiveRecord
...
...
@@ -120,9 +91,9 @@ module Geo
# rubocop:disable CodeReuse/ActiveRecord
def
find_retryable_failed_registries
(
batch_size
:,
except_ids:
[])
registries_for_lfs_objects
.
merge
(
Geo
::
LfsObjectRegistry
.
failed
)
.
merge
(
Geo
::
LfsObjectRegistry
.
retry_due
)
syncable
.
failed
.
retry_due
.
model_id_not_in
(
except_ids
)
.
limit
(
batch_size
)
end
...
...
@@ -130,7 +101,7 @@ module Geo
# rubocop:disable CodeReuse/ActiveRecord
def
find_retryable_synced_missing_on_primary_registries
(
batch_size
:,
except_ids:
[])
registries_for_lfs_objects
syncable
.
synced
.
missing_on_primary
.
retry_due
...
...
@@ -141,16 +112,12 @@ module Geo
private
def
lfs_objects
(
fdw:
true
)
local_storage_only?
(
fdw:
fdw
)
?
all_lfs_objects
(
fdw:
fdw
).
with_files_stored_locally
:
all_lfs_objects
(
fdw:
fdw
)
end
def
all_lfs_objects
(
fdw:
true
)
current_node
(
fdw:
fdw
).
lfs_objects
def
lfs_objects
local_storage_only?
(
fdw:
false
)
?
all_lfs_objects
.
with_files_stored_locally
:
all_lfs_objects
end
def
registries_for
_lfs_objects
current_node
.
lfs_object_registrie
s
def
all
_lfs_objects
current_node
(
fdw:
false
).
lfs_object
s
end
end
end
ee/spec/finders/geo/lfs_object_registry_finder_spec.rb
View file @
e7021c96
This diff is collapsed.
Click to expand it.
ee/spec/models/geo_node_status_spec.rb
View file @
e7021c96
...
...
@@ -244,9 +244,9 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
create
(
:geo_upload_registry
,
:failed
)
create
(
:geo_upload_registry
,
:avatar
)
create
(
:geo_upload_registry
,
file_type: :attachment
)
create
(
:geo_lfs_object_registry
,
:
with_lfs_object
,
:
failed
)
create
(
:geo_lfs_object_registry
,
:failed
)
create
(
:geo_lfs_object_registry
,
:with_lfs_object
)
create
(
:geo_lfs_object_registry
)
expect
(
subject
.
lfs_objects_synced_count
).
to
eq
(
1
)
end
...
...
@@ -258,9 +258,9 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
create
(
:geo_upload_registry
,
:failed
)
create
(
:geo_upload_registry
,
:avatar
,
missing_on_primary:
true
)
create
(
:geo_upload_registry
,
file_type: :attachment
,
missing_on_primary:
true
)
create
(
:geo_lfs_object_registry
,
:
with_lfs_object
,
:
failed
)
create
(
:geo_lfs_object_registry
,
:failed
)
create
(
:geo_lfs_object_registry
,
:with_lfs_object
,
missing_on_primary:
true
)
create
(
:geo_lfs_object_registry
,
missing_on_primary:
true
)
expect
(
subject
.
lfs_objects_synced_missing_on_primary_count
).
to
eq
(
1
)
end
...
...
@@ -272,40 +272,27 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
create
(
:geo_upload_registry
,
:failed
)
create
(
:geo_upload_registry
,
:avatar
,
:failed
)
create
(
:geo_upload_registry
,
:failed
,
file_type: :attachment
)
create
(
:geo_lfs_object_registry
,
:with_lfs_object
)
create
(
:geo_lfs_object_registry
)
create
(
:geo_lfs_object_registry
,
:
with_lfs_object
,
:
failed
)
create
(
:geo_lfs_object_registry
,
:failed
)
expect
(
subject
.
lfs_objects_failed_count
).
to
eq
(
1
)
end
end
describe
'#lfs_objects_synced_in_percentage'
do
let
(
:lfs_object_project
)
{
create
(
:lfs_objects_project
,
project:
project_1
)
}
before
do
allow
(
ProjectCacheWorker
).
to
receive
(
:perform_async
).
and_return
(
true
)
create
(
:lfs_objects_project
,
project:
project_1
)
create_list
(
:lfs_objects_project
,
2
,
project:
project_3
)
end
it
'returns 0 when no objects are available'
do
it
'returns 0 when there are no registries'
do
expect
(
subject
.
lfs_objects_synced_in_percentage
).
to
eq
(
0
)
end
it
'returns the right percentage with no group restrictions'
do
create
(
:geo_lfs_object_registry
,
lfs_object_id:
lfs_object_project
.
lfs_object_id
)
it
'returns the right percentage'
do
create
(
:geo_lfs_object_registry
)
create
(
:geo_lfs_object_registry
,
:failed
)
create
(
:geo_lfs_object_registry
,
:never_synced
)
create
(
:geo_lfs_object_registry
,
:never_synced
)
expect
(
subject
.
lfs_objects_synced_in_percentage
).
to
be_within
(
0.0001
).
of
(
25
)
end
it
'returns the right percentage with group restrictions'
do
secondary
.
update!
(
selective_sync_type:
'namespaces'
,
namespaces:
[
group
])
create
(
:geo_lfs_object_registry
,
lfs_object_id:
lfs_object_project
.
lfs_object_id
)
expect
(
subject
.
lfs_objects_synced_in_percentage
).
to
be_within
(
0.0001
).
of
(
50
)
end
end
describe
'#job_artifacts_synced_count'
do
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment