Commit 687bce53 authored by Kassio Borges's avatar Kassio Borges

BulkImports: always log the current pipeline

To have better understanding, and make it easier to filter, add the
current pipline class to every message logged in
BulkImports::Pipeline::Runner.

Also:
- add a info log message when the pipeline finishes.
- separete the pipeline step type from its class

** Before
```
{
  "severity":"",
  "time":"",
  "correlation_id":"",
  "bulk_import_entity_id":1,
  "bulk_import_entity_type":"",
  "extractor":""
}
```

** After
```
{
  "severity":"",
  "time":"",
  "correlation_id":"",
  "bulk_import_entity_id":1,
  "bulk_import_entity_type":"",
  "pipeline_class": "",
  "pipeline_step": "",
  "step_class": ""
}
```
parent f2b54f75
...@@ -10,7 +10,7 @@ module BulkImports ...@@ -10,7 +10,7 @@ module BulkImports
def run(context) def run(context)
raise MarkedAsFailedError if marked_as_failed?(context) raise MarkedAsFailedError if marked_as_failed?(context)
info(context, message: 'Pipeline started', pipeline_class: pipeline) info(context, message: 'Pipeline started')
extracted_data = extracted_data_from(context) extracted_data = extracted_data_from(context)
...@@ -27,6 +27,8 @@ module BulkImports ...@@ -27,6 +27,8 @@ module BulkImports
end end
after_run(context, extracted_data) if respond_to?(:after_run) after_run(context, extracted_data) if respond_to?(:after_run)
info(context, message: 'Pipeline finished')
rescue MarkedAsFailedError rescue MarkedAsFailedError
log_skip(context) log_skip(context)
end end
...@@ -36,7 +38,7 @@ module BulkImports ...@@ -36,7 +38,7 @@ module BulkImports
def run_pipeline_step(step, class_name, context) def run_pipeline_step(step, class_name, context)
raise MarkedAsFailedError if marked_as_failed?(context) raise MarkedAsFailedError if marked_as_failed?(context)
info(context, step => class_name) info(context, pipeline_step: step, step_class: class_name)
yield yield
rescue MarkedAsFailedError rescue MarkedAsFailedError
...@@ -100,7 +102,8 @@ module BulkImports ...@@ -100,7 +102,8 @@ module BulkImports
def log_base_params(context) def log_base_params(context)
{ {
bulk_import_entity_id: context.entity.id, bulk_import_entity_id: context.entity.id,
bulk_import_entity_type: context.entity.source_type bulk_import_entity_type: context.entity.source_type,
pipeline_class: pipeline
} }
end end
......
...@@ -74,28 +74,41 @@ RSpec.describe BulkImports::Pipeline::Runner do ...@@ -74,28 +74,41 @@ RSpec.describe BulkImports::Pipeline::Runner do
expect_next_instance_of(Gitlab::Import::Logger) do |logger| expect_next_instance_of(Gitlab::Import::Logger) do |logger|
expect(logger).to receive(:info) expect(logger).to receive(:info)
.with( .with(
bulk_import_entity_id: entity.id,
bulk_import_entity_type: 'group_entity',
message: 'Pipeline started', message: 'Pipeline started',
pipeline_class: 'BulkImports::MyPipeline', pipeline_class: 'BulkImports::MyPipeline'
)
expect(logger).to receive(:info)
.with(
bulk_import_entity_id: entity.id, bulk_import_entity_id: entity.id,
bulk_import_entity_type: 'group_entity' bulk_import_entity_type: 'group_entity',
pipeline_class: 'BulkImports::MyPipeline',
pipeline_step: :extractor,
step_class: 'BulkImports::Extractor'
) )
expect(logger).to receive(:info) expect(logger).to receive(:info)
.with( .with(
bulk_import_entity_id: entity.id, bulk_import_entity_id: entity.id,
bulk_import_entity_type: 'group_entity', bulk_import_entity_type: 'group_entity',
extractor: 'BulkImports::Extractor' pipeline_class: 'BulkImports::MyPipeline',
pipeline_step: :transformer,
step_class: 'BulkImports::Transformer'
) )
expect(logger).to receive(:info) expect(logger).to receive(:info)
.with( .with(
bulk_import_entity_id: entity.id, bulk_import_entity_id: entity.id,
bulk_import_entity_type: 'group_entity', bulk_import_entity_type: 'group_entity',
transformer: 'BulkImports::Transformer' pipeline_class: 'BulkImports::MyPipeline',
pipeline_step: :loader,
step_class: 'BulkImports::Loader'
) )
expect(logger).to receive(:info) expect(logger).to receive(:info)
.with( .with(
bulk_import_entity_id: entity.id, bulk_import_entity_id: entity.id,
bulk_import_entity_type: 'group_entity', bulk_import_entity_type: 'group_entity',
loader: 'BulkImports::Loader' message: 'Pipeline finished',
pipeline_class: 'BulkImports::MyPipeline'
) )
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment