Fix cloud native job logs not finalizing with Azure

Previously the cloud native job log (aka CI live trace) Fog
implementation used the `put_object` and `delete_object` API
calls. These are object-storage specific API calls, and they work for
AWS and Google.

However, Fog has a more general API that works for all providers. We
already use this API for backups. To make live job logs work with Azure,
we use that general API.

Relates to https://gitlab.com/gitlab-org/gitlab/-/issues/272853

app/models/ci/build_trace_chunks/fog.rb

@@ -8,13 +8,17 @@ module Ci
       end
 
       def data(model)
-        connection.get_object(bucket_name, key(model))[:body]
+        files.get(key(model))&.body
       rescue Excon::Error::NotFound
         # If the object does not exist in the object storage, this method returns nil.
       end
 
       def set_data(model, new_data)
-        connection.put_object(bucket_name, key(model), new_data)
+        # TODO: Support AWS S3 server side encryption
+        files.create({
+          key: key(model),
+          body: new_data
+        })
       end
 
       def append_data(model, new_data, offset)
@@ -43,7 +47,7 @@ module Ci
 
       def delete_keys(keys)
         keys.each do |key|
-          connection.delete_object(bucket_name, key_raw(*key))
+          files.destroy(key_raw(*key))
         end
       end
 
@@ -69,6 +73,14 @@ module Ci
         @connection ||= ::Fog::Storage.new(object_store.connection.to_hash.deep_symbolize_keys)
       end
 
+      def fog_directory
+        @fog_directory ||= connection.directories.new(key: bucket_name)
+      end
+
+      def files
+        @files ||= fog_directory.files
+      end
+
       def object_store
         Gitlab.config.artifacts.object_store
       end

changelogs/unreleased/sh-fix-live-trace-for-fog-azure.yml

+---
+title: Fix cloud native job logs not finalizing with Azure
+merge_request: 46209
+author:
+type: fixed

spec/models/ci/build_trace_chunks/fog_spec.rb

@@ -4,8 +4,12 @@ require 'spec_helper'
 
 RSpec.describe Ci::BuildTraceChunks::Fog do
   let(:data_store) { described_class.new }
+  let(:bucket) { 'artifacts' }
+  let(:connection_params) { Gitlab.config.artifacts.object_store.connection.symbolize_keys }
+  let(:connection) { ::Fog::Storage.new(connection_params) }
 
   before do
+    stub_object_storage(connection_params: connection_params, remote_directory: bucket)
     stub_artifacts_object_storage
   end
 
@@ -148,17 +152,17 @@ RSpec.describe Ci::BuildTraceChunks::Fog do
     end
 
     it 'deletes multiple data' do
-      ::Fog::Storage.new(JobArtifactUploader.object_store_credentials).tap do |connection|
-        expect(connection.get_object('artifacts', "tmp/builds/#{build.id}/chunks/0.log")[:body]).to be_present
-        expect(connection.get_object('artifacts', "tmp/builds/#{build.id}/chunks/1.log")[:body]).to be_present
-      end
+      files = connection.directories.new(key: bucket).files
+
+      expect(files.count).to eq(2)
+      expect(files[0].body).to be_present
+      expect(files[1].body).to be_present
 
       subject
 
-      ::Fog::Storage.new(JobArtifactUploader.object_store_credentials).tap do |connection|
-        expect { connection.get_object('artifacts', "tmp/builds/#{build.id}/chunks/0.log")[:body] }.to raise_error(Excon::Error::NotFound)
-        expect { connection.get_object('artifacts', "tmp/builds/#{build.id}/chunks/1.log")[:body] }.to raise_error(Excon::Error::NotFound)
-      end
+      files.reload
+
+      expect(files.count).to eq(0)
     end
   end
 end