Merge branch '26019-evidence-collection' into 'master'

Provide Release JSON schema for Evidence collection

See merge request gitlab-org/gitlab!17217

app/models/evidence.rb

+# frozen_string_literal: true
+
+class Evidence < ApplicationRecord
+  include ShaAttribute
+
+  belongs_to :release
+
+  before_validation :generate_summary_and_sha
+
+  default_scope { order(created_at: :asc) }
+
+  sha_attribute :summary_sha
+
+  def milestones
+    @milestones ||= release.milestones.includes(:issues)
+  end
+
+  private
+
+  def generate_summary_and_sha
+    summary = Evidences::EvidenceSerializer.new.represent(self) # rubocop: disable CodeReuse/Serializer
+    return unless summary
+
+    self.summary = summary
+    self.summary_sha = Gitlab::CryptoHelper.sha256(summary)
+  end
+end

app/models/release.rb

@@ -14,6 +14,7 @@ class Release < ApplicationRecord
 
   has_many :milestone_releases
   has_many :milestones, through: :milestone_releases
+  has_one :evidence
 
   default_value_for :released_at, allows_nil: false do
     Time.zone.now
@@ -28,6 +29,8 @@ class Release < ApplicationRecord
 
   delegate :repository, to: :project
 
+  after_commit :create_evidence!, on: :create
+
   def commit
     strong_memoize(:commit) do
       repository.commit(actual_sha)
@@ -66,6 +69,10 @@ class Release < ApplicationRecord
       repository.find_tag(tag)
     end
   end
+
+  def create_evidence!
+    CreateEvidenceWorker.perform_async(self.id)
+  end
 end
 
 Release.prepend_if_ee('EE::Release')

app/serializers/evidences/evidence_entity.rb

+# frozen_string_literal: true
+
+module Evidences
+  class EvidenceEntity < Grape::Entity
+    expose :release, using: Evidences::ReleaseEntity
+  end
+end

app/serializers/evidences/author_entity.rb → app/serializers/evidences/evidence_serializer.rb

 # frozen_string_literal: true
 
 module Evidences
-  class AuthorEntity < Grape::Entity
-    expose :id
-    expose :name
-    expose :email
+  class EvidenceSerializer < BaseSerializer
+    entity EvidenceEntity
   end
 end

app/serializers/evidences/issue_entity.rb

@@ -5,7 +5,6 @@ module Evidences
     expose :id
     expose :title
     expose :description
-    expose :author, using: AuthorEntity
     expose :state
     expose :iid
     expose :confidential

app/serializers/evidences/milestone_entity.rb

@@ -9,6 +9,6 @@ module Evidences
     expose :iid
     expose :created_at
     expose :due_date
-    expose :issues, using: IssueEntity
+    expose :issues, using: Evidences::IssueEntity
   end
 end

app/serializers/evidences/release_entity.rb

@@ -7,7 +7,7 @@ module Evidences
     expose :name
     expose :description
     expose :created_at
-    expose :project, using: ProjectEntity
-    expose :milestones, using: MilestoneEntity
+    expose :project, using: Evidences::ProjectEntity
+    expose :milestones, using: Evidences::MilestoneEntity
   end
 end

app/workers/all_queues.yml

@@ -173,3 +173,4 @@
 - delete_stored_files
 - import_issues_csv
 - project_daily_statistics
+- create_evidence

app/workers/create_evidence_worker.rb

+# frozen_string_literal: true
+
+class CreateEvidenceWorker
+  include ApplicationWorker
+
+  def perform(release_id)
+    release = Release.find_by_id(release_id)
+    return unless release
+
+    Evidence.create!(release: release)
+  end
+end

changelogs/unreleased/26019-evidence-collection.yml

+---
+title: Creation of Evidence collection of new releases.
+merge_request: 17217
+author:
+type: added

config/sidekiq_queues.yml

@@ -96,6 +96,7 @@
   - [phabricator_import_import_tasks, 1]
   - [update_namespace_statistics, 1]
   - [chaos, 2]
+  - [create_evidence, 2]
 
   # EE-specific queues
   - [ldap_group_sync, 2]

db/migrate/20190919091300_create_evidences.rb

+# frozen_string_literal: true
+
+class CreateEvidences < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+
+  def change
+    create_table :evidences do |t|
+      t.references :release, foreign_key: { on_delete: :cascade }, null: false
+      t.timestamps_with_timezone
+      t.binary :summary_sha
+      t.jsonb :summary, null: false, default: {}
+    end
+  end
+end

db/schema.rb

@@ -1424,6 +1424,15 @@ ActiveRecord::Schema.define(version: 2019_10_16_072826) do
     t.index ["target_type", "target_id"], name: "index_events_on_target_type_and_target_id"
   end
 
+  create_table "evidences", force: :cascade do |t|
+    t.bigint "release_id", null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.binary "summary_sha"
+    t.jsonb "summary", default: {}, null: false
+    t.index ["release_id"], name: "index_evidences_on_release_id"
+  end
+
   create_table "external_pull_requests", force: :cascade do |t|
     t.datetime_with_timezone "created_at", null: false
     t.datetime_with_timezone "updated_at", null: false
@@ -4079,6 +4088,7 @@ ActiveRecord::Schema.define(version: 2019_10_16_072826) do
   add_foreign_key "events", "namespaces", column: "group_id", name: "fk_61fbf6ca48", on_delete: :cascade
   add_foreign_key "events", "projects", on_delete: :cascade
   add_foreign_key "events", "users", column: "author_id", name: "fk_edfd187b6f", on_delete: :cascade
+  add_foreign_key "evidences", "releases", on_delete: :cascade
   add_foreign_key "external_pull_requests", "projects", on_delete: :cascade
   add_foreign_key "fork_network_members", "fork_networks", on_delete: :cascade
   add_foreign_key "fork_network_members", "projects", column: "forked_from_project_id", name: "fk_b01280dae4", on_delete: :nullify

spec/factories/evidences.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :evidence do
+    release
+  end
+end

spec/fixtures/api/schemas/evidences/author.json → spec/fixtures/api/schemas/evidences/evidence.json

 {
   "type": "object",
   "required": [
-    "id",
-    "name",
-    "email"
+    "release"
   ],
   "properties": {
-    "id": { "type": "integer" },
-    "name": { "type": "string" },
-    "email": { "type": "string" }
+    "release": { "$ref": "release.json" }
   },
   "additionalProperties": false
 }
+

spec/fixtures/api/schemas/evidences/issue.json

@@ -14,13 +14,12 @@
   "properties": {
     "id": { "type": "integer" },
     "title": { "type": "string" },
-    "description": { "type": "string" },
-    "author": { "$ref": "author.json" },
+    "description": { "type": ["string", "null"] },
     "state": { "type": "string" },
     "iid": { "type": "integer" },
     "confidential": { "type": "boolean" },
     "created_at": { "type": "date" },
-    "due_date": { "type": "date" }
+    "due_date": { "type": ["date", "null"] }
   },
   "additionalProperties": false
 }

spec/fixtures/api/schemas/evidences/milestone.json

@@ -13,11 +13,11 @@
   "properties": {
     "id": { "type": "integer" },
     "title": { "type": "string" },
-    "description": { "type": "string" },
+    "description": { "type": ["string", "null"] },
     "state": { "type": "string" },
     "iid": { "type": "integer" },
     "created_at": { "type": "date" },
-    "due_date": { "type": "date" },
+    "due_date": { "type": ["date", "null"] },
     "issues": {
       "type": "array",
       "items": { "$ref": "issue.json" }

spec/fixtures/api/schemas/evidences/project.json

@@ -9,7 +9,7 @@
   "properties": {
     "id": { "type": "integer" },
     "name": { "type": "string" },
-    "description": { "type": "string" },
+    "description": { "type": ["string", "null"] },
     "created_at": { "type": "date" }
   },
   "additionalProperties": false

spec/fixtures/api/schemas/evidences/release.json

@@ -2,7 +2,7 @@
   "type": "object",
   "required": [
     "id",
-    "tag",
+    "tag_name",
     "name",
     "description",
     "created_at",
@@ -11,8 +11,8 @@
   ],
   "properties": {
     "id": { "type": "integer" },
-    "tag": { "type": "string" },
-    "name": { "type": "string" },
+    "tag_name": { "type": "string" },
+    "name": { "type": ["string", "null"] },
     "description": { "type": "string" },
     "created_at": { "type": "date" },
     "project": { "$ref": "project.json" },

spec/lib/gitlab/import_export/all_models.yml

@@ -81,6 +81,7 @@ releases:
 - links
 - milestone_releases
 - milestones
+- evidence
 links:
 - release
 project_members:
@@ -506,6 +507,8 @@ lists:
 milestone_releases:
 - milestone
 - release
+evidences:
+- release
 design: &design
 - issue
 - actions

spec/lib/gitlab/import_export/safe_model_attributes.yml

@@ -127,6 +127,12 @@ Release:
 - created_at
 - updated_at
 - released_at
+Evidence:
+- id
+- release_id
+- summary
+- created_at
+- updated_at
 Releases::Link:
 - id
 - release_id

spec/models/evidence_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Evidence do
+  let_it_be(:project) { create(:project) }
+  let(:release) { create(:release, project: project) }
+  let(:schema_file) { 'evidences/evidence' }
+  let(:summary_json) { described_class.last.summary.to_json }
+
+  describe 'associations' do
+    it { is_expected.to belong_to(:release) }
+  end
+
+  describe 'summary_sha' do
+    it 'returns nil if summary is nil' do
+      expect(build(:evidence, summary: nil).summary_sha).to be_nil
+    end
+  end
+
+  describe '#generate_summary_and_sha' do
+    before do
+      described_class.create!(release: release)
+    end
+
+    context 'when a release name is not provided' do
+      let(:release) { create(:release, project: project, name: nil) }
+
+      it 'creates a valid JSON object' do
+        expect(release.name).to be_nil
+        expect(summary_json).to match_schema(schema_file)
+      end
+    end
+
+    context 'when a release is associated to a milestone' do
+      let(:milestone) { create(:milestone, project: project) }
+      let(:release) { create(:release, project: project, milestones: [milestone]) }
+
+      context 'when a milestone has no issue associated with it' do
+        it 'creates a valid JSON object' do
+          expect(milestone.issues).to be_empty
+          expect(summary_json).to match_schema(schema_file)
+        end
+      end
+
+      context 'when a milestone has no description' do
+        let(:milestone) { create(:milestone, project: project, description: nil) }
+
+        it 'creates a valid JSON object' do
+          expect(milestone.description).to be_nil
+          expect(summary_json).to match_schema(schema_file)
+        end
+      end
+
+      context 'when a milestone has no due_date' do
+        let(:milestone) { create(:milestone, project: project, due_date: nil) }
+
+        it 'creates a valid JSON object' do
+          expect(milestone.due_date).to be_nil
+          expect(summary_json).to match_schema(schema_file)
+        end
+      end
+
+      context 'when a milestone has an issue' do
+        context 'when the issue has no description' do
+          let(:issue) { create(:issue, project: project, description: nil, state: 'closed') }
+
+          before do
+            milestone.issues << issue
+          end
+
+          it 'creates a valid JSON object' do
+            expect(milestone.issues.first.description).to be_nil
+            expect(summary_json).to match_schema(schema_file)
+          end
+        end
+      end
+    end
+
+    context 'when a release is not associated to any milestone' do
+      it 'creates a valid JSON object' do
+        expect(release.milestones).to be_empty
+        expect(summary_json).to match_schema(schema_file)
+      end
+    end
+  end
+end

spec/models/release_spec.rb

@@ -15,11 +15,13 @@ RSpec.describe Release do
     it { is_expected.to have_many(:links).class_name('Releases::Link') }
     it { is_expected.to have_many(:milestones) }
     it { is_expected.to have_many(:milestone_releases) }
+    it { is_expected.to have_one(:evidence) }
   end
 
   describe 'validation' do
     it { is_expected.to validate_presence_of(:project) }
     it { is_expected.to validate_presence_of(:description) }
+    it { is_expected.to validate_presence_of(:tag) }
 
     context 'when a release exists in the database without a name' do
       it 'does not require name' do
@@ -89,4 +91,22 @@ RSpec.describe Release do
       end
     end
   end
+
+  describe 'evidence' do
+    describe '#create_evidence!' do
+      context 'when a release is created' do
+        it 'creates one Evidence object too' do
+          expect { release }.to change(Evidence, :count).by(1)
+        end
+      end
+    end
+
+    context 'when a release is deleted' do
+      it 'also deletes the associated evidence' do
+        release = create(:release)
+
+        expect { release.destroy }.to change(Evidence, :count).by(-1)
+      end
+    end
+  end
 end

spec/serializers/evidences/author_entity_spec.rb → spec/serializers/evidences/evidence_entity_spec.rb

@@ -2,12 +2,13 @@
 
 require 'spec_helper'
 
-describe Evidences::AuthorEntity do
-  let(:entity) { described_class.new(build(:author)) }
+describe Evidences::EvidenceEntity do
+  let(:evidence) { build(:evidence) }
+  let(:entity) { described_class.new(evidence) }
 
   subject { entity.as_json }
 
   it 'exposes the expected fields' do
-    expect(subject.keys).to contain_exactly(:id, :name, :email)
+    expect(subject.keys).to contain_exactly(:release)
   end
 end

spec/serializers/evidences/evidence_serializer_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Evidences::EvidenceSerializer do
+  it 'represents an EvidenceEntity entity' do
+    expect(described_class.entity_class).to eq(Evidences::EvidenceEntity)
+  end
+end

spec/serializers/evidences/issue_entity_spec.rb

@@ -8,6 +8,6 @@ describe Evidences::IssueEntity do
   subject { entity.as_json }
 
   it 'exposes the expected fields' do
-    expect(subject.keys).to contain_exactly(:id, :title, :description, :author, :state, :iid, :confidential, :created_at, :due_date)
+    expect(subject.keys).to contain_exactly(:id, :title, :description, :state, :iid, :confidential, :created_at, :due_date)
   end
 end

spec/serializers/evidences/milestone_entity_spec.rb

@@ -12,7 +12,7 @@ describe Evidences::MilestoneEntity do
     expect(subject.keys).to contain_exactly(:id, :title, :description, :state, :iid, :created_at, :due_date, :issues)
   end
 
-  context 'when there issues linked to this milestone' do
+  context 'when there are issues linked to this milestone' do
     let(:issue_1) { build(:issue) }
     let(:issue_2) { build(:issue) }
     let(:milestone) { build(:milestone, issues: [issue_1, issue_2]) }

spec/support/shared_examples/evidence_updated_exposed_fields.rb

+# frozen_string_literal: true
+
+shared_examples 'updated exposed field' do
+  it 'creates another Evidence object' do
+    model.send("#{updated_field}=", updated_value)
+
+    expect(model.evidence_summary_keys).to include(updated_field)
+    expect { model.save! }.to change(Evidence, :count).by(1)
+    expect(updated_json_field).to eq(updated_value)
+  end
+end
+
+shared_examples 'updated non-exposed field' do
+  it 'does not create any Evidence object' do
+    model.send("#{updated_field}=", updated_value)
+
+    expect(model.evidence_summary_keys).not_to include(updated_field)
+    expect { model.save! }.not_to change(Evidence, :count)
+  end
+end
+
+shared_examples 'updated field on non-linked entity' do
+  it 'does not create any Evidence object' do
+    model.send("#{updated_field}=", updated_value)
+
+    expect(model.evidence_summary_keys).to be_empty
+    expect { model.save! }.not_to change(Evidence, :count)
+  end
+end

spec/workers/create_evidence_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe CreateEvidenceWorker do
+  let!(:release) { create(:release) }
+
+  it 'creates a new Evidence' do
+    expect { described_class.new.perform(release.id) }.to change(Evidence, :count).by(1)
+  end
+end