Add API for generating Markdown changelogs

This adds an API that allows users to generate Markdown changelogs,
using Git commit messages and Git trailers as the input. The API is
hidden behind a feature flag called "changelog_api", and is disabled by
default.

For more information, see merge request
https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52116 and epic
https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/351.

app/finders/merge_requests/oldest_per_commit_finder.rb

+# frozen_string_literal: true
+
+module MergeRequests
+  # OldestPerCommitFinder is used to retrieve the oldest merge requests for
+  # every given commit, grouped per commit SHA.
+  #
+  # This finder is useful when you need to efficiently retrieve the first/oldest
+  # merge requests for multiple commits, and you want to do so in batches;
+  # instead of running a query for every commit.
+  class OldestPerCommitFinder
+    def initialize(project)
+      @project = project
+    end
+
+    # Returns a Hash that maps a commit ID to the oldest merge request that
+    # introduced that commit.
+    def execute(commits)
+      id_rows = MergeRequestDiffCommit
+        .oldest_merge_request_id_per_commit(@project.id, commits.map(&:id))
+
+      mrs = MergeRequest
+        .preload_target_project
+        .id_in(id_rows.map { |r| r[:merge_request_id] })
+        .index_by(&:id)
+
+      id_rows.each_with_object({}) do |row, hash|
+        if (mr = mrs[row[:merge_request_id]])
+          hash[row[:sha]] = mr
+        end
+      end
+    end
+  end
+end

app/models/merge_request_diff_commit.rb

@@ -35,4 +35,23 @@ class MergeRequestDiffCommit < ApplicationRecord
 
     Gitlab::Database.bulk_insert(self.table_name, rows) # rubocop:disable Gitlab/BulkInsert
   end
+
+  def self.oldest_merge_request_id_per_commit(project_id, shas)
+    # This method is defined here and not on MergeRequest, otherwise the SHA
+    # values used in the WHERE below won't be encoded correctly.
+    select(['merge_request_diff_commits.sha AS sha', 'min(merge_requests.id) AS merge_request_id'])
+      .joins(:merge_request_diff)
+      .joins(
+        'INNER JOIN merge_requests ' \
+          'ON merge_requests.latest_merge_request_diff_id = merge_request_diffs.id'
+      )
+      .where(sha: shas)
+      .where(
+        merge_requests: {
+          target_project_id: project_id,
+          state_id: MergeRequest.available_states[:merged]
+        }
+      )
+      .group(:sha)
+  end
 end

app/services/repositories/changelog_service.rb

+# frozen_string_literal: true
+
+module Repositories
+  # A service class for generating a changelog section.
+  class ChangelogService
+    DEFAULT_TRAILER = 'Changelog'
+    DEFAULT_FILE = 'CHANGELOG.md'
+
+    # The `project` specifies the `Project` to generate the changelog section
+    # for.
+    #
+    # The `user` argument specifies a `User` to use for committing the changes
+    # to the Git repository.
+    #
+    # The `version` arguments must be a version `String` using semantic
+    # versioning as the format.
+    #
+    # The arguments `from` and `to` must specify a Git ref or SHA to use for
+    # fetching the commits to include in the changelog. The SHA/ref set in the
+    # `from` argument isn't included in the list.
+    #
+    # The `date` argument specifies the date of the release, and defaults to the
+    # current time/date.
+    #
+    # The `branch` argument specifies the branch to commit the changes to. The
+    # branch must already exist.
+    #
+    # The `trailer` argument is the Git trailer to use for determining what
+    # commits to include in the changelog.
+    #
+    # The `file` arguments specifies the name/path of the file to commit the
+    # changes to. If the file doesn't exist, it's created automatically.
+    #
+    # The `message` argument specifies the commit message to use when committing
+    # the changelog changes.
+    #
+    # rubocop: disable Metrics/ParameterLists
+    def initialize(
+      project,
+      user,
+      version:,
+      from:,
+      to:,
+      date: DateTime.now,
+      branch: project.default_branch_or_master,
+      trailer: DEFAULT_TRAILER,
+      file: DEFAULT_FILE,
+      message: "Add changelog for version #{version}"
+    )
+      @project = project
+      @user = user
+      @version = version
+      @from = from
+      @to = to
+      @date = date
+      @branch = branch
+      @trailer = trailer
+      @file = file
+      @message = message
+    end
+    # rubocop: enable Metrics/ParameterLists
+
+    def execute
+      # For every entry we want to only include the merge request that
+      # originally introduced the commit, which is the oldest merge request that
+      # contains the commit. We fetch there merge requests in batches, reducing
+      # the number of SQL queries needed to get this data.
+      mrs_finder = MergeRequests::OldestPerCommitFinder.new(@project)
+      config = Gitlab::Changelog::Config.from_git(@project)
+      release = Gitlab::Changelog::Release
+        .new(version: @version, date: @date, config: config)
+
+      commits =
+        CommitsWithTrailerFinder.new(project: @project, from: @from, to: @to)
+
+      commits.each_page(@trailer) do |page|
+        mrs = mrs_finder.execute(page)
+
+        # Preload the authors. This ensures we only need a single SQL query per
+        # batch of commits, instead of needing a query for every commit.
+        page.each(&:lazy_author)
+
+        page.each do |commit|
+          release.add_entry(
+            title: commit.title,
+            commit: commit,
+            category: commit.trailers.fetch(@trailer),
+            author: commit.author,
+            merge_request: mrs[commit.id]
+          )
+        end
+      end
+
+      Gitlab::Changelog::Committer
+        .new(@project, @user)
+        .commit(release: release, file: @file, branch: @branch, message: @message)
+    end
+  end
+end

changelogs/unreleased/changelog-api.yml

+---
+title: Add API for generating Markdown changelogs
+merge_request: 52116
+author:
+type: added

config/feature_flags/development/changelog_api.yml

+---
+name: changelog_api
+introduced_by_url: '13.9'
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/300043
+milestone: '13.9'
+type: development
+group: group::source code
+default_enabled: false

db/migrate/20210127143025_add_oldest_merge_requests_index.rb

+# frozen_string_literal: true
+
+class AddOldestMergeRequestsIndex < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::SchemaHelpers
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  INDEX = 'index_on_merge_requests_for_latest_diffs'
+
+  def up
+    return if index_exists_by_name?('merge_requests', INDEX)
+
+    execute "CREATE INDEX CONCURRENTLY #{INDEX} ON merge_requests " \
+      'USING btree (target_project_id) INCLUDE (id, latest_merge_request_diff_id)'
+
+    create_comment(
+      'INDEX',
+      INDEX,
+      'Index used to efficiently obtain the oldest merge request for a commit SHA'
+    )
+  end
+
+  def down
+    return unless index_exists_by_name?('merge_requests', INDEX)
+
+    execute "DROP INDEX CONCURRENTLY #{INDEX}"
+  end
+end

db/schema_migrations/20210127143025

+c173ba86340efe39977f1b319d1ebcead634e3bfe819a30e230fb4f81766f28a
\ No newline at end of file

db/structure.sql

@@ -22433,6 +22433,10 @@ CREATE UNIQUE INDEX index_on_instance_statistics_recorded_at_and_identifier ON a
 
 CREATE INDEX index_on_label_links_all_columns ON label_links USING btree (target_id, label_id, target_type);
 
+CREATE INDEX index_on_merge_requests_for_latest_diffs ON merge_requests USING btree (target_project_id) INCLUDE (id, latest_merge_request_diff_id);
+
+COMMENT ON INDEX index_on_merge_requests_for_latest_diffs IS 'Index used to efficiently obtain the oldest merge request for a commit SHA';
+
 CREATE INDEX index_on_namespaces_lower_name ON namespaces USING btree (lower((name)::text));
 
 CREATE INDEX index_on_namespaces_lower_path ON namespaces USING btree (lower((path)::text));

doc/api/repositories.md

@@ -5,7 +5,7 @@ info: "To determine the technical writer assigned to the Stage/Group associated
 type: reference, api
 ---
 
-# Repositories API
+# Repositories API **(CORE)**
 
 ## List repository tree
 
@@ -18,14 +18,15 @@ This command provides essentially the same functionality as the `git ls-tree` co
 GET /projects/:id/repository/tree
 ```
 
-Parameters:
+Supported attributes:
 
-- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
-- `path` (optional) - The path inside repository. Used to get content of subdirectories
-- `ref` (optional) - The name of a repository branch or tag or if not given the default branch
-- `recursive` (optional) - Boolean value used to get a recursive tree (false by default)
-- `per_page` (optional) - Number of results to show per page. If not specified, defaults to `20`.
-  Read more on [pagination](README.md#pagination).
+| Attribute   | Type           | Required | Description |
+| :---------- | :------------- | :------- | :---------- |
+| `id`        | integer/string | no       | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
+| `path`      | string         | yes      | The path inside repository. Used to get content of subdirectories. |
+| `ref`       | string         | yes      | The name of a repository branch or tag or if not given the default branch. |
+| `recursive` | boolean        | yes      | Boolean value used to get a recursive tree (false by default). |
+| `per_page`  | integer        | yes      | Number of results to show per page. If not specified, defaults to `20`. [Learn more on pagination](README.md#pagination). |
 
 ```json
 [
@@ -91,10 +92,12 @@ without authentication if the repository is publicly accessible.
 GET /projects/:id/repository/blobs/:sha
 ```
 
-Parameters:
+Supported attributes:
 
-- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
-- `sha` (required) - The blob SHA
+| Attribute | Type           | Required | Description |
+| :-------- | :------------- | :------- | :---------- |
+| `id`      | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
+| `sha`     | string         | yes      | The blob SHA. |
 
 ## Raw blob content
 
@@ -105,10 +108,12 @@ without authentication if the repository is publicly accessible.
 GET /projects/:id/repository/blobs/:sha/raw
 ```
 
-Parameters:
+Supported attributes:
 
-- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
-- `sha` (required) - The blob SHA
+| Attribute | Type     | Required | Description |
+| :-------- | :------- | :------- | :---------- |
+| `id`      | datatype | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
+| `sha`     | datatype | yes      | The blob SHA. |
 
 ## Get file archive
 
@@ -128,10 +133,14 @@ GET /projects/:id/repository/archive[.format]
 `bz2`, `tar`, and `zip`. For example, specifying `archive.zip`
 would send an archive in ZIP format.
 
-Parameters:
+Supported attributes:
+
+| Attribute   | Type           | Required | Description           |
+|:------------|:---------------|:---------|:----------------------|
+| `id`        | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
+| `sha`       | string         | no       | The commit SHA to download. A tag, branch reference, or SHA can be used. This defaults to the tip of the default branch if not specified. |
 
-- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
-- `sha` (optional) - The commit SHA to download. A tag, branch reference, or SHA can be used. This defaults to the tip of the default branch if not specified. For example:
+Example request:
 
 ```shell
 curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.com/api/v4/projects/<project_id>/repository/archive?sha=<commit_sha>"
@@ -146,21 +155,22 @@ publicly accessible. Note that diffs could have an empty diff string if [diff li
 GET /projects/:id/repository/compare
 ```
 
-Parameters:
+Supported attributes:
 
-- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
-- `from` (required) - the commit SHA or branch name
-- `to` (required) - the commit SHA or branch name
-- `straight` (optional) - comparison method, `true` for direct comparison between `from` and `to` (`from`..`to`), `false` to compare using merge base (`from`...`to`)'. Default is `false`.
+| Attribute  | Type           | Required | Description |
+| :--------- | :------------- | :------- | :---------- |
+| `id`       | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
+| `from`     | string         | yes      | The commit SHA or branch name. |
+| `to`       | string         | yes      | The commit SHA or branch name. |
+| `straight` | boolean        | no       | Comparison method, `true` for direct comparison between `from` and `to` (`from`..`to`), `false` to compare using merge base (`from`...`to`)'. Default is `false`. |
 
 ```plaintext
 GET /projects/:id/repository/compare?from=master&to=feature
 ```
 
-Response:
+Example response:
 
 ```json
-
 {
   "commit": {
     "id": "12d65c8dd2b2676fa3ac47d955accc085a37a9c1",
@@ -203,15 +213,17 @@ GET /projects/:id/repository/contributors
 ```
 
 WARNING:
-The `additions` and `deletions` attributes are deprecated [as of GitLab 13.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39653) because they [always return `0`](https://gitlab.com/gitlab-org/gitlab/-/issues/233119).
+The `additions` and `deletions` attributes are deprecated [as of GitLab 13.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39653), because they [always return `0`](https://gitlab.com/gitlab-org/gitlab/-/issues/233119).
 
-Parameters:
+Supported attributes:
 
-- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
-- `order_by` (optional) - Return contributors ordered by `name`, `email`, or `commits` (orders by commit date) fields. Default is `commits`
-- `sort` (optional) - Return contributors sorted in `asc` or `desc` order. Default is `asc`
+| Attribute  | Type           | Required | Description |
+| :--------- | :------------- | :------- | :---------- |
+| `id`       | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
+| `order_by` | string         | no       | Return contributors ordered by `name`, `email`, or `commits` (orders by commit date) fields. Default is `commits`. |
+| `sort`     | string         | no       | Return contributors sorted in `asc` or `desc` order. Default is `asc`. |
 
-Response:
+Example response:
 
 ```json
 [{
@@ -238,10 +250,12 @@ GET /projects/:id/repository/merge_base
 ```
 
 | Attribute | Type           | Required | Description |
-| --------- | ---- | -------- | ----------- |
+| --------- | -------------- | -------- | ------------------------------------------------------------------------------- |
 | `id`      | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) |
 | `refs`    | array          | yes      | The refs to find the common ancestor of, multiple refs can be passed            |
 
+Example request:
+
 ```shell
 curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/repository/merge_base?refs[]=304d257dcb821665ab5110318fc58a007bd104ed&refs[]=0031876facac3f2b2702a0e53a26e89939a42209"
 ```
@@ -264,3 +278,252 @@ Example response:
   "committed_date": "2014-02-27T08:03:18.000Z"
 }
 ```
+
+## Generate changelog data
+
+> - [Introduced](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/351) in GitLab 13.9.
+> - It's [deployed behind a feature flag](../user/feature_flags.md), disabled by default.
+> - It's disabled on GitLab.com.
+> - It's not yet recommended for production use.
+> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-generating-changelog-data).
+
+WARNING:
+This feature might not be available to you. Check the **version history** note above for details.
+
+Generate changelog data based on commits in a repository.
+
+Given a version (using semantic versioning) and a range of commits,
+GitLab generates a changelog for all commits that use a particular
+[Git trailer](https://git-scm.com/docs/git-interpret-trailers).
+
+The output of this process is a new section in a changelog file in the Git
+repository of the given project. The output format is in Markdown, and can be
+customized.
+
+```plaintext
+POST /projects/:id/repository/changelog
+```
+
+Supported attributes:
+
+| Attribute | Type     | Required   | Description |
+| :-------- | :------- | :--------- | :---------- |
+| `version` | string   | yes | The version to generate the changelog for. The format must follow [semantic versioning](https://semver.org/). |
+| `from`    | string   | yes | The start of the range of commits (as a SHA) to use for generating the changelog. This commit itself isn't included in the list. |
+| `to`      | string   | yes | The end of the range of commits (as a SHA) to use for the changelog. This commit _is_ included in the list. |
+| `date`    | datetime | no | The date and time of the release, defaults to the current time. |
+| `branch`  | string   | no | The branch to commit the changelog changes to, defaults to the project's default branch. |
+| `trailer` | string   | no | The Git trailer to use for including commits, defaults to `Changelog`. |
+| `file`    | string   | no | The file to commit the changes to, defaults to `CHANGELOG.md`. |
+| `message` | string   | no | The commit message to produce when committing the changes, defaults to `Add changelog for version X` where X is the value of the `version` argument. |
+
+### How it works
+
+Changelogs are generated based on commit titles. Commits are only included if
+they contain a specific Git trailer. GitLab uses the value of this trailer to
+categorize the changes.
+
+GitLab uses Git trailers, because Git trailers are
+supported by Git out of the box. We use commits as input, as this is the only
+source of data every project uses. In addition, commits can be retrieved when
+operating on a mirror. This is important for GitLab itself, because during a security
+release we might need to include changes from both public projects and private
+security mirrors.
+
+Changelogs are generated by taking the title of the commits to include and using
+these as the changelog entries. You can enrich entries with additional data,
+such as a link to the merge request or details about the commit author. You can
+[customize the format of a changelog](#customize-the-changelog-output) section with a template.
+
+### Customize the changelog output
+
+The output is customized using a YAML configuration file stored in your
+project's Git repository. This file must reside in
+`.gitlab/changelog_config.yml`.
+
+You can set the following variables in this file:
+
+- `date_format`: the date format to use in the title of the newly added
+  changelog data. This uses regular `strftime` formatting.
+- `template`: a custom template to use for generating the changelog data.
+- `categories`: a hash that maps raw category names to the names to use in the
+  changelog.
+
+Using the default settings, generating a changelog results in a section along
+the lines of the following:
+
+```markdown
+## 1.0.0 (2021-01-05)
+
+### Features (4 changes)
+
+- [Feature 1](gitlab-org/gitlab@123abc) by @alice ([merge request](gitlab-org/gitlab!123))
+- [Feature 2](gitlab-org/gitlab@456abc) ([merge request](gitlab-org/gitlab!456))
+- [Feature 3](gitlab-org/gitlab@234abc) by @steve
+- [Feature 4](gitlab-org/gitlab@456)
+```
+
+Each section starts with a title that contains the version and release date.
+While the format of the date can be customized, the rest of the title can't be
+changed. When adding a new section, GitLab parses these titles to determine
+where in the file the new section should be placed. GitLab sorts sections
+according to their versions, not their dates.
+
+Each section can have categories, each with their
+corresponding changes. In the above example, "Features" is one such category.
+You can customize the format of these sections.
+
+The section names are derived from the values of the Git trailer used to include
+or exclude commits.
+
+For example, if the trailer to use is called `Changelog`,
+and its value is `feature`, then the commit is grouped in the `feature`
+category. The names of these raw values might differ from what you want to
+show in a changelog, you can remap them. Let's say we use the `Changelog`
+trailer and developers use the following values: `feature`, `bug`, and
+`performance`.
+
+You can remap these using the following YAML configuration file:
+
+```yaml
+---
+categories:
+  feature: Features
+  bug: Bug fixes
+  performance: Performance improvements
+```
+
+When generating the changelog data, the category titles are then `### Features`,
+`### Bug fixes`, and `### Performance improvements`.
+
+### Custom templates
+
+The category sections are generated using a template. The default template is as
+follows:
+
+```plaintext
+{% if categories %}
+{% each categories %}
+### {{ title }} ({% if single_change %}1 change{% else %}{{ count }} changes{% end %})
+
+{% each entries %}
+- [{{ title }}]({{ commit.reference }})\
+{% if author.contributor %} by {{ author.reference }}{% end %}\
+{% if merge_request %} ([merge request]({{ merge_request.reference }})){% end %}
+{% end %}
+
+{% end %}
+{% else %}
+No changes.
+{% end %}
+```
+
+The `{% ... %}` tags are for statements, and `{{ ... }}` is used for printing
+data. Statements must be terminated using a `{% end %}` tag. Both the `if` and
+`each` statements require a single argument.
+
+For example, if we have a variable `valid`, and we want to display "yes"
+when this value is true, and display "nope" otherwise. We can do so as follows:
+
+```plaintext
+{% if valid %}
+yes
+{% else %}
+nope
+{% end %}
+```
+
+The use of `else` is optional. A value is considered true when it's a non-empty
+value or boolean `true`. Empty arrays and hashes are considered false.
+
+Looping is done using `each`, and variables inside a loop are scoped to it.
+Referring to the current value in a loop is done using the variable tag `{{ it
+}}`. Other variables read their value from the current loop value. Take
+this template for example:
+
+```plaintext
+{% each users %}
+{{name}}
+{% end %}
+```
+
+Assuming `users` is an array of objects, each with a `name` field, this would
+then print the name of every user.
+
+Using variable tags, you can access nested objects. For example, `{{
+users.0.name }}` prints the name of the first user in the `users` variable.
+
+If a line ends in a backslash, the next newline is ignored. This allows you to
+wrap code across multiple lines, without introducing unnecessary newlines in the
+Markdown output.
+
+You can specify a custom template in your configuration like so:
+
+```yaml
+---
+template: >
+  {% if categories %}
+  {% each categories %}
+  ### {{ title }}
+
+  {% each entries %}
+  - [{{ title }}]({{ commit.reference }})\
+  {% if author.contributor %} by {{ author.reference }}{% end %}
+  {% end %}
+
+  {% end %}
+  {% else %}
+  No changes.
+  {% end %}
+```
+
+### Template data
+
+At the top level, the following variable is available:
+
+- `categories`: an array of objects, one for every changelog category.
+
+In a category, the following variables are available:
+
+- `title`: the title of the category (after it has been remapped).
+- `count`: the number of entries in this category.
+- `single_change`: a boolean that indicates if there is only one change (`true`),
+  or multiple changes (`false`).
+- `entries`: the entries that belong to this category.
+
+In an entry, the following variables are available (here `foo.bar` means that
+`bar` is a sub-field of `foo`):
+
+- `title`: the title of the changelog entry (this is the commit title).
+- `commit.reference`: a reference to the commit, for example,
+  `gitlab-org/gitlab@0a4cdd86ab31748ba6dac0f69a8653f206e5cfc7`.
+- `commit.trailers`: an object containing all the Git trailers that were present
+  in the commit body.
+- `author.reference`: a reference to the commit author (for example, `@alice`).
+- `author.contributor`: a boolean set to `true` when the author is an external
+  contributor, otherwise this is set to `false`.
+- `merge_request.reference`: a reference to the merge request that first
+  introduced the change (for example, `gitlab-org/gitlab!50063`).
+
+The `author` and `merge_request` objects might not be present if the data couldn't
+be determined (for example, when a commit was created without a corresponding merge
+request).
+
+### Enable or disable generating changelog data **(CORE ONLY)**
+
+This feature is under development and not ready for production use. It is
+deployed behind a feature flag that is **disabled by default**.
+[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md)
+can enable it.
+
+To enable it for a project:
+
+```ruby
+Feature.enable(:changelog_api, Project.find(id_of_the_project))
+```
+
+To disable it for a project:
+
+```ruby
+Feature.disable(:changelog_api, Project.find(id_of_the_project))
+```

lib/api/repositories.rb

@@ -170,6 +170,67 @@ module API
           not_found!("Merge Base")
         end
       end
+
+      desc 'Generates a changelog section for a release' do
+        detail 'This feature was introduced in GitLab 13.9'
+      end
+      params do
+        requires :version,
+          type: String,
+          regexp: Gitlab::Regex.unbounded_semver_regex,
+          desc: 'The version of the release, using the semantic versioning format'
+
+        requires :from,
+          type: String,
+          desc: 'The first commit in the range of commits to use for the changelog'
+
+        requires :to,
+          type: String,
+          desc: 'The last commit in the range of commits to use for the changelog'
+
+        optional :date,
+          type: DateTime,
+          desc: 'The date and time of the release'
+
+        optional :branch,
+          type: String,
+          desc: 'The branch to commit the changelog changes to'
+
+        optional :trailer,
+          type: String,
+          desc: 'The Git trailer to use for determining if commits are to be included in the changelog',
+          default: ::Repositories::ChangelogService::DEFAULT_TRAILER
+
+        optional :file,
+          type: String,
+          desc: 'The file to commit the changelog changes to',
+          default: ::Repositories::ChangelogService::DEFAULT_FILE
+
+        optional :message,
+          type: String,
+          desc: 'The commit message to use when committing the changelog'
+      end
+      post ':id/repository/changelog' do
+        not_found! unless Feature.enabled?(:changelog_api, user_project)
+
+        branch = params[:branch] || user_project.default_branch_or_master
+        access = Gitlab::UserAccess.new(current_user, container: user_project)
+
+        unless access.can_push_to_branch?(branch)
+          forbidden!("You are not allowed to commit a changelog on this branch")
+        end
+
+        service = ::Repositories::ChangelogService.new(
+          user_project,
+          current_user,
+          **declared_params(include_missing: false)
+        )
+
+        service.execute
+        status(200)
+      rescue => ex
+        render_api_error!("Failed to generate the changelog: #{ex.message}", 500)
+      end
     end
   end
 end

lib/gitlab/changelog/committer.rb

@@ -26,7 +26,13 @@ module Gitlab
         # scratch, otherwise we may end up throwing away changes. As such, all
         # the logic is contained within the retry block.
         Retriable.retriable(on: CommitError) do
-          commit = @project.commit(branch)
+          commit = Gitlab::Git::Commit.last_for_path(
+            @project.repository,
+            branch,
+            file,
+            literal_pathspec: true
+          )
+
           content = blob_content(file, commit)
 
           # If the release has already been added (e.g. concurrently by another

lib/gitlab/changelog/config.rb

@@ -37,7 +37,10 @@ module Gitlab
         end
 
         if (template = hash['template'])
-          config.template = Template::Compiler.new.compile(template)
+          # We use the full namespace here (and further down) as otherwise Rails
+          # may use the wrong constant when autoloading is used.
+          config.template =
+            ::Gitlab::Changelog::Template::Compiler.new.compile(template)
         end
 
         if (categories = hash['categories'])
@@ -54,7 +57,8 @@ module Gitlab
       def initialize(project)
         @project = project
         @date_format = DEFAULT_DATE_FORMAT
-        @template = Template::Compiler.new.compile(DEFAULT_TEMPLATE)
+        @template =
+          ::Gitlab::Changelog::Template::Compiler.new.compile(DEFAULT_TEMPLATE)
         @categories = {}
       end
 

lib/gitlab/changelog/template/compiler.rb

@@ -98,19 +98,27 @@ module Gitlab
         ESCAPED_NEWLINE = /\\\n$/.freeze
 
         # The start tag for ERB tags. These tags will be escaped, preventing
-        # users FROM USING erb DIRECTLY.
-        ERB_START_TAG = '<%'
+        # users from using ERB directly.
+        ERB_START_TAG = /<\\?\s*\\?\s*%/.freeze
 
         def compile(template)
           transformed_lines = ['<% it = variables %>']
 
+          # ERB tags must be stripped here, otherwise a user may introduce ERB
+          # tags by making clever use of whitespace. See
+          # https://gitlab.com/gitlab-org/gitlab/-/issues/300224 for more
+          # information.
+          template = template.gsub(ERB_START_TAG, '<%%')
+
           template.each_line { |line| transformed_lines << transform(line) }
-          Template.new(transformed_lines.join)
+
+          # We use the full namespace here as otherwise Rails may use the wrong
+          # constant when autoloading is used.
+          ::Gitlab::Changelog::Template::Template.new(transformed_lines.join)
         end
 
         def transform(line)
           line.gsub!(ESCAPED_NEWLINE, '')
-          line.gsub!(ERB_START_TAG, '<%%')
 
           # This replacement ensures that "end" blocks on their own lines
           # don't add extra newlines. Using an ERB -%> tag sadly swallows too

spec/finders/merge_requests/oldest_per_commit_finder_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe MergeRequests::OldestPerCommitFinder do
+  describe '#execute' do
+    it 'returns a Hash mapping commit SHAs to their oldest merge requests' do
+      project = create(:project)
+      mr1 = create(:merge_request, :merged, target_project: project)
+      mr2 = create(:merge_request, :merged, target_project: project)
+      mr1_diff = create(:merge_request_diff, merge_request: mr1)
+      mr2_diff = create(:merge_request_diff, merge_request: mr2)
+      sha1 = Digest::SHA1.hexdigest('foo')
+      sha2 = Digest::SHA1.hexdigest('bar')
+
+      create(:merge_request_diff_commit, merge_request_diff: mr1_diff, sha: sha1)
+      create(:merge_request_diff_commit, merge_request_diff: mr2_diff, sha: sha1)
+      create(
+        :merge_request_diff_commit,
+        merge_request_diff: mr2_diff,
+        sha: sha2,
+        relative_order: 1
+      )
+
+      commits = [double(:commit, id: sha1), double(:commit, id: sha2)]
+
+      expect(described_class.new(project).execute(commits)).to eq(
+        sha1 => mr1,
+        sha2 => mr2
+      )
+    end
+
+    it 'skips merge requests that are not merged' do
+      mr = create(:merge_request)
+      mr_diff = create(:merge_request_diff, merge_request: mr)
+      sha = Digest::SHA1.hexdigest('foo')
+
+      create(:merge_request_diff_commit, merge_request_diff: mr_diff, sha: sha)
+
+      commits = [double(:commit, id: sha)]
+
+      expect(described_class.new(mr.target_project).execute(commits))
+        .to be_empty
+    end
+  end
+end

spec/lib/gitlab/changelog/committer_spec.rb

@@ -86,5 +86,43 @@ RSpec.describe Gitlab::Changelog::Committer do
         end.not_to raise_error
       end
     end
+
+    context "when the changelog changes before saving the changes" do
+      it 'raises a CommitError' do
+        release1 = Gitlab::Changelog::Release
+          .new(version: '1.0.0', date: Time.utc(2020, 1, 1), config: config)
+
+        release2 = Gitlab::Changelog::Release
+          .new(version: '2.0.0', date: Time.utc(2020, 1, 1), config: config)
+
+        # This creates the initial commit we'll later use to see if the
+        # changelog changed before saving our changes.
+        committer.commit(
+          release: release1,
+          file: 'CHANGELOG.md',
+          branch: 'master',
+          message: 'Initial commit'
+        )
+
+        allow(Gitlab::Git::Commit)
+          .to receive(:last_for_path)
+          .with(
+            project.repository,
+            'master',
+            'CHANGELOG.md',
+            literal_pathspec: true
+          )
+          .and_return(double(:commit, sha: 'foo'))
+
+        expect do
+          committer.commit(
+            release: release2,
+            file: 'CHANGELOG.md',
+            branch: 'master',
+            message: 'Test commit'
+          )
+        end.to raise_error(described_class::CommitError)
+      end
+    end
   end
 end

spec/lib/gitlab/changelog/template/compiler_spec.rb

@@ -125,5 +125,12 @@ RSpec.describe Gitlab::Changelog::Template::Compiler do
 
       expect(compile(input)).to eq(input)
     end
+
+    it 'ignores malicious code that makes use of whitespace' do
+      input = "x<\\\n%::Kernel.system(\"id\")%>"
+
+      expect(Kernel).not_to receive(:system).with('id')
+      expect(compile(input)).to eq('x<%::Kernel.system("id")%>')
+    end
   end
 end

spec/requests/api/repositories_spec.rb

@@ -610,4 +610,102 @@ RSpec.describe API::Repositories do
       end
     end
   end
+
+  describe 'POST /projects/:id/repository/changelog' do
+    context 'when the changelog_api feature flag is enabled' do
+      it 'generates the changelog for a version' do
+        spy = instance_spy(Repositories::ChangelogService)
+
+        allow(Repositories::ChangelogService)
+          .to receive(:new)
+          .with(
+            project,
+            user,
+            version: '1.0.0',
+            from: 'foo',
+            to: 'bar',
+            date: DateTime.new(2020, 1, 1),
+            branch: 'kittens',
+            trailer: 'Foo',
+            file: 'FOO.md',
+            message: 'Commit message'
+          )
+          .and_return(spy)
+
+        allow(spy).to receive(:execute)
+
+        post(
+          api("/projects/#{project.id}/repository/changelog", user),
+          params: {
+            version: '1.0.0',
+            from: 'foo',
+            to: 'bar',
+            date: '2020-01-01',
+            branch: 'kittens',
+            trailer: 'Foo',
+            file: 'FOO.md',
+            message: 'Commit message'
+          }
+        )
+
+        expect(response).to have_gitlab_http_status(:ok)
+      end
+
+      it 'produces an error when generating the changelog fails' do
+        spy = instance_spy(Repositories::ChangelogService)
+
+        allow(Repositories::ChangelogService)
+          .to receive(:new)
+          .with(
+            project,
+            user,
+            version: '1.0.0',
+            from: 'foo',
+            to: 'bar',
+            date: DateTime.new(2020, 1, 1),
+            branch: 'kittens',
+            trailer: 'Foo',
+            file: 'FOO.md',
+            message: 'Commit message'
+          )
+          .and_return(spy)
+
+        allow(spy)
+          .to receive(:execute)
+          .and_raise(Gitlab::Changelog::Committer::CommitError.new('oops'))
+
+        post(
+          api("/projects/#{project.id}/repository/changelog", user),
+          params: {
+            version: '1.0.0',
+            from: 'foo',
+            to: 'bar',
+            date: '2020-01-01',
+            branch: 'kittens',
+            trailer: 'Foo',
+            file: 'FOO.md',
+            message: 'Commit message'
+          }
+        )
+
+        expect(response).to have_gitlab_http_status(:internal_server_error)
+        expect(json_response['message']).to eq('Failed to generate the changelog: oops')
+      end
+    end
+
+    context 'when the changelog_api feature flag is disabled' do
+      before do
+        stub_feature_flags(changelog_api: false)
+      end
+
+      it 'responds with a 404 Not Found' do
+        post(
+          api("/projects/#{project.id}/repository/changelog", user),
+          params: { version: '1.0.0', from: 'foo', to: 'bar' }
+        )
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+  end
 end

spec/services/repositories/changelog_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Repositories::ChangelogService do
+  describe '#execute' do
+    it 'generates and commits a changelog section' do
+      project = create(:project, :empty_repo)
+      creator = project.creator
+      author1 = create(:user)
+      author2 = create(:user)
+
+      project.add_maintainer(author1)
+      project.add_maintainer(author2)
+
+      mr1 = create(:merge_request, :merged, target_project: project)
+      mr2 = create(:merge_request, :merged, target_project: project)
+
+      # The range of commits ignores the first commit, but includes the last
+      # commit. To ensure both the commits below are included, we must create an
+      # extra commit.
+      #
+      # In the real world, the start commit of the range will be the last commit
+      # of the previous release, so ignoring that is expected and desired.
+      sha1 = create_commit(
+        project,
+        creator,
+        commit_message: 'Initial commit',
+        actions: [{ action: 'create', content: 'test', file_path: 'README.md' }]
+      )
+
+      sha2 = create_commit(
+        project,
+        author1,
+        commit_message: "Title 1\n\nChangelog: feature",
+        actions: [{ action: 'create', content: 'foo', file_path: 'a.txt' }]
+      )
+
+      sha3 = create_commit(
+        project,
+        author2,
+        commit_message: "Title 2\n\nChangelog: feature",
+        actions: [{ action: 'create', content: 'bar', file_path: 'b.txt' }]
+      )
+
+      commit1 = project.commit(sha2)
+      commit2 = project.commit(sha3)
+
+      allow(MergeRequestDiffCommit)
+        .to receive(:oldest_merge_request_id_per_commit)
+        .with(project.id, [commit2.id, commit1.id])
+        .and_return([
+          { sha: sha2, merge_request_id: mr1.id },
+          { sha: sha3, merge_request_id: mr2.id }
+        ])
+
+      recorder = ActiveRecord::QueryRecorder.new do
+        described_class
+          .new(project, creator, version: '1.0.0', from: sha1, to: sha3)
+          .execute
+      end
+
+      changelog = project.repository.blob_at('master', 'CHANGELOG.md')&.data
+
+      expect(recorder.count).to eq(10)
+      expect(changelog).to include('Title 1', 'Title 2')
+    end
+  end
+
+  def create_commit(project, user, params)
+    params = { start_branch: 'master', branch_name: 'master' }.merge(params)
+    Files::MultiService.new(project, user, params).execute.fetch(:result)
+  end
+end