Merge branch 'georgekoltsov/add-github-gitea-filtering' into 'master'

Add GitHub & Gitea importer projects filtering

See merge request gitlab-org/gitlab!16823

app/assets/javascripts/import_projects/components/import_projects_table.vue

 <script>
+import _ from 'underscore';
 import { mapActions, mapState, mapGetters } from 'vuex';
 import { GlLoadingIcon } from '@gitlab/ui';
 import LoadingButton from '~/vue_shared/components/loading_button.vue';
@@ -7,6 +8,8 @@ import ImportedProjectTableRow from './imported_project_table_row.vue';
 import ProviderRepoTableRow from './provider_repo_table_row.vue';
 import eventHub from '../event_hub';
 
+const reposFetchThrottleDelay = 1000;
+
 export default {
   name: 'ImportProjectsTable',
   components: {
@@ -23,11 +26,11 @@ export default {
   },
 
   computed: {
-    ...mapState(['importedProjects', 'providerRepos', 'isLoadingRepos']),
+    ...mapState(['importedProjects', 'providerRepos', 'isLoadingRepos', 'filter']),
     ...mapGetters(['isImportingAnyRepo', 'hasProviderRepos', 'hasImportedProjects']),
 
     emptyStateText() {
-      return sprintf(__('No %{providerTitle} repositories available to import'), {
+      return sprintf(__('No %{providerTitle} repositories found'), {
         providerTitle: this.providerTitle,
       });
     },
@@ -47,21 +50,38 @@ export default {
   },
 
   methods: {
-    ...mapActions(['fetchRepos', 'fetchJobs', 'stopJobsPolling', 'clearJobsEtagPoll']),
+    ...mapActions([
+      'fetchRepos',
+      'fetchReposFiltered',
+      'fetchJobs',
+      'stopJobsPolling',
+      'clearJobsEtagPoll',
+      'setFilter',
+    ]),
 
     importAll() {
       eventHub.$emit('importAll');
     },
+
+    handleFilterInput({ target }) {
+      this.setFilter(target.value);
+    },
+
+    throttledFetchRepos: _.throttle(function fetch() {
+      eventHub.$off('importAll');
+      this.fetchRepos();
+    }, reposFetchThrottleDelay),
   },
 };
 </script>
 
 <template>
   <div>
+    <p class="light text-nowrap mt-2">
+      {{ s__('ImportProjects|Select the projects you want to import') }}
+    </p>
+
     <div class="d-flex justify-content-between align-items-end flex-wrap mb-3">
-      <p class="light text-nowrap mt-2 my-sm-0">
-        {{ s__('ImportProjects|Select the projects you want to import') }}
-      </p>
       <loading-button
         container-class="btn btn-success js-import-all"
         :loading="isImportingAnyRepo"
@@ -70,6 +90,19 @@ export default {
         type="button"
         @click="importAll"
       />
+      <form novalidate @submit.prevent>
+        <input
+          :value="filter"
+          data-qa-selector="githubish_import_filter_field"
+          class="form-control"
+          name="filter"
+          :placeholder="__('Filter your projects by name')"
+          autofocus
+          size="40"
+          @input="handleFilterInput($event)"
+          @keyup.enter="throttledFetchRepos"
+        />
+      </form>
     </div>
     <gl-loading-icon
       v-if="isLoadingRepos"

app/assets/javascripts/import_projects/index.js

@@ -38,7 +38,7 @@ export default function mountImportProjectsTable(mountElement) {
     },
 
     methods: {
-      ...mapActions(['setInitialData']),
+      ...mapActions(['setInitialData', 'setFilter']),
     },
 
     render(createElement) {

app/assets/javascripts/import_projects/store/actions.js

@@ -5,6 +5,7 @@ import Poll from '~/lib/utils/poll';
 import createFlash from '~/flash';
 import { s__, sprintf } from '~/locale';
 import axios from '~/lib/utils/axios_utils';
+import { jobsPathWithFilter, reposPathWithFilter } from './getters';
 
 let eTagPoll;
 
@@ -19,16 +20,20 @@ export const restartJobsPolling = () => {
 };
 
 export const setInitialData = ({ commit }, data) => commit(types.SET_INITIAL_DATA, data);
+export const setFilter = ({ commit }, filter) => commit(types.SET_FILTER, filter);
 
 export const requestRepos = ({ commit }, repos) => commit(types.REQUEST_REPOS, repos);
 export const receiveReposSuccess = ({ commit }, repos) =>
   commit(types.RECEIVE_REPOS_SUCCESS, repos);
 export const receiveReposError = ({ commit }) => commit(types.RECEIVE_REPOS_ERROR);
 export const fetchRepos = ({ state, dispatch }) => {
+  dispatch('stopJobsPolling');
   dispatch('requestRepos');
 
+  const { provider } = state;
+
   return axios
-    .get(state.reposPath)
+    .get(reposPathWithFilter(state))
     .then(({ data }) =>
       dispatch('receiveReposSuccess', convertObjectPropsToCamelCase(data, { deep: true })),
     )
@@ -36,7 +41,7 @@ export const fetchRepos = ({ state, dispatch }) => {
     .catch(() => {
       createFlash(
         sprintf(s__('ImportProjects|Requesting your %{provider} repositories failed'), {
-          provider: state.provider,
+          provider,
         }),
       );
 
@@ -77,16 +82,23 @@ export const fetchImport = ({ state, dispatch }, { newName, targetNamespace, rep
 export const receiveJobsSuccess = ({ commit }, updatedProjects) =>
   commit(types.RECEIVE_JOBS_SUCCESS, updatedProjects);
 export const fetchJobs = ({ state, dispatch }) => {
-  if (eTagPoll) return;
+  const { filter } = state;
+
+  if (eTagPoll) {
+    stopJobsPolling();
+    clearJobsEtagPoll();
+  }
 
   eTagPoll = new Poll({
     resource: {
-      fetchJobs: () => axios.get(state.jobsPath),
+      fetchJobs: () => axios.get(jobsPathWithFilter(state)),
     },
     method: 'fetchJobs',
     successCallback: ({ data }) =>
       dispatch('receiveJobsSuccess', convertObjectPropsToCamelCase(data, { deep: true })),
-    errorCallback: () => createFlash(s__('ImportProjects|Updating the imported projects failed')),
+    errorCallback: () =>
+      createFlash(s__('ImportProjects|Update of imported projects with realtime changes failed')),
+    data: { filter },
   });
 
   if (!Visibility.hidden()) {

app/assets/javascripts/import_projects/store/getters.js

@@ -20,3 +20,8 @@ export const isImportingAnyRepo = state => state.reposBeingImported.length > 0;
 export const hasProviderRepos = state => state.providerRepos.length > 0;
 
 export const hasImportedProjects = state => state.importedProjects.length > 0;
+
+export const reposPathWithFilter = ({ reposPath, filter = '' }) =>
+  filter ? `${reposPath}?filter=${filter}` : reposPath;
+export const jobsPathWithFilter = ({ jobsPath, filter = '' }) =>
+  filter ? `${jobsPath}?filter=${filter}` : jobsPath;

app/assets/javascripts/import_projects/store/mutation_types.js

@@ -9,3 +9,5 @@ export const RECEIVE_IMPORT_SUCCESS = 'RECEIVE_IMPORT_SUCCESS';
 export const RECEIVE_IMPORT_ERROR = 'RECEIVE_IMPORT_ERROR';
 
 export const RECEIVE_JOBS_SUCCESS = 'RECEIVE_JOBS_SUCCESS';
+
+export const SET_FILTER = 'SET_FILTER';

app/assets/javascripts/import_projects/store/mutations.js

@@ -6,6 +6,10 @@ export default {
     Object.assign(state, data);
   },
 
+  [types.SET_FILTER](state, filter) {
+    state.filter = filter;
+  },
+
   [types.REQUEST_REPOS](state) {
     state.isLoadingRepos = true;
   },

app/assets/javascripts/import_projects/store/state.js

@@ -12,4 +12,5 @@ export default () => ({
   isLoadingRepos: false,
   canSelectNamespace: false,
   ciCdOnly: false,
+  filter: '',
 });

app/controllers/import/github_controller.rb

@@ -2,6 +2,7 @@
 
 class Import::GithubController < Import::BaseController
   include ImportHelper
+  include ActionView::Helpers::SanitizeHelper
 
   before_action :verify_import_enabled
   before_action :provider_auth, only: [:status, :realtime_changes, :create]
@@ -55,7 +56,7 @@ class Import::GithubController < Import::BaseController
   def realtime_changes
     Gitlab::PollingInterval.set_header(response, interval: 3_000)
 
-    render json: find_jobs(provider)
+    render json: already_added_projects.to_json(only: [:id], methods: [:import_status])
   end
 
   private
@@ -82,7 +83,7 @@ class Import::GithubController < Import::BaseController
   end
 
   def already_added_projects
-    @already_added_projects ||= find_already_added_projects(provider)
+    @already_added_projects ||= filtered(find_already_added_projects(provider))
   end
 
   def already_added_project_names
@@ -104,7 +105,7 @@ class Import::GithubController < Import::BaseController
   end
 
   def client_repos
-    @client_repos ||= client.repos
+    @client_repos ||= filtered(client.repos)
   end
 
   def verify_import_enabled
@@ -185,6 +186,20 @@ class Import::GithubController < Import::BaseController
   def extra_import_params
     {}
   end
+
+  def sanitized_filter_param
+    @filter ||= sanitize(params[:filter])
+  end
+
+  def filter_attribute
+    :name
+  end
+
+  def filtered(collection)
+    return collection unless sanitized_filter_param
+
+    collection.select { |item| item[filter_attribute].include?(sanitized_filter_param) }
+  end
 end
 
 Import::GithubController.prepend_if_ee('EE::Import::GithubController')

changelogs/unreleased/georgekoltsov-add-github-importer-filtering.yml

+---
+title: Add GitHub & Gitea importers project filtering
+merge_request: 16823
+author:
+type: added

doc/user/project/import/gitea.md

@@ -66,10 +66,14 @@ From there, you can see the import statuses of your Gitea repositories.
 - whereas those that are not yet imported will have an **Import** button on the
   right side of the table.
 
-If you want, you can import all your Gitea projects in one go by hitting
-**Import all projects** in the upper left corner.
+You also can:
 
-![Gitea importer page](img/import_projects_from_github_importer.png)
+- Import all your Gitea projects in one go by hitting **Import all projects** in
+  the upper left corner
+- Filter projects by name. If filter is applied, hitting **Import all projects**
+  will only import matched projects
+
+![Gitea importer page](img/import_projects_from_gitea_importer_v12_3.png)
 
 ---
 

doc/user/project/import/github.md

@@ -115,11 +115,14 @@ your GitHub repositories are listed.
 
 1. By default, the proposed repository namespaces match the names as they exist in GitHub, but based on your permissions,
    you can choose to edit these names before you proceed to import any of them.
-1. Select the **Import** button next to any number of repositories, or select **Import all repositories**.
+1. Select the **Import** button next to any number of repositories, or select **Import all repositories**. Additionally,
+   you can filter projects by name. If filter is applied, **Import all repositories** only imports matched repositories.
 1. The **Status** column shows the import status of each repository. You can choose to leave the page open and it will
    update in realtime or you can return to it later.
 1. Once a repository has been imported, click its GitLab path to open its GitLab URL.
 
+![Github importer page](img/import_projects_from_github_importer_v12_3.png)
+
 ## Mirroring and pipeline status sharing
 
 Depending your GitLab tier, [project mirroring](../../../workflow/repository_mirroring.md) can be set up to keep

locale/gitlab.pot

@@ -8261,7 +8261,7 @@ msgstr ""
 msgid "ImportProjects|The repository could not be created."
 msgstr ""
 
-msgid "ImportProjects|Updating the imported projects failed"
+msgid "ImportProjects|Update of imported projects with realtime changes failed"
 msgstr ""
 
 msgid "Improve Issue boards"
@@ -10205,7 +10205,7 @@ msgstr ""
 msgid "No %{header} for this request."
 msgstr ""
 
-msgid "No %{providerTitle} repositories available to import"
+msgid "No %{providerTitle} repositories found"
 msgstr ""
 
 msgid "No Epic"

spec/frontend/import_projects/components/import_projects_table_spec.js

@@ -93,7 +93,7 @@ describe('ImportProjectsTable', () => {
     return vm.$nextTick().then(() => {
       expect(vm.$el.querySelector('.js-loading-button-icon')).toBeNull();
       expect(vm.$el.querySelector('.table')).toBeNull();
-      expect(vm.$el.innerText).toMatch(`No ${providerTitle} repositories available to import`);
+      expect(vm.$el.innerText).toMatch(`No ${providerTitle} repositories found`);
     });
   });
 
@@ -182,4 +182,10 @@ describe('ImportProjectsTable', () => {
         expect(vm.$el.querySelector(`.ic-status_${statusObject.icon}`)).not.toBeNull();
       });
   });
+
+  it('renders filtering input field', () => {
+    expect(
+      vm.$el.querySelector('input[data-qa-selector="githubish_import_filter_field"]'),
+    ).not.toBeNull();
+  });
 });

spec/frontend/import_projects/store/actions_spec.js

@@ -97,6 +97,7 @@ describe('import_projects store actions', () => {
 
   describe('fetchRepos', () => {
     let mock;
+    const payload = { imported_projects: [{}], provider_repos: [{}], namespaces: [{}] };
 
     beforeEach(() => {
       localState.reposPath = `${TEST_HOST}/endpoint.json`;
@@ -105,8 +106,7 @@ describe('import_projects store actions', () => {
 
     afterEach(() => mock.restore());
 
-    it('dispatches requestRepos and receiveReposSuccess actions on a successful request', done => {
-      const payload = { imported_projects: [{}], provider_repos: [{}], namespaces: [{}] };
+    it('dispatches stopJobsPolling, requestRepos and receiveReposSuccess actions on a successful request', done => {
       mock.onGet(`${TEST_HOST}/endpoint.json`).reply(200, payload);
 
       testAction(
@@ -115,6 +115,7 @@ describe('import_projects store actions', () => {
         localState,
         [],
         [
+          { type: 'stopJobsPolling' },
           { type: 'requestRepos' },
           {
             type: 'receiveReposSuccess',
@@ -128,7 +129,7 @@ describe('import_projects store actions', () => {
       );
     });
 
-    it('dispatches requestRepos and receiveReposSuccess actions on an unsuccessful request', done => {
+    it('dispatches stopJobsPolling, requestRepos and receiveReposError actions on an unsuccessful request', done => {
       mock.onGet(`${TEST_HOST}/endpoint.json`).reply(500);
 
       testAction(
@@ -136,10 +137,39 @@ describe('import_projects store actions', () => {
         null,
         localState,
         [],
-        [{ type: 'requestRepos' }, { type: 'receiveReposError' }],
+        [{ type: 'stopJobsPolling' }, { type: 'requestRepos' }, { type: 'receiveReposError' }],
         done,
       );
     });
+
+    describe('when filtered', () => {
+      beforeEach(() => {
+        localState.filter = 'filter';
+      });
+
+      it('fetches repos with filter applied', done => {
+        mock.onGet(`${TEST_HOST}/endpoint.json?filter=filter`).reply(200, payload);
+
+        testAction(
+          fetchRepos,
+          null,
+          localState,
+          [],
+          [
+            { type: 'stopJobsPolling' },
+            { type: 'requestRepos' },
+            {
+              type: 'receiveReposSuccess',
+              payload: convertObjectPropsToCamelCase(payload, { deep: true }),
+            },
+            {
+              type: 'fetchJobs',
+            },
+          ],
+          done,
+        );
+      });
+    });
   });
 
   describe('requestImport', () => {
@@ -249,6 +279,7 @@ describe('import_projects store actions', () => {
 
   describe('fetchJobs', () => {
     let mock;
+    const updatedProjects = [{ name: 'imported/project' }, { name: 'provider/repo' }];
 
     beforeEach(() => {
       localState.jobsPath = `${TEST_HOST}/endpoint.json`;
@@ -263,7 +294,6 @@ describe('import_projects store actions', () => {
     afterEach(() => mock.restore());
 
     it('dispatches requestJobs and receiveJobsSuccess actions on a successful request', done => {
-      const updatedProjects = [{ name: 'imported/project' }, { name: 'provider/repo' }];
       mock.onGet(`${TEST_HOST}/endpoint.json`).reply(200, updatedProjects);
 
       testAction(
@@ -280,5 +310,29 @@ describe('import_projects store actions', () => {
         done,
       );
     });
+
+    describe('when filtered', () => {
+      beforeEach(() => {
+        localState.filter = 'filter';
+      });
+
+      it('fetches realtime changes with filter applied', done => {
+        mock.onGet(`${TEST_HOST}/endpoint.json?filter=filter`).reply(200, updatedProjects);
+
+        testAction(
+          fetchJobs,
+          null,
+          localState,
+          [],
+          [
+            {
+              type: 'receiveJobsSuccess',
+              payload: convertObjectPropsToCamelCase(updatedProjects, { deep: true }),
+            },
+          ],
+          done,
+        );
+      });
+    });
   });
 });

spec/support/controllers/githubish_import_controller_shared_examples.rb

@@ -139,6 +139,38 @@ shared_examples 'a GitHub-ish import controller: GET status' do
     expect { get :status, format: :json }
       .not_to exceed_all_query_limit(control_count)
   end
+
+  context 'when filtering' do
+    let(:repo_2) { OpenStruct.new(login: 'emacs', full_name: 'asd/emacs', name: 'emacs', owner: { login: 'owner' }) }
+    let(:project) { create(:project, import_type: provider, namespace: user.namespace, import_status: :finished, import_source: 'example/repo') }
+    let(:group) { create(:group) }
+
+    before do
+      group.add_owner(user)
+      stub_client(repos: [repo, repo_2, org_repo], orgs: [org], org_repos: [org_repo])
+    end
+
+    it 'filters list of repositories by name' do
+      get :status, params: { filter: 'emacs' }, format: :json
+
+      expect(response).to have_gitlab_http_status(200)
+      expect(json_response.dig("imported_projects").count).to eq(0)
+      expect(json_response.dig("provider_repos").count).to eq(1)
+      expect(json_response.dig("provider_repos", 0, "id")).to eq(repo_2.id)
+      expect(json_response.dig("namespaces", 0, "id")).to eq(group.id)
+    end
+
+    context 'when user input contains html' do
+      let(:expected_filter) { 'test' }
+      let(:filter) { "<html>#{expected_filter}</html>" }
+
+      it 'sanitizes user input' do
+        get :status, params: { filter: filter }, format: :json
+
+        expect(assigns(:filter)).to eq(expected_filter)
+      end
+    end
+  end
 end
 
 shared_examples 'a GitHub-ish import controller: POST create' do