Merge branch 'es_specs' into 'master'

Tests for ES



See merge request !159

.gitlab-ci.yml

@@ -36,17 +36,21 @@ spec:api:
 
 spec:models:
   script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:models
+    - RAILS_ENV=test SIMPLECOV=true ELASTIC_HOST=elasticsearch bundle exec rake spec:models
   tags:
     - ruby
     - mysql
+  services:
+    - elasticsearch:latest
 
 spec:lib:
   script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:lib
+    - RAILS_ENV=test SIMPLECOV=true ELASTIC_HOST=elasticsearch bundle exec rake spec:lib
   tags:
     - ruby
     - mysql
+  services:
+    - elasticsearch:latest
 
 spec:services:
   script:
@@ -65,10 +69,12 @@ spec:benchmark:
 
 spec:other:
   script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:other
+    - RAILS_ENV=test SIMPLECOV=true ELASTIC_HOST=elasticsearch bundle exec rake spec:other
   tags:
     - ruby
     - mysql
+  services:
+    - elasticsearch:latest
 
 spinach:project:half:
   script:
@@ -79,10 +85,12 @@ spinach:project:half:
 
 spinach:project:rest:
   script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:project:rest
+    - RAILS_ENV=test SIMPLECOV=true ELASTIC_HOST=elasticsearch bundle exec rake spinach:project:rest
   tags:
     - ruby
     - mysql
+  services:
+    - elasticsearch:latest
 
 spinach:other:
   script:

config/gitlab.yml.example

@@ -141,8 +141,8 @@ production: &base
   # regular database search
   elasticsearch:
     enabled: false
-    host: localhost
-    port: 9200
+  #  host: localhost
+  #  port: 9200
 
   ## GitLab Pages
   pages:

config/initializers/1_settings.rb

@@ -257,8 +257,8 @@ Settings.gitlab['import_sources'] ||= ['github','bitbucket','gitlab','gitorious'
 #
 Settings['elasticsearch'] ||= Settingslogic.new({})
 Settings.elasticsearch['enabled'] = false if Settings.elasticsearch['enabled'].nil?
-Settings.elasticsearch['host'] = "localhost" if Settings.elasticsearch['host'].nil?
-Settings.elasticsearch['port'] = 9200 if Settings.elasticsearch['port'].nil?
+Settings.elasticsearch['host'] ||=  ENV['ELASTIC_HOST'] || "localhost"
+Settings.elasticsearch['port'] ||= ENV['ELASTIC_PORT'] || 9200
 
 #
 # CI

features/project/elastic/global_search.feature

+Feature: Global Search
+  Background:
+    Given I sign in as a user
+    And I own project "Shop"
+    And Elasticsearch is enabled
+
+  Scenario: I search through the all projects
+    Given project has all data available for the search
+    And I visit dashboard page
+    Then I search "initial"
+    And I find an Issue
+    And I find a Merge Request
+    And I find a Milestone
\ No newline at end of file

features/project/elastic/project_search.feature

+Feature: Project Search
+  Background:
+    Given I sign in as a user
+    And I own project "Shop"
+    And Elasticsearch is enabled
+
+  Scenario: I search through the all project items
+    Given project has all data available for the search
+    And I visit my project's home page
+    Then I search "initial"
+    And I find an Issue
+    And I find a Merge Request
+    And I find a Milestone
+    And I find a Comment
+    And I find a Commit
+    And I find a Wiki Page
+    Then I visit my project's home page
+    Then I search "def"
+    And I find a Code

features/project/elastic/snippets_search.feature

+Feature: Snippets Search
+  Background:
+    Given I sign in as a user
+    And Elasticsearch is enabled
+
+  Scenario: I search through the snippets
+    Given there is a snippet "index" with "php rocks" string
+    And there is a snippet "php" with "benefits" string
+    And I visit snippets page
+    Then I search "php"
+    And I find "index" snippet
+    Then I select search by titles and filenames
+    And I find "php" snippet
\ No newline at end of file

features/steps/project/elastic/global_search.rb

+class Spinach::Features::GlobalSearch < Spinach::FeatureSteps
+  include SharedAuthentication
+  include SharedPaths
+  include SharedProject
+  include SharedElastic
+
+  before do
+    [::Project, Issue, MergeRequest, Milestone].each do |model|
+      model.__elasticsearch__.create_index!
+    end
+  end
+
+  after do
+    [::Project, Issue, MergeRequest, Milestone].each do |model|
+      model.__elasticsearch__.delete_index!
+    end
+
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
+  end
+
+  step 'project has all data available for the search' do
+    @project = create :project
+    @project.team << [current_user, :master]
+
+    @issue = create :issue, title: 'bla-bla initial', project: @project
+    @merge_request = create :merge_request, title: 'bla-bla initial', source_project: @project
+    @milestone = create :milestone, title: 'bla-bla initial', project: @project
+  end
+end

features/steps/project/elastic/project_search.rb

+class Spinach::Features::ProjectSearch < Spinach::FeatureSteps
+  include SharedAuthentication
+  include SharedPaths
+  include SharedProject
+  include SharedElastic
+
+  before do
+    [::Project, Repository, Note, MergeRequest, Milestone, ::ProjectWiki, Issue].each do |model|
+      model.__elasticsearch__.create_index!
+    end
+  end
+
+  after do
+    [::Project, Repository, Note, MergeRequest, Milestone, ::ProjectWiki, Issue].each do |model|
+      model.__elasticsearch__.delete_index!
+    end
+
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
+  end
+
+  step 'project has all data available for the search' do
+    @project = create :project
+    @project.team << [current_user, :master]
+
+    @issue = create :issue, title: 'bla-bla initial', project: @project
+    @merge_request = create :merge_request, title: 'bla-bla initial', source_project: @project
+    @milestone = create :milestone, title: 'bla-bla initial', project: @project
+    @note = create :note, note: 'bla-bla initial', project: @project, noteable: @issue
+
+    @project.repository.index_blobs
+    @project.repository.index_commits
+
+    @project.wiki.create_page("index_page", "Bla bla initial")
+  end
+
+  step 'I search "def"' do
+    fill_in "search", with: "def"
+    click_button "Go"
+  end
+
+  step 'I find a Comment' do
+    select_filter("Comments")
+
+    expect(page.find('.search-result-row')).to have_content(@note.note)
+  end
+
+  step 'I find a Wiki Page' do
+    select_filter("Wiki")
+
+    expect(page.find('.blob-result')).to have_content('Bla bla init')
+  end
+
+  step 'I find a Commit' do
+    select_filter("Commits")
+
+    expect(page.find('.search-result-row')).to have_content("Initial commit")
+  end
+
+  step 'I find a Code' do
+    expect(page.first('.blob-result')).to have_content("def")
+  end
+end

features/steps/project/elastic/snippets_search.rb

+class Spinach::Features::SnippetsSearch < Spinach::FeatureSteps
+  include SharedAuthentication
+  include SharedPaths
+  include SharedProject
+  include SharedElastic
+
+  before do
+    Snippet.__elasticsearch__.create_index!
+  end
+
+  after do
+    Snippet.__elasticsearch__.delete_index!
+
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
+  end
+
+  step 'there is a snippet "index" with "php rocks" string' do
+    create :personal_snippet, :public, content: "php rocks", title: "index"
+    Snippet.__elasticsearch__.refresh_index!
+  end
+
+  step 'there is a snippet "php" with "benefits" string' do
+    create :personal_snippet, :public, content: "benefits", title: "php"
+    Snippet.__elasticsearch__.refresh_index!
+  end
+
+  step 'I search "php"' do
+    fill_in "search", with: "php"
+    click_button "Go"
+  end
+
+  step 'I find "index" snippet' do
+    expect(page.find('.file-holder')).to have_content("php rocks")
+  end
+
+  step 'I select search by titles and filenames' do
+    select_filter("Titles and Filenames")
+  end
+
+  step 'I find "php" snippet' do
+    expect(page.find('.search-result-row')).to have_content("php")
+  end
+end

features/steps/shared/elastic.rb

+module SharedElastic
+  include Spinach::DSL
+
+  step 'I search "initial"' do
+    fill_in "search", with: "initial"
+    click_button "Go"
+  end
+
+  step 'I find an Issue' do
+    select_filter("Issues")
+
+    expect(page.find('.search-result-row')).to have_content(@issue.title)
+  end
+
+  step 'I find a Merge Request' do
+    select_filter("Merge requests")
+
+    expect(page.find('.search-result-row')).to have_content(@merge_request.title)
+  end
+
+  step 'I find a Milestone' do
+    select_filter("Milestones")
+
+    expect(page.find('.search-result-row')).to have_content(@milestone.title)
+  end
+
+  step 'Elasticsearch is enabled' do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
+  end
+
+  def select_filter(name)
+    find(:xpath, "//ul[contains(@class, 'search-filter')]//a[contains(.,'#{name}')]").click
+  end
+end

lib/elastic/application_search.rb

@@ -30,16 +30,20 @@ module Elastic
           }
         }
       
-      if Gitlab.config.elasticsearch.enabled
-        after_commit on: :create do
+      after_commit on: :create do
+        if Gitlab.config.elasticsearch.enabled
           ElasticIndexerWorker.perform_async(:index, self.class.to_s, self.id)
         end
+      end
 
-        after_commit on: :update do
+      after_commit on: :update do
+        if Gitlab.config.elasticsearch.enabled
           ElasticIndexerWorker.perform_async(:update, self.class.to_s, self.id)
         end
+      end
 
-        after_commit on: :destroy do
+      after_commit on: :destroy do
+        if Gitlab.config.elasticsearch.enabled
           ElasticIndexerWorker.perform_async(:delete, self.class.to_s, self.id)
         end
       end

spec/lib/elastic/issue_spec.rb

+require 'spec_helper'
+
+describe "Issue", elastic: true do
+  before do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
+    Issue.__elasticsearch__.create_index!
+  end
+
+  after do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
+    Issue.__elasticsearch__.delete_index!
+  end
+
+  it "searches issues" do
+    project = create :empty_project
+
+    create :issue, title: 'bla-bla term', project: project
+    create :issue, description: 'bla-bla term', project: project
+    create :issue, project: project
+
+    # The issue I have no access to
+    create :issue, title: 'bla-bla term'
+
+    Issue.__elasticsearch__.refresh_index!
+
+    options = { projects_ids: [project.id] }
+
+    expect(Issue.elastic_search('term', options: options).total_count).to eq(2)
+  end
+end

spec/lib/elastic/merge_request_spec.rb

+require 'spec_helper'
+
+describe "MergeRequest", elastic: true do
+  before do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
+    MergeRequest.__elasticsearch__.create_index!
+  end
+
+  after do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
+    MergeRequest.__elasticsearch__.delete_index!
+  end
+
+  it "searches merge requests" do
+    project = create :project
+
+    create :merge_request, title: 'bla-bla term', source_project: project
+    create :merge_request, description: 'term in description', source_project: project, target_branch: "feature2"
+    create :merge_request, source_project: project, target_branch: "feature3"
+
+    # The merge request you have no access to
+    create :merge_request, title: 'also with term'
+
+    MergeRequest.__elasticsearch__.refresh_index!
+
+    options = { projects_ids: [project.id] }
+
+    expect(MergeRequest.elastic_search('term', options: options).total_count).to eq(2)
+  end
+end

spec/lib/elastic/milestone_spec.rb

+require 'spec_helper'
+
+describe "Milestone", elastic: true do
+  before do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
+    Milestone.__elasticsearch__.create_index!
+  end
+
+  after do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
+    Milestone.__elasticsearch__.delete_index!
+  end
+
+  it "searches milestones" do
+    project = create :empty_project
+
+    create :milestone, title: 'bla-bla term', project: project
+    create :milestone, description: 'bla-bla term', project: project
+    create :milestone, project: project
+
+    create :milestone, title: 'bla-bla term'
+
+    Milestone.__elasticsearch__.refresh_index!
+
+    options = { projects_ids: [project.id] }
+
+    expect(Milestone.elastic_search('term', options: options).total_count).to eq(2)
+  end
+end

spec/lib/elastic/note_spec.rb

+require 'spec_helper'
+
+describe "Note", elastic: true do
+  before do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
+    Note.__elasticsearch__.create_index!
+  end
+
+  after do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
+    Note.__elasticsearch__.delete_index!
+  end
+
+  it "searches notes" do
+    issue = create :issue
+
+    create :note, note: 'bla-bla term', project: issue.project
+    create :note, project: issue.project
+
+    # The note in the project you have no access to
+    create :note, note: 'bla-bla term'
+
+    Note.__elasticsearch__.refresh_index!
+
+    options = { projects_ids: [issue.project.id] }
+
+    expect(Note.elastic_search('term', options: options).total_count).to eq(1)
+  end
+end

spec/lib/elastic/project_spec.rb

+require 'spec_helper'
+
+describe "Projects", elastic: true do
+  before do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
+    Project.__elasticsearch__.create_index!
+  end
+
+  after do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
+    Project.__elasticsearch__.delete_index!
+  end
+
+  it "searches projects" do
+    @project = create :empty_project, name: 'test'
+    @project1 = create :empty_project, path: 'test1'
+    @project2 = create :empty_project
+    @project3 = create :empty_project, path: 'someone_elses_project'
+    @project_ids = [@project.id, @project1.id, @project2.id]
+    
+    Project.__elasticsearch__.refresh_index!
+
+    expect(Project.elastic_search('test', options: { pids: @project_ids }).total_count).to eq(1)
+    expect(Project.elastic_search('test1', options: { pids: @project_ids }).total_count).to eq(1)
+    expect(Project.elastic_search('someone_elses_project', options: { pids: @project_ids }).total_count).to eq(0)
+  end
+end

spec/lib/elastic/repositories_spec.rb

+require 'spec_helper'
+
+describe "Repository", elastic: true do
+  before do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
+    Repository.__elasticsearch__.create_index!
+  end
+
+  after do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
+    Repository.__elasticsearch__.delete_index!
+  end
+
+  it "searches blobs and commits" do
+    project = create :project
+
+    project.repository.index_blobs
+    project.repository.index_commits
+    
+    Repository.__elasticsearch__.refresh_index!
+
+    expect(project.repository.search('def popen')[:blobs][:total_count]).to eq(1)
+    expect(project.repository.search('initial')[:commits][:total_count]).to eq(1)
+  end
+end

spec/lib/elastic/snippet_spec.rb

+require 'spec_helper'
+
+describe "Snippet", elastic: true do
+  before do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
+    Snippet.__elasticsearch__.create_index!
+  end
+
+  after do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
+    Snippet.__elasticsearch__.delete_index!
+  end
+
+  it "searches snippets by code" do
+    @snippet = create :personal_snippet, content: 'genius code'
+    @snippet1 = create :personal_snippet
+
+    # the snippet I have no access to
+    @snippet2 = create :personal_snippet, content: 'genius code'
+
+    @snippet_ids = [@snippet.id, @snippet1.id]
+
+    Snippet.__elasticsearch__.refresh_index!
+
+    options = { ids: @snippet_ids }
+
+    expect(Snippet.elastic_search_code('genius code', options: options).total_count).to eq(1)
+  end
+
+  it "searches snippets by title and file_name" do
+    @snippet = create :snippet, title: 'home'
+    @snippet1 = create :snippet, file_name: 'index.php'
+    @snippet2 = create :snippet
+
+    # the snippet I have no access to
+    @snippet3 = create :snippet, title: 'home'
+
+    @snippet_ids = [@snippet.id, @snippet1.id, @snippet2.id]
+    
+    Snippet.__elasticsearch__.refresh_index!
+
+    options = { ids: @snippet_ids }
+
+    expect(Snippet.elastic_search('home', options: options).total_count).to eq(1)
+    expect(Snippet.elastic_search('index.php', options:  options).total_count).to eq(1)
+  end
+end

spec/lib/elastic/wiki_reposotories_spec.rb

+require 'spec_helper'
+
+describe "ProjectWiki", elastic: true do
+  before do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
+    ProjectWiki.__elasticsearch__.create_index!
+  end
+
+  after do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
+    ProjectWiki.__elasticsearch__.delete_index!
+  end
+
+  it "searches wiki page" do
+    project = create :empty_project
+
+    project.wiki.create_page("index_page", "Bla bla")
+
+    project.wiki.index_blobs
+    
+    ProjectWiki.__elasticsearch__.refresh_index!
+
+    expect(project.wiki.search('bla', type: :blob)[:blobs][:total_count]).to eq(1)
+  end
+end

spec/lib/gitlab/elastic/project_search_results_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Elastic::ProjectSearchResults, lib: true do
+  before do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(true)
+    Project.__elasticsearch__.create_index!
+
+    @project = create(:project)
+  end
+
+  after do
+    allow(Gitlab.config.elasticsearch).to receive(:enabled).and_return(false)
+    Project.__elasticsearch__.delete_index!
+  end
+
+  let(:query) { 'hello world' }
+
+  describe 'initialize with empty ref' do
+    let(:results) { Gitlab::Elastic::ProjectSearchResults.new(@project.id, query, '') }
+
+    it { expect(results.project).to eq(@project) }
+    it { expect(results.repository_ref).to be_nil }
+    it { expect(results.query).to eq('hello world') }
+  end
+
+  describe 'initialize with ref' do
+    let(:ref) { 'refs/heads/test' }
+    let(:results) { Gitlab::Elastic::ProjectSearchResults.new(@project.id, query, ref) }
+
+    it { expect(results.project).to eq(@project) }
+    it { expect(results.repository_ref).to eq(ref) }
+    it { expect(results.query).to eq('hello world') }
+  end
+end

spec/models/repository_spec.rb

@@ -239,4 +239,49 @@ describe Repository, models: true do
       it { is_expected.to eq(true) }
     end
   end
+
+  describe "Elastic search", elastic: true do
+    before do
+      Repository.__elasticsearch__.create_index!
+    end
+
+    after do
+      Repository.__elasticsearch__.delete_index!
+    end
+
+    describe :find_commits_by_message_with_elastic do
+      it "returns commits" do
+        project = create :project
+
+        project.repository.index_commits
+        
+        Repository.__elasticsearch__.refresh_index!
+
+        expect(project.repository.find_commits_by_message_with_elastic('initial').first).to be_a(Commit)
+        expect(project.repository.find_commits_by_message_with_elastic('initial').count).to eq(1)
+      end
+    end
+
+    describe :parse_search_result_from_elastic do
+      it "returns parsed result" do
+        project = create :project
+
+        project.repository.index_blobs
+
+        Repository.__elasticsearch__.refresh_index!
+
+        result = project.repository.search(
+          'def popen',
+          type: :blob,
+          options: { highlight: true }
+        )[:blobs][:results][0]
+
+        parsed_result = project.repository.parse_search_result_from_elastic(result)
+
+        expect(parsed_result.filename).to eq('files/ruby/popen.rb')
+        expect(parsed_result.startline).to eq(2)
+        expect(parsed_result.data).to include("Popen")
+      end
+    end
+  end
 end

spec/support/webmock.rb

 require 'webmock'
 require 'webmock/rspec'
 
-WebMock.disable_net_connect!(allow_localhost: true)
+WebMock.disable_net_connect!(allow_localhost: true, allow: 'elasticsearch')