Commit 1ab96d89 authored by Dmitriy Zaporozhets's avatar Dmitriy Zaporozhets

Merge remote-tracking branch 'ee-com/master' into...

Merge remote-tracking branch 'ee-com/master' into 4249-show-results-from-docker-image-scan-in-the-merge-request-widget
Signed-off-by: default avatarDmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
parents 2d9b05af 42654600
This diff is collapsed.
......@@ -248,7 +248,6 @@ export default {
:project-path="projectPath"
:project-namespace="projectNamespace"
:show-delete-button="showDeleteButton"
:can-attach-file="canAttachFile"
:enable-autocomplete="enableAutocomplete"
/>
<div v-else>
......
......@@ -17,11 +17,6 @@
type: String,
required: true,
},
canAttachFile: {
type: Boolean,
required: false,
default: true,
},
enableAutocomplete: {
type: Boolean,
required: false,
......@@ -47,7 +42,6 @@
<markdown-field
:markdown-preview-path="markdownPreviewPath"
:markdown-docs-path="markdownDocsPath"
:can-attach-file="canAttachFile"
:enable-autocomplete="enableAutocomplete"
>
<textarea
......
......@@ -41,11 +41,6 @@
required: false,
default: true,
},
canAttachFile: {
type: Boolean,
required: false,
default: true,
},
enableAutocomplete: {
type: Boolean,
required: false,
......@@ -94,7 +89,6 @@
:form-state="formState"
:markdown-preview-path="markdownPreviewPath"
:markdown-docs-path="markdownDocsPath"
:can-attach-file="canAttachFile"
:enable-autocomplete="enableAutocomplete"
/>
<edit-actions
......
......@@ -9,7 +9,7 @@ export default class Job {
this.state = null;
this.options = options || $('.js-build-options').data();
this.pageUrl = this.options.pageUrl;
this.pagePath = this.options.pagePath;
this.buildStatus = this.options.buildStatus;
this.state = this.options.logState;
this.buildStage = this.options.buildStage;
......@@ -167,11 +167,11 @@ export default class Job {
getBuildTrace() {
return $.ajax({
url: `${this.pageUrl}/trace.json`,
url: `${this.pagePath}/trace.json`,
data: { state: this.state },
})
.done((log) => {
setCiStatusFavicon(`${this.pageUrl}/status.json`);
setCiStatusFavicon(`${this.pagePath}/status.json`);
if (log.state) {
this.state = log.state;
......@@ -209,7 +209,7 @@ export default class Job {
}
if (log.status !== this.buildStatus) {
gl.utils.visitUrl(this.pageUrl);
gl.utils.visitUrl(this.pagePath);
}
})
.fail(() => {
......
......@@ -8,10 +8,18 @@ document.addEventListener('DOMContentLoaded', () => new Vue({
},
data() {
const notesDataset = document.getElementById('js-vue-notes').dataset;
const parsedUserData = JSON.parse(notesDataset.currentUserData);
const currentUserData = parsedUserData ? {
id: parsedUserData.id,
name: parsedUserData.name,
username: parsedUserData.username,
avatar_url: parsedUserData.avatar_path || parsedUserData.avatar_url,
path: parsedUserData.path,
} : {};
return {
noteableData: JSON.parse(notesDataset.noteableData),
currentUserData: JSON.parse(notesDataset.currentUserData),
currentUserData,
notesData: {
lastFetchedAt: notesDataset.lastFetchedAt,
discussionsPath: notesDataset.discussionsPath,
......
import Flash from '../../../flash';
import AssigneeTitle from './assignee_title';
import Assignees from './assignees';
import Store from '../../stores/sidebar_store';
import Mediator from '../../sidebar_mediator';
import eventHub from '../../event_hub';
export default {
name: 'SidebarAssignees',
data() {
return {
mediator: new Mediator(),
store: new Store(),
loading: false,
field: '',
};
},
props: {
mediator: {
type: Object,
required: true,
},
field: {
type: String,
required: true,
},
signedIn: {
type: Boolean,
required: false,
default: false,
},
},
components: {
'assignee-title': AssigneeTitle,
assignees: Assignees,
......@@ -61,10 +71,6 @@ export default {
eventHub.$off('sidebar.removeAllAssignees', this.removeAllAssignees);
eventHub.$off('sidebar.saveAssignees', this.saveAssignees);
},
beforeMount() {
this.field = this.$el.dataset.field;
this.signedIn = typeof this.$el.dataset.signedIn !== 'undefined';
},
template: `
<div>
<assignee-title
......
<script>
import Store from '../../stores/sidebar_store';
import Mediator from '../../sidebar_mediator';
import participants from './participants.vue';
export default {
data() {
return {
mediator: new Mediator(),
store: new Store(),
};
},
props: {
mediator: {
type: Object,
required: true,
},
},
components: {
participants,
},
......@@ -21,6 +25,7 @@ export default {
<participants
:loading="store.isFetching.participants"
:participants="store.participants"
:number-of-less-participants="7" />
:number-of-less-participants="7"
/>
</div>
</template>
<script>
import Store from '../../stores/sidebar_store';
import Mediator from '../../sidebar_mediator';
import eventHub from '../../event_hub';
import Flash from '../../../flash';
import { __ } from '../../../locale';
......@@ -9,11 +8,15 @@ import subscriptions from './subscriptions.vue';
export default {
data() {
return {
mediator: new Mediator(),
store: new Store(),
};
},
props: {
mediator: {
type: Object,
required: true,
},
},
components: {
subscriptions,
},
......
......@@ -10,6 +10,27 @@ import Translate from '../vue_shared/translate';
Vue.use(Translate);
function mountAssigneesComponent(mediator) {
const el = document.getElementById('js-vue-sidebar-assignees');
if (!el) return;
// eslint-disable-next-line no-new
new Vue({
el,
components: {
SidebarAssignees,
},
render: createElement => createElement('sidebar-assignees', {
props: {
mediator,
field: el.dataset.field,
signedIn: el.hasAttribute('data-signed-in'),
},
}),
});
}
function mountConfidentialComponent(mediator) {
const el = document.getElementById('js-confidential-entry-point');
......@@ -49,9 +70,10 @@ function mountLockComponent(mediator) {
}).$mount(el);
}
function mountParticipantsComponent() {
function mountParticipantsComponent(mediator) {
const el = document.querySelector('.js-sidebar-participants-entry-point');
// eslint-disable-next-line no-new
if (!el) return;
// eslint-disable-next-line no-new
......@@ -60,11 +82,15 @@ function mountParticipantsComponent() {
components: {
sidebarParticipants,
},
render: createElement => createElement('sidebar-participants', {}),
render: createElement => createElement('sidebar-participants', {
props: {
mediator,
},
}),
});
}
function mountSubscriptionsComponent() {
function mountSubscriptionsComponent(mediator) {
const el = document.querySelector('.js-sidebar-subscriptions-entry-point');
if (!el) return;
......@@ -75,22 +101,35 @@ function mountSubscriptionsComponent() {
components: {
sidebarSubscriptions,
},
render: createElement => createElement('sidebar-subscriptions', {}),
render: createElement => createElement('sidebar-subscriptions', {
props: {
mediator,
},
}),
});
}
function mount(mediator) {
const sidebarAssigneesEl = document.getElementById('js-vue-sidebar-assignees');
// Only create the sidebarAssignees vue app if it is found in the DOM
// We currently do not use sidebarAssignees for the MR page
if (sidebarAssigneesEl) {
new Vue(SidebarAssignees).$mount(sidebarAssigneesEl);
}
function mountTimeTrackingComponent() {
const el = document.getElementById('issuable-time-tracker');
if (!el) return;
// eslint-disable-next-line no-new
new Vue({
el,
components: {
SidebarTimeTracking,
},
render: createElement => createElement('sidebar-time-tracking', {}),
});
}
export function mountSidebar(mediator) {
mountAssigneesComponent(mediator);
mountConfidentialComponent(mediator);
mountLockComponent(mediator);
mountParticipantsComponent();
mountSubscriptionsComponent();
mountParticipantsComponent(mediator);
mountSubscriptionsComponent(mediator);
new SidebarMoveIssue(
mediator,
......@@ -98,7 +137,9 @@ function mount(mediator) {
$('.js-move-issue-confirmation-button'),
).init();
new Vue(SidebarTimeTracking).$mount('#issuable-time-tracker');
mountTimeTrackingComponent();
}
export default mount;
export function getSidebarOptions() {
return JSON.parse(document.querySelector('.js-sidebar-options').innerHTML);
}
import mountSidebarEE from 'ee/sidebar/mount_sidebar';
import Mediator from 'ee/sidebar/sidebar_mediator';
import mountSidebar from './mount_sidebar';
import Mediator from './sidebar_mediator';
import { mountSidebar, getSidebarOptions } from './mount_sidebar';
function domContentLoaded() {
const sidebarOptions = JSON.parse(document.querySelector('.js-sidebar-options').innerHTML);
const mediator = new Mediator(sidebarOptions);
const mediator = new Mediator(getSidebarOptions());
mediator.fetch();
mountSidebar(mediator);
mountSidebarEE(mediator);
}
document.addEventListener('DOMContentLoaded', domContentLoaded);
......
......@@ -7,7 +7,6 @@ export default class SidebarMediator {
if (!SidebarMediator.singleton) {
this.initSingleton(options);
}
return SidebarMediator.singleton;
}
......
export default class SidebarStore {
constructor(store) {
constructor(options) {
if (!SidebarStore.singleton) {
const { currentUser, rootPath, editable } = store;
this.currentUser = currentUser;
this.rootPath = rootPath;
this.editable = editable;
this.timeEstimate = 0;
this.totalTimeSpent = 0;
this.humanTimeEstimate = '';
this.humanTimeSpent = '';
this.assignees = [];
this.isFetching = {
assignees: true,
participants: true,
subscriptions: true,
};
this.isLoading = {};
this.autocompleteProjects = [];
this.moveToProjectId = 0;
this.isLockDialogOpen = false;
this.participants = [];
this.subscribed = null;
SidebarStore.singleton = this;
this.initSingleton(options);
}
return SidebarStore.singleton;
}
initSingleton(options) {
const { currentUser, rootPath, editable } = options;
this.currentUser = currentUser;
this.rootPath = rootPath;
this.editable = editable;
this.timeEstimate = 0;
this.totalTimeSpent = 0;
this.humanTimeEstimate = '';
this.humanTimeSpent = '';
this.assignees = [];
this.isFetching = {
assignees: true,
participants: true,
subscriptions: true,
};
this.isLoading = {};
this.autocompleteProjects = [];
this.moveToProjectId = 0;
this.isLockDialogOpen = false;
this.participants = [];
this.subscribed = null;
SidebarStore.singleton = this;
}
setAssigneeData(data) {
this.isFetching.assignees = false;
if (data.assignees) {
......
......@@ -25,11 +25,6 @@
type: String,
required: false,
},
canAttachFile: {
type: Boolean,
required: false,
default: true,
},
enableAutocomplete: {
type: Boolean,
required: false,
......@@ -139,7 +134,6 @@
<markdown-toolbar
:markdown-docs-path="markdownDocsPath"
:quick-actions-docs-path="quickActionsDocsPath"
:can-attach-file="canAttachFile"
/>
</div>
</div>
......
......@@ -9,11 +9,6 @@
type: String,
required: false,
},
canAttachFile: {
type: Boolean,
required: false,
default: true,
},
},
};
</script>
......@@ -46,10 +41,7 @@
are supported
</template>
</div>
<span
v-if="canAttachFile"
class="uploading-container"
>
<span class="uploading-container">
<span class="uploading-progress-container hide">
<i
class="fa fa-file-image-o toolbar-button-icon"
......
......@@ -58,3 +58,4 @@
@import "framework/snippets";
@import "framework/memory_graph";
@import "framework/responsive_tables";
@import "framework/stacked-progress-bar";
......@@ -50,6 +50,11 @@
&:not(.disabled) {
cursor: pointer;
}
svg {
width: $gl-padding;
height: $gl-padding;
}
}
}
......
.stacked-progress-bar {
display: flex;
height: 20px;
width: 350px;
border-radius: 10px;
overflow: hidden;
background-color: $theme-gray-100;
.status-unavailable,
.status-green,
.status-neutral,
.status-red, {
height: 100%;
font-weight: normal;
color: $white-light;
line-height: 20px;
&.has-value {
padding: 0 10px;
}
&:hover {
cursor: pointer;
}
}
.status-unavailable {
padding: 0 10px;
color: $theme-gray-700;
}
.status-green {
background-color: $green-500;
&:hover {
background-color: $green-600;
}
}
.status-neutral {
background-color: $theme-gray-200;
color: $gl-gray-dark;
&:hover {
background-color: $theme-gray-300;
}
}
.status-red {
background-color: $red-500;
&:hover {
background-color: $red-600;
}
}
}
.geo-admin-container {
.page-title,
.page-title-separator {
margin-top: 10px;
}
.title-text {
line-height: 34px;
}
.page-subtitle {
margin-bottom: 24px;
}
}
.geo-node-status {
td {
vertical-align: top;
}
.help-block {
width: 135px;
text-align: right;
}
.node-info {
font-weight: $gl-font-weight-bold;
}
.event-timestamp {
font-weight: normal;
color: $theme-gray-800;
}
.sync-status {
font-weight: normal;
svg {
vertical-align: middle;
}
.sync-status-icon svg,
.sync-status-timestamp {
fill: $theme-gray-700;
color: $theme-gray-700;
}
&.sync-status-failure {
.sync-status-icon svg,
.sync-status-timestamp {
fill: $red-700;
color: $red-700;
}
}
}
}
.advanced-geo-node-status-toggler {
display: block;
.show-advance-chevron {
margin-top: 2px;
}
}
.geo-node-healthy {
color: $gl-success;
}
......@@ -52,6 +117,20 @@
white-space: pre-wrap;
}
.geo-nodes {
.health-message {
padding: 1px 8px;
background-color: $red-100;
color: $red-500;
border-radius: $border-radius-default;
font-weight: 500;
}
}
.geo-node-version-mismatch {
color: $gl-danger;
}
.node-badge {
color: $white-light;
display: inline-block;
......
......@@ -471,7 +471,8 @@
}
}
.milestone-title span {
.milestone-title span,
.collapse-truncated-title {
@include str-truncated(100%);
display: block;
margin: 0 4px;
......
......@@ -811,6 +811,10 @@
.failed {
color: $red-500;
}
.neutral {
color: $gl-gray-light;
}
}
}
}
......
module UploadsActions
include Gitlab::Utils::StrongMemoize
def create
link_to_file = UploadService.new(model, params[:file], uploader_class).execute
......@@ -24,4 +26,25 @@ module UploadsActions
send_file uploader.file.path, disposition: disposition
end
private
def uploader
strong_memoize(:uploader) do
return if show_model.nil?
file_uploader = FileUploader.new(show_model, params[:secret])
file_uploader.retrieve_from_store!(params[:filename])
file_uploader
end
end
def image_or_video?
uploader && uploader.exists? && uploader.image_or_video?
end
def uploader_class
FileUploader
end
end
class Groups::UploadsController < Groups::ApplicationController
include UploadsActions
skip_before_action :group, if: -> { action_name == 'show' && image_or_video? }
before_action :authorize_upload_file!, only: [:create]
private
def show_model
strong_memoize(:show_model) do
group_id = params[:group_id]
Group.find_by_full_path(group_id)
end
end
def authorize_upload_file!
render_404 unless can?(current_user, :upload_file, group)
end
def uploader
strong_memoize(:uploader) do
file_uploader = uploader_class.new(show_model, params[:secret])
file_uploader.retrieve_from_store!(params[:filename])
file_uploader
end
end
def uploader_class
NamespaceFileUploader
end
alias_method :model, :group
end
......@@ -8,31 +8,13 @@ class Projects::UploadsController < Projects::ApplicationController
private
def uploader
return @uploader if defined?(@uploader)
def show_model
strong_memoize(:show_model) do
namespace = params[:namespace_id]
id = params[:project_id]
namespace = params[:namespace_id]
id = params[:project_id]
file_project = Project.find_by_full_path("#{namespace}/#{id}")
if file_project.nil?
@uploader = nil
return
Project.find_by_full_path("#{namespace}/#{id}")
end
@uploader = FileUploader.new(file_project, params[:secret])
@uploader.retrieve_from_store!(params[:filename])
@uploader
end
def image_or_video?
uploader && uploader.exists? && uploader.image_or_video?
end
def uploader_class
FileUploader
end
alias_method :model, :project
......
......@@ -20,8 +20,7 @@ module BuildsHelper
def javascript_build_options
{
page_url: project_job_url(@project, @build),
build_url: project_job_url(@project, @build, :json),
page_path: project_job_path(@project, @build),
build_status: @build.status,
build_stage: @build.stage,
log_state: ''
......
......@@ -234,7 +234,7 @@ module Ci
variables += pipeline.predefined_variables
variables += runner.predefined_variables if runner
variables += project.container_registry_variables
variables += project.deployment_variables if has_environment?
variables += project.deployment_variables(environment: environment) if has_environment?
variables += project.auto_devops_variables
variables += yaml_variables
variables += user_variables
......
module Ci
class JobArtifact < ActiveRecord::Base
include AfterCommitQueue
extend Gitlab::Ci::Model
belongs_to :project
......@@ -9,6 +10,12 @@ module Ci
mount_uploader :file, JobArtifactUploader
after_save if: :file_changed?, on: [:create, :update] do
run_after_commit do
file.schedule_migration_to_object_storage
end
end
enum file_type: {
archive: 1,
metadata: 2
......
......@@ -3,7 +3,7 @@ module Ci
extend Gitlab::Ci::Model
include HasVariable
include Presentable
prepend EE::Ci::Variable
prepend HasEnvironmentScope
belongs_to :project
......
module Clusters
class Cluster < ActiveRecord::Base
include Presentable
prepend HasEnvironmentScope
self.table_name = 'clusters'
......@@ -26,6 +27,7 @@ module Clusters
accepts_nested_attributes_for :platform_kubernetes, update_only: true
validates :name, cluster_name: true
validate :unique_environment_scope, if: :has_project?
validate :restrict_modification, on: :update
delegate :status, to: :provider, allow_nil: true
......@@ -91,6 +93,15 @@ module Clusters
private
def unique_environment_scope
if project.clusters.where(environment_scope: environment_scope).where.not(id: self.id).exists?
errors.add(:base, "cannot add duplicated environment scope")
return false
end
true
end
def restrict_modification
if provider&.on_creation?
errors.add(:base, "cannot modify during creation")
......@@ -99,5 +110,9 @@ module Clusters
true
end
def has_project?
projects.exists?
end
end
end
......@@ -137,16 +137,20 @@ class Environment < ActiveRecord::Base
end
end
def deployment_platform
project.deployment_platform(environment: self)
end
def has_terminals?
project.deployment_platform.present? && available? && last_deployment.present?
deployment_platform.present? && available? && last_deployment.present?
end
def terminals
project.deployment_platform.terminals(self) if has_terminals?
deployment_platform.terminals(self) if has_terminals?
end
def rollout_status
project.deployment_platform.rollout_status(self) if has_terminals?
deployment_platform.rollout_status(self) if has_terminals?
end
def has_metrics?
......
......@@ -120,104 +120,22 @@ class GeoNode < ActiveRecord::Base
end
end
def projects_include?(project_id)
return true if restricted_project_ids.nil?
restricted_project_ids.include?(project_id)
end
def restricted_project_ids
return unless namespaces.presence
relations = namespaces.map { |namespace| namespace.all_projects.select(:id) }
Project.unscoped
.from("(#{Gitlab::SQL::Union.new(relations).to_sql}) #{Project.table_name}")
.pluck(:id)
end
def lfs_objects
relation =
if restricted_project_ids
LfsObject.joins(:projects).where(projects: { id: restricted_project_ids })
else
LfsObject.all
end
relation.with_files_stored_locally
end
def projects
if restricted_project_ids
Project.where(id: restricted_project_ids)
if selective_sync?
Project.where(namespace_id: Gitlab::GroupHierarchy.new(namespaces).base_and_descendants.select(:id))
else
Project.all
end
end
def project_registries
if restricted_project_ids
Geo::ProjectRegistry.where(project_id: restricted_project_ids)
else
Geo::ProjectRegistry.all
end
end
def filtered_project_registries(type = nil)
case type
when 'repository'
project_registries.failed_repos
when 'wiki'
project_registries.failed_wikis
else
project_registries.failed
end
end
def uploads
if restricted_project_ids
uploads_table = Upload.arel_table
group_uploads = uploads_table[:model_type].eq('Namespace').and(uploads_table[:model_id].in(Gitlab::Geo.current_node.namespace_ids))
project_uploads = uploads_table[:model_type].eq('Project').and(uploads_table[:model_id].in(restricted_project_ids))
other_uploads = uploads_table[:model_type].not_in(%w[Namespace Project])
Upload.where(group_uploads.or(project_uploads).or(other_uploads))
else
Upload.all
end
end
def lfs_objects_synced_count
return unless secondary?
relation = Geo::FileRegistry.lfs_objects.synced
if restricted_project_ids
relation = relation.where(file_id: lfs_objects.pluck(:id))
end
relation.count
end
def lfs_objects_failed_count
return unless secondary?
Geo::FileRegistry.lfs_objects.failed.count
end
def attachments_synced_count
return unless secondary?
upload_ids = uploads.pluck(:id)
synced_ids = Geo::FileRegistry.attachments.synced.pluck(:file_id)
def projects_include?(project_id)
return true unless selective_sync?
(synced_ids & upload_ids).length
projects.where(id: project_id).exists?
end
def attachments_failed_count
return unless secondary?
Geo::FileRegistry.attachments.failed.count
def selective_sync?
namespaces.exists?
end
def find_or_build_status
......
......@@ -2,7 +2,7 @@ class GeoNodeStatus < ActiveRecord::Base
belongs_to :geo_node
# Whether we were successful in reaching this node
attr_accessor :success
attr_accessor :success, :health_status, :version, :revision
# Be sure to keep this consistent with Prometheus naming conventions
PROMETHEUS_METRICS = {
......@@ -26,7 +26,6 @@ class GeoNodeStatus < ActiveRecord::Base
def self.current_node_status
current_node = Gitlab::Geo.current_node
return unless current_node
status = current_node.find_or_build_status
......@@ -47,8 +46,8 @@ class GeoNodeStatus < ActiveRecord::Base
end
def self.allowed_params
excluded_params = %w(id last_successful_status_check_at created_at updated_at).freeze
extra_params = %w(success health last_event_timestamp cursor_last_event_timestamp).freeze
excluded_params = %w(id created_at updated_at).freeze
extra_params = %w(success health health_status last_event_timestamp cursor_last_event_timestamp version revision).freeze
self.column_names - excluded_params + extra_params
end
......@@ -63,21 +62,21 @@ class GeoNodeStatus < ActiveRecord::Base
latest_event = Geo::EventLog.latest_event
self.last_event_id = latest_event&.id
self.last_event_date = latest_event&.created_at
self.repositories_count = geo_node.projects.count
self.lfs_objects_count = geo_node.lfs_objects.count
self.attachments_count = geo_node.uploads.count
self.repositories_count = projects_finder.count_projects
self.lfs_objects_count = lfs_objects_finder.count_lfs_objects
self.attachments_count = attachments_finder.count_attachments
self.last_successful_status_check_at = Time.now
if Gitlab::Geo.secondary?
self.db_replication_lag_seconds = Gitlab::Geo::HealthCheck.db_replication_lag_seconds
self.cursor_last_event_id = Geo::EventLogState.last_processed&.event_id
self.cursor_last_event_date = Geo::EventLog.find_by(id: self.cursor_last_event_id)&.created_at
self.repositories_synced_count = geo_node.project_registries.synced.count
self.repositories_failed_count = geo_node.project_registries.failed.count
self.lfs_objects_synced_count = geo_node.lfs_objects_synced_count
self.lfs_objects_failed_count = geo_node.lfs_objects_failed_count
self.attachments_synced_count = geo_node.attachments_synced_count
self.attachments_failed_count = geo_node.attachments_failed_count
self.repositories_synced_count = projects_finder.count_synced_project_registries
self.repositories_failed_count = projects_finder.count_failed_project_registries
self.lfs_objects_synced_count = lfs_objects_finder.count_synced_lfs_objects
self.lfs_objects_failed_count = lfs_objects_finder.count_failed_lfs_objects
self.attachments_synced_count = attachments_finder.count_synced_attachments
self.attachments_failed_count = attachments_finder.count_failed_attachments
end
self
......@@ -89,6 +88,10 @@ class GeoNodeStatus < ActiveRecord::Base
status_message.blank? || status_message == 'Healthy'.freeze
end
def health_status
@health_status || (healthy? ? 'Healthy' : 'Unhealthy')
end
def last_successful_status_check_timestamp
self.last_successful_status_check_at.to_i
end
......@@ -131,6 +134,18 @@ class GeoNodeStatus < ActiveRecord::Base
private
def attachments_finder
@attachments_finder ||= Geo::AttachmentRegistryFinder.new(current_node: geo_node)
end
def lfs_objects_finder
@lfs_objects_finder ||= Geo::LfsObjectRegistryFinder.new(current_node: geo_node)
end
def projects_finder
@projects_finder ||= Geo::ProjectRegistryFinder.new(current_node: geo_node)
end
def sync_percentage(total, synced)
return 0 if !total.present? || total.zero?
......
......@@ -344,6 +344,10 @@ class Group < Namespace
end
end
def hashed_storage?(_feature)
false
end
private
def update_two_factor_requirement
......
class LfsObject < ActiveRecord::Base
prepend EE::LfsObject
include AfterCommitQueue
has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
has_many :projects, through: :lfs_objects_projects
......@@ -10,6 +11,12 @@ class LfsObject < ActiveRecord::Base
mount_uploader :file, LfsObjectUploader
after_save if: :file_changed?, on: [:create, :update] do
run_after_commit do
file.schedule_migration_to_object_storage
end
end
def project_allowed_access?(project)
projects.exists?(project.lfs_storage_project.id)
end
......
......@@ -232,7 +232,6 @@ class Project < ActiveRecord::Base
delegate :members, to: :team, prefix: true
delegate :add_user, :add_users, to: :team
delegate :add_guest, :add_reporter, :add_developer, :add_master, to: :team
delegate :empty_repo?, to: :repository
# Validations
validates :creator, presence: true, on: :create
......@@ -507,6 +506,10 @@ class Project < ActiveRecord::Base
auto_devops&.enabled.nil? && !current_application_settings.auto_devops_enabled?
end
def empty_repo?
repository.empty?
end
def repository_storage_path
Gitlab.config.repositories.storages[repository_storage].try(:[], 'path')
end
......@@ -905,8 +908,7 @@ class Project < ActiveRecord::Base
@ci_service ||= ci_services.reorder(nil).find_by(active: true)
end
# TODO: This will be extended for multiple enviroment clusters
def deployment_platform
def deployment_platform(environment: nil)
@deployment_platform ||= clusters.find_by(enabled: true)&.platform_kubernetes
@deployment_platform ||= services.where(category: :deployment).reorder(nil).find_by(active: true)
end
......@@ -1557,10 +1559,8 @@ class Project < ActiveRecord::Base
ProtectedTag.protected?(self, ref)
end
def deployment_variables
return [] unless deployment_platform
deployment_platform.predefined_variables
def deployment_variables(environment: nil)
deployment_platform(environment: environment)&.predefined_variables || []
end
def auto_devops_variables
......
......@@ -44,7 +44,7 @@ class Repository
issue_template_names merge_request_template_names).freeze
# Methods that use cache_method but only memoize the value
MEMOIZED_CACHED_METHODS = %i(license empty_repo?).freeze
MEMOIZED_CACHED_METHODS = %i(license).freeze
# Certain method caches should be refreshed when certain types of files are
# changed. This Hash maps file types (as returned by Gitlab::FileDetector) to
......@@ -504,7 +504,11 @@ class Repository
end
cache_method :exists?
delegate :empty?, to: :raw_repository
def empty?
return true unless exists?
!has_visible_content?
end
cache_method :empty?
# The size of this repository in megabytes.
......@@ -993,13 +997,8 @@ class Repository
end
end
def empty_repo?
!exists? || !has_visible_content?
end
cache_method :empty_repo?, memoize_only: true
def search_files_by_content(query, ref)
return [] if empty_repo? || query.blank?
return [] if empty? || query.blank?
offset = 2
args = %W(grep -i -I -n --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref})
......@@ -1008,7 +1007,7 @@ class Repository
end
def search_files_by_name(query, ref)
return [] if empty_repo? || query.blank?
return [] if empty? || query.blank?
args = %W(ls-tree --full-tree -r #{ref || root_ref} --name-status | #{Regexp.escape(query)})
......
......@@ -16,6 +16,7 @@ class SystemNoteMetadata < ActiveRecord::Base
opened closed merged duplicate locked unlocked
outdated
approved unapproved relate unrelate
epic_issue_added issue_added_to_epic epic_issue_removed issue_removed_from_epic
].freeze
validates :note, presence: true
......
......@@ -40,6 +40,7 @@ class GroupPolicy < BasePolicy
rule { guest }.policy do
enable :read_group
enable :read_list
enable :upload_file
end
rule { admin } .enable :read_group
......
......@@ -7,6 +7,7 @@ class GeoNodeStatusEntity < Grape::Entity
expose :health do |node|
node.healthy? ? 'Healthy' : node.health
end
expose :health_status
expose :attachments_count
expose :attachments_synced_count
......@@ -37,4 +38,23 @@ class GeoNodeStatusEntity < Grape::Entity
expose :cursor_last_event_timestamp
expose :last_successful_status_check_timestamp
expose :version
expose :revision
expose :namespaces, using: NamespaceEntity
private
def namespaces
object.geo_node.namespaces
end
def version
Gitlab::VERSION
end
def revision
Gitlab::REVISION
end
end
class IssueSidebarEntity < IssuableSidebarEntity
prepend ::EE::IssueSidebarEntity
expose :assignees, using: API::Entities::UserBasic
end
class NamespaceEntity < Grape::Entity
expose :id, :name, :path, :kind, :full_path, :parent_id
end
......@@ -2,7 +2,7 @@ module Geo
class FileService
attr_reader :object_type, :object_db_id
DEFAULT_OBJECT_TYPES = %w[attachment avatar file personal_file].freeze
DEFAULT_OBJECT_TYPES = %w[attachment avatar file namespace_file personal_file].freeze
DEFAULT_SERVICE_TYPE = 'file'.freeze
def initialize(object_type, object_db_id)
......
......@@ -6,7 +6,8 @@ module Geo
def call(geo_node)
return GeoNodeStatus.current_node_status if geo_node.current?
data = { success: false }
data = GeoNodeStatus.find_or_initialize_by(geo_node: geo_node).attributes
data = data.merge(success: false, health_status: 'Offline')
begin
response = self.class.get(geo_node.status_url, headers: headers, timeout: timeout)
......@@ -29,8 +30,10 @@ module Geo
end
rescue Gitlab::Geo::GeoNodeNotFoundError
data[:health] = 'This GitLab instance does not appear to be configured properly as a Geo node. Make sure the URLs are using the correct fully-qualified domain names.'
data[:health_status] = 'Unhealthy'
rescue OpenSSL::Cipher::CipherError
data[:health] = 'Error decrypting the Geo secret from the database. Check that the primary uses the correct db_key_base.'
data[:health_status] = 'Unhealthy'
rescue HTTParty::Error, Timeout::Error, SocketError, SystemCallError, OpenSSL::SSL::SSLError => e
data[:health] = e.message
end
......
module ProtectedBranches
class AccessLevelParams
prepend EE::ProtectedBranches::AccessLevelParams
attr_reader :type, :params
def initialize(type, params)
@type = type
@params = params_with_default(params)
end
def access_levels
ce_style_access_level
end
private
def params_with_default(params)
params[:"#{type}_access_level"] ||= Gitlab::Access::MASTER if use_default_access_level?(params)
params
end
def use_default_access_level?(params)
true
end
def ce_style_access_level
access_level = params[:"#{type}_access_level"]
return [] unless access_level
[{ access_level: access_level }]
end
end
end
module ProtectedBranches
class ApiService < BaseService
prepend EE::ProtectedBranches::ApiService
def create
@push_params = AccessLevelParams.new(:push, params)
@merge_params = AccessLevelParams.new(:merge, params)
verify_params!
protected_branch_params = {
name: params[:name],
push_access_levels_attributes: @push_params.access_levels,
merge_access_levels_attributes: @merge_params.access_levels
}
::ProtectedBranches::CreateService.new(@project, @current_user, protected_branch_params).execute
end
private
def verify_params!
# EE-only
end
end
end
......@@ -566,6 +566,36 @@ module SystemNoteService
create_note(NoteSummary.new(noteable, noteable.project, user, body, action: 'unrelate'))
end
def epic_issue(epic, issue, user, type)
return unless validate_epic_issue_action_type(type)
action = type == :added ? 'epic_issue_added' : 'epic_issue_removed'
body = "#{type} issue #{issue.to_reference(epic.group)}"
create_note(NoteSummary.new(epic, nil, user, body, action: action))
end
def issue_on_epic(issue, epic, user, type)
return unless validate_epic_issue_action_type(type)
if type == :added
direction = 'to'
action = 'issue_added_to_epic'
else
direction = 'from'
action = 'issue_removed_from_epic'
end
body = "#{type} #{direction} epic #{epic.to_reference(issue.project)}"
create_note(NoteSummary.new(issue, issue.project, user, body, action: action))
end
def validate_epic_issue_action_type(type)
[:added, :removed].include?(type)
end
# Called when the merge request is approved by user
#
# noteable - Noteable object
......
......@@ -29,11 +29,11 @@ class FileUploader < GitlabUploader
# model - Object that responds to `full_path` and `disk_path`
#
# Returns a String without a trailing slash
def self.dynamic_path_segment(project)
if project.hashed_storage?(:attachments)
dynamic_path_builder(project.disk_path)
def self.dynamic_path_segment(model)
if model.hashed_storage?(:attachments)
dynamic_path_builder(model.disk_path)
else
dynamic_path_builder(project.full_path)
dynamic_path_builder(model.full_path)
end
end
......
class LfsObjectUploader < ObjectStoreUploader
storage_options Gitlab.config.lfs
after :store, :schedule_migration_to_object_storage
def self.local_store_path
Gitlab.config.lfs.storage_path
......
class NamespaceFileUploader < FileUploader
def self.base_dir
File.join(root_dir, '-', 'system', 'namespace')
end
def self.dynamic_path_segment(model)
dynamic_path_builder(model.id.to_s)
end
private
def secure_url
File.join('/uploads', @secret, file.filename)
end
end
- page_title 'Geo nodes'
%h3.page-title
Geo Nodes
- @content_class = "geo-admin-container"
%h2.page-title.clearfix
%span.title-text.pull-left
Geo Nodes
= link_to "New node", new_admin_geo_node_path, class: 'btn btn-create pull-right'
%hr.page-title-separator
%p.light
%p.page-subtitle.light
With #{link_to 'GitLab Geo', help_page_path('gitlab-geo/README'), class: 'vlink'} you can install a special
read-only and replicated instance anywhere.
Before you add nodes, follow the
......@@ -11,16 +15,6 @@
%strong exact order
they appear.
%hr
- if Gitlab::Geo.license_allows?
= form_for [:admin, @node], as: :geo_node, url: admin_geo_nodes_path, html: { class: 'form-horizontal js-geo-node-form' } do |f|
= render partial: 'form', locals: { form: f, geo_node: @node }
.form-actions
= f.submit 'Add Node', class: 'btn btn-create'
%hr
- if @nodes.any?
.panel.panel-default
.panel-heading
......@@ -35,62 +29,100 @@
.node-badge.current-node Current node
- if node.primary?
.node-badge.primary-node Primary
%span.help-block Primary node
%p
%span.help-block Primary node
%p
%span.help-block
GitLab version:
%span.js-primary-version= Gitlab::VERSION
%small.js-primary-revision
= "(#{Gitlab::REVISION})"
- else
= status_loading_icon
- if node.restricted_project_ids
%p
%span.help-block
Namespaces to replicate:
%strong.node-info
= node_selected_namespaces_to_replicate(node)
.js-geo-node-status{ style: 'display: none' }
%table.geo-node-status.js-geo-node-status.hidden
%tr
%td
.help-block
GitLab version:
%td
.node-info.prepend-top-5.prepend-left-5.js-secondary-version
- if node.enabled?
%p
%span.help-block
Health Status:
%span.js-health-status
%p
%span.help-block
Repositories synced:
%strong.node-info.js-repositories-synced
%p
%span.help-block
Repositories failed:
%strong.node-info.js-repositories-failed
%p
%span.help-block
LFS objects synced:
%strong.node-info.js-lfs-objects-synced
%p
%span.help-block
LFS objects failed:
%strong.node-info.js-lfs-objects-failed
%p
%span.help-block
Attachments synced:
%strong.node-info.js-attachments-synced
%p
%span.help-block
Attachments failed:
%strong.node-info.js-attachments-failed
%p
.advanced-geo-node-status-container
.advanced-status.hidden
%span.help-block
%tr
%td
.help-block.prepend-top-10
Health Status:
%td
.health-status.prepend-top-10.prepend-left-5.js-health-status
%tr
%td
.help-block.prepend-top-10
Repositories:
%td
.node-info.prepend-top-10.prepend-left-5.stacked-progress-bar.js-repositories
%span.status-unavailable.js-stats-unavailable
Not available
%span.status-green.js-synced
%span.status-neutral.js-waiting
%span.status-red.js-failed
%tr
%td
.help-block.prepend-top-10
LFS objects:
%td
.node-info.prepend-top-10.prepend-left-5.stacked-progress-bar.js-lfs-objects
%span.status-unavailable.js-stats-unavailable
Not available
%span.status-green.js-synced
%span.status-neutral.js-waiting
%span.status-red.js-failed
%tr
%td
.help-block.prepend-top-10
Attachments:
%td
.node-info.prepend-top-10.prepend-left-5.stacked-progress-bar.js-attachments
%span.status-unavailable.js-stats-unavailable
Not available
%span.status-green.js-synced
%span.status-neutral.js-waiting
%span.status-red.js-failed
%tr
%td
.help-block.prepend-top-10
Sync settings:
%td
.node-info.prepend-top-10.prepend-left-5.js-sync-settings
%span.js-sync-type
%span.has-tooltip.sync-status.js-sync-status
%i.sync-status-icon.js-sync-status-icon
%span.sync-status-timestamp.js-sync-status-timestamp
%tr.js-advanced-status.hidden
%td
.help-block.prepend-top-10
Database replication lag:
%strong.node-info.js-db-replication-lag
%span.help-block
%td
.node-info.prepend-top-10.prepend-left-5.js-db-replication-lag
%tr.js-advanced-status.hidden
%td
.help-block.prepend-top-10
Last event ID seen from primary:
%strong.node-info.js-last-event-seen
%span.help-block
%td
.node-info.prepend-top-10.prepend-left-5.js-last-event-seen
%span.js-event-id
%span.event-timestamp.js-event-timestamp.has-tooltip
%tr.js-advanced-status.hidden
%td
.help-block.prepend-top-10
Last event ID processed by cursor:
%strong.node-info.js-last-cursor-event
%button.btn-link.js-advanced-geo-node-status-toggler
%span> Advanced
= icon('angle-down')
%p
.js-health
%td
.node-info.prepend-top-10.prepend-left-5.js-last-cursor-event
%span.js-event-id
%span.event-timestamp.js-event-timestamp.has-tooltip
%button.btn-link.advanced-geo-node-status-toggler.js-advanced-geo-node-status-toggler
%span> Advanced
%span.js-advance-toggle.show-advance-chevron.pull-right.inline.prepend-left-5
= sprite_icon('angle-down', css_class: 's16')
%p.health-message.hidden.js-health-message
- if Gitlab::Database.read_write?
.node-actions
......
- page_title 'New node'
- @content_class = "geo-admin-container"
%h2.page-title
%span.title-text
New node
%hr.page-title-separator
- if Gitlab::Geo.license_allows?
= form_for [:admin, @node], as: :geo_node, url: admin_geo_nodes_path, html: { class: 'form-horizontal js-geo-node-form' } do |f|
= render partial: 'form', locals: { form: f, geo_node: @node }
.form-actions
= f.submit 'Add Node', class: 'btn btn-create'
......@@ -13,7 +13,15 @@
.location-badge= label
.search-input-wrap
.dropdown{ data: { url: search_autocomplete_path } }
= search_field_tag 'search', nil, placeholder: 'Search', class: 'search-input dropdown-menu-toggle no-outline js-search-dashboard-options', spellcheck: false, tabindex: '1', autocomplete: 'off', data: { toggle: 'dropdown', issues_path: issues_dashboard_url, mr_path: merge_requests_dashboard_url }, aria: { label: 'Search' }
= search_field_tag 'search', nil, placeholder: 'Search',
class: 'search-input dropdown-menu-toggle no-outline js-search-dashboard-options',
spellcheck: false,
tabindex: '1',
autocomplete: 'off',
data: { toggle: 'dropdown',
issues_path: issues_dashboard_path,
mr_path: merge_requests_dashboard_path },
aria: { label: 'Search' }
.dropdown-menu.dropdown-select
= dropdown_content do
%ul
......
......@@ -4,4 +4,10 @@
- nav "group"
- @left_sidebar = true
- content_for :page_specific_javascripts do
- if current_user
-# haml-lint:disable InlineJavaScript
:javascript
window.uploads_path = "#{group_uploads_path(@group)}";
= render template: "layouts/application"
......@@ -14,7 +14,7 @@
%td.branch-commit
- if can?(current_user, :read_build, job)
= link_to project_job_url(job.project, job) do
= link_to project_job_path(job.project, job) do
%span.build-link ##{job.id}
- else
%span.build-link ##{job.id}
......
......@@ -15,4 +15,7 @@
= s_("ClusterIntegration|All")
%span.badge= @all_count
.nav-controls
= link_to s_("ClusterIntegration|Add cluster"), new_project_cluster_path(@project), class: "btn btn-success btn-add-cluster disabled has-tooltip js-add-cluster", title: s_("ClusterIntegration|Multiple clusters are available in GitLab Entreprise Edition Premium and Ultimate")
- if @project.feature_available?(:multiple_clusters)
= link_to s_("ClusterIntegration|Add cluster"), new_project_cluster_path(@project), class: "btn btn-success btn-add-cluster has-tooltip js-add-cluster"
- else
= link_to s_("ClusterIntegration|Add cluster"), new_project_cluster_path(@project), class: "btn btn-success btn-add-cluster disabled has-tooltip js-add-cluster", title: s_("ClusterIntegration|Multiple clusters are available in GitLab Entreprise Edition Premium and Ultimate")
......@@ -3,7 +3,7 @@
- page_title _("Commits"), @ref
= content_for :meta_tags do
= auto_discovery_link_tag(:atom, project_commits_url(@project, @ref, rss_url_options), title: "#{@project.name}:#{@ref} commits")
= auto_discovery_link_tag(:atom, project_commits_path(@project, @ref, rss_url_options), title: "#{@project.name}:#{@ref} commits")
.js-project-commits-show{ 'data-commits-limit' => @limit }
%div{ class: container_class }
......
......@@ -14,4 +14,4 @@
notes_path: notes_url,
last_fetched_at: Time.now.to_i,
noteable_data: serialize_issuable(@issue),
current_user_data: UserSerializer.new.represent(current_user).to_json } }
current_user_data: UserSerializer.new.represent(current_user, only_path: true).to_json } }
......@@ -87,10 +87,11 @@
%h3.panel-title
Related issues
#merge-requests{ data: { url: referenced_merge_requests_project_issue_url(@project, @issue) } }
#merge-requests{ data: { url: referenced_merge_requests_project_issue_path(@project, @issue) } }
// This element is filled in using JavaScript.
#related-branches{ data: { url: related_branches_project_issue_url(@project, @issue) } }
#related-branches{ data: { url: related_branches_project_issue_path(@project, @issue) } }
// This element is filled in using JavaScript.
.content-block.emoji-block
......
......@@ -85,7 +85,7 @@
= icon('bug', text: 'Fogbugz')
%div
- if gitea_import_enabled?
= link_to new_import_gitea_url, class: 'btn import_gitea' do
= link_to new_import_gitea_path, class: 'btn import_gitea' do
= custom_icon('go_logo')
Gitea
%div
......
......@@ -3,5 +3,5 @@
GitLab may not work properly because you are using an outdated web browser.
%br
Please install a
= link_to 'supported web browser', help_page_url('install/requirements', anchor: 'supported-web-browsers')
= link_to 'supported web browser', help_page_path('install/requirements', anchor: 'supported-web-browsers')
for a better experience.
- todo = issuable_todo(issuable)
- content_for :page_specific_javascripts do
= page_specific_javascript_bundle_tag('common_vue')
= page_specific_javascript_bundle_tag('sidebar')
= page_specific_javascript_bundle_tag('ee_sidebar')
%aside.right-sidebar.js-right-sidebar.js-issuable-sidebar{ data: { signed: { in: current_user.present? } }, class: sidebar_gutter_collapsed_class, 'aria-live' => 'polite' }
.issuable-sidebar{ data: { endpoint: "#{issuable_json_path(issuable)}" } }
......@@ -21,6 +21,7 @@
= render "shared/issuable/sidebar_todo", todo: todo, issuable: issuable, is_collapsed: true
.block.assignee
= render "shared/issuable/sidebar_assignees", issuable: issuable, can_edit_issuable: can_edit_issuable, signed_in: current_user.present?
= render "shared/issuable/sidebar_item_epic", issuable: issuable
.block.milestone
.sidebar-collapsed-icon
= icon('clock-o', 'aria-hidden': 'true')
......
......@@ -69,6 +69,9 @@ module Geo
private
def worker_metadata
end
def db_retrieve_batch_size
DB_RETRIEVE_BATCH_SIZE
end
......@@ -119,7 +122,8 @@ module Geo
end
def schedule_jobs
num_to_schedule = [max_capacity - scheduled_job_ids.size, pending_resources.size].min
capacity = max_capacity
num_to_schedule = [capacity - scheduled_job_ids.size, pending_resources.size].min
to_schedule = pending_resources.shift(num_to_schedule)
scheduled = to_schedule.map do |args|
......@@ -129,7 +133,7 @@ module Geo
scheduled_jobs.concat(scheduled)
log_info("Loop #{loops}", enqueued: scheduled.length, pending: pending_resources.length, scheduled: scheduled_jobs.length)
log_info("Loop #{loops}", enqueued: scheduled.length, pending: pending_resources.length, scheduled: scheduled_jobs.length, capacity: capacity)
end
def scheduled_job_ids
......@@ -152,11 +156,13 @@ module Geo
def log_info(message, extra_args = {})
args = { class: self.class.name, message: message }.merge(extra_args)
args.merge!(worker_metadata) if worker_metadata
Gitlab::Geo::Logger.info(args)
end
def log_error(message, extra_args = {})
args = { class: self.class.name, message: message }.merge(extra_args)
args.merge!(worker_metadata) if worker_metadata
Gitlab::Geo::Logger.error(args)
end
end
......
......@@ -12,11 +12,9 @@ module Geo
# Prevent multiple Sidekiq workers from performing repositories clean up
try_obtain_lease do
geo_node = GeoNode.find(geo_node_id)
return unless geo_node.selective_sync?
restricted_project_ids = geo_node.restricted_project_ids
return unless restricted_project_ids
Project.where.not(id: restricted_project_ids).find_in_batches(batch_size: BATCH_SIZE) do |batch|
Project.where.not(id: geo_node.projects).find_in_batches(batch_size: BATCH_SIZE) do |batch|
batch.each do |project|
clean_up_repositories(project)
end
......
module Geo
class RepositoryShardSyncWorker < Geo::BaseSchedulerWorker
# We may have many long-running threads, so split them out
# into their own queue to make it possible for other jobs to run.
sidekiq_options queue: :geo_repository_shard_sync, retry: false
attr_accessor :shard_name
def perform(shard_name)
@shard_name = shard_name
return unless Gitlab::Geo::ShardHealthCache.healthy_shard?(shard_name)
super()
end
private
def worker_metadata
{ shard: shard_name }
end
# We need a custom key here since we are running one worker per shard
def lease_key
@lease_key ||= "#{self.class.name.underscore}:shard:#{shard_name}"
end
def max_capacity
healthy_count = Gitlab::Geo::ShardHealthCache.healthy_shard_count
# If we don't have a count, that means that for some reason
# RepositorySyncWorker stopped running/updating the cache. We might
# be trying to shut down Geo while this job may still be running.
return 0 unless healthy_count.to_i > 0
capacity_per_shard = current_node.repos_max_capacity / healthy_count
[1, capacity_per_shard.to_i].max
end
def schedule_job(project_id)
job_id = Geo::ProjectSyncWorker.perform_async(project_id, Time.now)
{ id: project_id, job_id: job_id } if job_id
end
def finder
@finder ||= ProjectRegistryFinder.new(current_node: current_node)
end
def load_pending_resources
resources = find_project_ids_not_synced(batch_size: db_retrieve_batch_size)
remaining_capacity = db_retrieve_batch_size - resources.size
if remaining_capacity.zero?
resources
else
resources + find_project_ids_updated_recently(batch_size: remaining_capacity)
end
end
def find_project_ids_not_synced(batch_size:)
shard_restriction(finder.find_unsynced_projects(batch_size: batch_size))
.reorder(last_repository_updated_at: :desc)
.pluck(:id)
end
def find_project_ids_updated_recently(batch_size:)
shard_restriction(finder.find_projects_updated_recently(batch_size: batch_size))
.order(Gitlab::Database.nulls_first_order(:last_repository_updated_at, :desc))
.pluck(:id)
end
def shard_restriction(relation)
relation.where(repository_storage: shard_name)
end
end
end
module Geo
class RepositorySyncWorker < Geo::BaseSchedulerWorker
private
class RepositorySyncWorker
include ApplicationWorker
include CronjobQueue
def max_capacity
current_node.repos_max_capacity
end
def schedule_job(project_id)
job_id = Geo::ProjectSyncWorker.perform_async(project_id, Time.now)
def perform
return unless Gitlab::Geo.geo_database_configured?
return unless Gitlab::Geo.secondary?
{ id: project_id, job_id: job_id } if job_id
end
shards = healthy_shards
def finder
@finder ||= ProjectRegistryFinder.new(current_node: current_node)
end
Gitlab::Geo::ShardHealthCache.update(shards)
def load_pending_resources
resources = find_project_ids_not_synced(batch_size: db_retrieve_batch_size)
remaining_capacity = db_retrieve_batch_size - resources.size
if remaining_capacity.zero?
resources
else
resources + find_project_ids_updated_recently(batch_size: remaining_capacity)
shards.each do |shard_name|
RepositoryShardSyncWorker.perform_async(shard_name)
end
end
def find_project_ids_not_synced(batch_size:)
healthy_shards_restriction(finder.find_unsynced_projects(batch_size: batch_size))
.reorder(last_repository_updated_at: :desc)
.pluck(:id)
end
def find_project_ids_updated_recently(batch_size:)
healthy_shards_restriction(finder.find_projects_updated_recently(batch_size: batch_size))
.order(Gitlab::Database.nulls_first_order(:last_repository_updated_at, :desc))
.pluck(:id)
end
def healthy_shards_restriction(relation)
configured = Gitlab.config.repositories.storages.keys
referenced = Project.distinct(:repository_storage).pluck(:repository_storage)
healthy = healthy_shards
known = configured | referenced
return relation if (known - healthy).empty?
relation.where(repository_storage: healthy)
end
def healthy_shards
Gitlab::HealthChecks::FsShardsCheck
.readiness
......
---
title: Add performance metrics to the merge request widget
merge_request: 3507
author:
type: added
---
title: Enhancements for Geo admin screen
merge_request: 3545
author:
type: changed
---
title: Support multiple Kubernetes cluster per project
merge_request: 3603
author:
type: added
---
title: Add system notes for issue - epic association
merge_request:
author:
type: added
---
title: Geo - Improve performance when calculating the node status
merge_request: 3595
author:
type: performance
---
title: Geo - Show GitLab version for each node in the Geo status page
merge_request:
author:
type: added
---
title: Add epic information to issue sidebar
merge_request:
author:
type: added
---
title: Fix Git message when pushing to Geo secondary
merge_request: 3616
author:
type: fixed
---
title: 'Geo: rake task to refresh foreign table schema (FDW support)'
merge_request: 3626
author:
type: added
---
title: ProtectedBranches API allows individual users and group to be specified
merge_request: 3516
author:
type: changed
---
title: 'Geo: Increase parallelism by scheduling project repositories by shard'
merge_request: 3606
author:
type: added
---
title: Document how to set up GitLab Geo for HA
merge_request: 3468
author:
type: other
---
title: Transfer job archives to object storage after creation
merge_request:
author:
type: added
---
title: Fix related branches/Merge requests failing to load when the hostname setting
is changed
merge_request:
author:
type: fixed
---
title: Add untracked files to uploads table
merge_request: 15270
author:
type: other
......@@ -771,6 +771,7 @@ test:
object_store:
enabled: false
remote_directory: artifacts # The bucket name
background_upload: false
connection:
provider: AWS # Only AWS supported at the moment
aws_access_key_id: AWS_ACCESS_KEY_ID
......
......@@ -127,7 +127,7 @@ namespace :admin do
get :download, on: :member
end
resources :geo_nodes, only: [:index, :create, :edit, :update, :destroy] do
resources :geo_nodes, only: [:index, :create, :new, :edit, :update, :destroy] do
member do
post :repair
post :toggle
......
......@@ -52,6 +52,12 @@ constraints(GroupUrlConstrainer.new) do
patch :override, on: :member ## EE-specific
end
resources :uploads, only: [:create] do
collection do
get ":secret/:filename", action: :show, as: :show, constraints: { filename: /[^\/]+/ }
end
end
## EE-specific
resource :analytics, only: [:show]
resource :ldap, only: [] do
......
......@@ -77,6 +77,7 @@
- [admin_emails, 1]
- [geo_project_sync, 1]
- [geo_file_download, 1]
- [geo_repository_shard_sync, 1]
- [elastic_batch_project_indexer, 1]
- [elastic_indexer, 1]
- [elastic_commit_indexer, 1]
......
......@@ -84,6 +84,7 @@ var config = {
registry_list: './registry/index.js',
repo: './repo/index.js',
sidebar: './sidebar/sidebar_bundle.js',
ee_sidebar: 'ee/sidebar/sidebar_bundle.js',
schedule_form: './pipeline_schedules/pipeline_schedule_form_bundle.js',
schedules_index: './pipeline_schedules/pipeline_schedules_index_bundle.js',
snippet: './snippet/snippet_bundle.js',
......
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class SetUploadsPathSizeForMysql < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
# Set this constant to true if this migration requires downtime.
DOWNTIME = false
def up
# We need at least 297 at the moment. For more detail on that number, see:
# https://gitlab.com/gitlab-org/gitlab-ce/issues/40168#what-is-the-expected-correct-behavior
#
# Rails + PostgreSQL `string` is equivalent to a `text` field, but
# Rails + MySQL `string` is `varchar(255)` by default. Also, note that we
# have an upper limit because with a unique index, MySQL has a max key
# length of 3072 bytes which seems to correspond to `varchar(1024)`.
change_column :uploads, :path, :string, limit: 511
end
def down
# It was unspecified, which is varchar(255) by default in Rails for MySQL.
change_column :uploads, :path, :string
end
end
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class TrackUntrackedUploads < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
DOWNTIME = false
MIGRATION = 'PrepareUntrackedUploads'
def up
BackgroundMigrationWorker.perform_async(MIGRATION)
end
def down
if table_exists?(:untracked_files_for_uploads)
drop_table :untracked_files_for_uploads
end
end
end
......@@ -2215,7 +2215,7 @@ ActiveRecord::Schema.define(version: 20171205190711) do
create_table "uploads", force: :cascade do |t|
t.integer "size", limit: 8, null: false
t.string "path", null: false
t.string "path", limit: 511, null: false
t.string "checksum", limit: 64
t.integer "model_id"
t.string "model_type"
......
This diff is collapsed.
This diff is collapsed.
......@@ -97,7 +97,9 @@ Example response:
"last_event_timestamp": 1509681166,
"cursor_last_event_id": 23,
"cursor_last_event_timestamp": 1509681166,
"last_successful_status_check_timestamp": 1510125024
"last_successful_status_check_timestamp": 1510125024,
"version": "10.3.0",
"revision": "33d33a096a"
}
]
```
......@@ -136,7 +138,9 @@ Example response:
"last_event_timestamp": 1509681166,
"cursor_last_event_id": 23,
"cursor_last_event_timestamp": 1509681166,
"last_successful_status_check_timestamp": 1510125268
"last_successful_status_check_timestamp": 1510125268,
"version": "10.3.0",
"revision": "33d33a096a"
}
```
......
......@@ -114,6 +114,9 @@ curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" 'https://gitl
| `name` | string | yes | The name of the branch or wildcard |
| `push_access_level` | string | no | Access levels allowed to push (defaults: `40`, master access level) |
| `merge_access_level` | string | no | Access levels allowed to merge (defaults: `40`, master access level) |
| `allowed_to_push` | array | no | Array of access levels allowed to push, with each described by a hash |
| `allowed_to_merge` | array | no | Array of access levels allowed to merge, with each described by a hash |
Example response:
......@@ -139,6 +142,39 @@ Example response:
}
```
### Example with user / group level access
Elements in the `allowed_to_push` / `allowed_to_merge` array should take the
form `{user_id: integer}`, `{group_id: integer}` or `{access_level: integer}`. Each user must have access to the project and each group must [have this project shared](../user/project/members/share_project_with_groups.md). These access levels allow [more granular control over protected branch access](../user/project/protected_branches.md#restricting-push-and-merge-access-to-certain-users) and were [added to the API in ][ee-3516] in GitLab 10.3 EE.
```bash
curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" 'https://gitlab.example.com/api/v4/projects/5/protected_branches?name=*-stable&allowed_to_push%5B%5D%5Buser_id%5D=1'
```
Example response:
```json
{
"name":"*-stable",
"push_access_levels": [
{
"access_level":null,
"user_id":1,
"group_id":null,
"access_level_description":"Administrator"
}
],
"merge_access_levels": [
{
"access_level":40,
"user_id":null,
"group_id":null,
"access_level_description":"Masters"
}
]
}
```
## Unprotect repository branches
Unprotects the given protected branch or wildcard protected branch.
......@@ -148,10 +184,12 @@ DELETE /projects/:id/protected_branches/:name
```
```bash
curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" 'https://gitlab.example.com/api/v4/projects/5/protected_branches/*-stable'
curl --request DELETE --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" 'https://gitlab.example.com/api/v4/projects/5/protected_branches/*-stable'
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
| `name` | string | yes | The name of the branch |
[ee-3516]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3516 "ProtectedBranches API handles per user/group granularity"
......@@ -127,22 +127,29 @@ sync again.
## Setup instructions
In order to set up one or more GitLab Geo instances, follow the steps below in
the **exact order** they appear. **Make sure the GitLab version is the same on
all nodes.**
These instructions assume you have a working instance of GitLab. They will
guide you through making your existing instance the primary Geo node and
adding secondary Geo nodes.
The steps below should be followed in the order they appear. **Make sure the
GitLab version is the same on all nodes.**
### Using Omnibus GitLab
If you installed GitLab using the Omnibus packages (highly recommended):
1. [Install GitLab Enterprise Edition][install-ee] on the server that will serve
as the **secondary** Geo node. Do not login or set up anything else in the
secondary node for the moment.
1. [Upload the GitLab License](../user/admin_area/license.md) on the **primary** Geo Node to unlock GitLab Geo.
1. [Setup the database replication](database.md) (`primary (read-write) <-> secondary (read-only)` topology).
1. [Lookup authorized SSH keys in the database](../administration/operations/speed_up_ssh.html), do this step for both primary AND secondary nodes.
as the **secondary** Geo node. Do not create an account or login to the new
secondary node.
1. [Upload the GitLab License](../user/admin_area/license.md) on the **primary**
Geo node to unlock GitLab Geo.
1. [Setup the database replication](database.md) (`primary (read-write) <->
secondary (read-only)` topology).
1. [Lookup authorized SSH keys in the database](../administration/operations/speed_up_ssh.html),
do this step for both primary AND secondary nodes.
1. [Configure GitLab](configuration.md) to set the primary and secondary nodes.
1. Optional: [Configure a secondary LDAP server](../administration/auth/ldap.md) for the secondary. See [notes on LDAP](#ldap).
1. Optional: [Configure a secondary LDAP server](../administration/auth/ldap.md)
for the secondary. See [notes on LDAP](#ldap).
1. [Follow the "Using a Geo Server" guide](using_a_geo_server.md).
[install-ee]: https://about.gitlab.com/downloads-ee/ "GitLab Enterprise Edition Omnibus packages downloads page"
......@@ -152,11 +159,14 @@ If you installed GitLab using the Omnibus packages (highly recommended):
If you installed GitLab from source:
1. [Install GitLab Enterprise Edition][install-ee-source] on the server that
will serve as the **secondary** Geo node. Do not login or set up anything
else in the secondary node for the moment.
1. [Upload the GitLab License](../user/admin_area/license.md) on the **primary** Geo Node to unlock GitLab Geo.
1. [Setup the database replication](database_source.md) (`primary (read-write) <-> secondary (read-only)` topology).
1. [Lookup authorized SSH keys in the database](../administration/operations/speed_up_ssh.html), do this step for both primary AND secondary nodes.
will serve as the **secondary** Geo node. Do not create an account or login
to the new secondary node.
1. [Upload the GitLab License](../user/admin_area/license.md) on the **primary**
Geo node to unlock GitLab Geo.
1. [Setup the database replication](database_source.md) (`primary (read-write)
<-> secondary (read-only)` topology).
1. [Lookup authorized SSH keys in the database](../administration/operations/speed_up_ssh.html),
do this step for both primary AND secondary nodes.
1. [Configure GitLab](configuration_source.md) to set the primary and secondary
nodes.
1. [Follow the "Using a Geo Server" guide](using_a_geo_server.md).
......@@ -171,6 +181,10 @@ Read through the [GitLab Geo configuration](configuration.md) documentation.
Read how to [update your Geo nodes to the latest GitLab version](updating_the_geo_nodes.md).
## Configuring GitLab Geo HA
Read through the [Geo High Availability documentation](ha.md).
## Current limitations
- You cannot push code to secondary nodes
......@@ -202,6 +216,10 @@ example:
This message shows that Geo detected that a repository update was needed for project 1.
## Tuning Geo
Read the [GitLab Geo tuning](tuning.md) documentation.
## Troubleshooting
Read the [troubleshooting document](troubleshooting.md).
......
......@@ -59,6 +59,13 @@ Then save and close the file.
service gitlab restart
```
The secondary will start automatically replicating missing data from the
primary in a process known as backfill. Meanwhile, the primary node will start
to notify changes to the secondary, which will act on those notifications
immediately. Make sure the secondary instance is running and accessible.
### Step 2. (Optional) Enabling hashed storage
Once restarted, the secondary will automatically start replicating missing data
from the primary in a process known as backfill. Meanwhile, the primary node
will start to notify the secondary of any changes, so that the secondary can
......@@ -87,7 +94,33 @@ cp primary.geo.example.com.crt /usr/local/share/ca-certificates
update-ca-certificates
```
### Step 4. Managing the secondary GitLab node
Congratulations! Your secondary geo node is now configured!
The initial replication, or 'backfill', will probably still be in progress.
You can monitor the synchronization process on each geo node from the primary
node's Geo Nodes dashboard (Admin Area ➔ Geo Nodes, `/admin/geo_nodes`) in your
browser.
![GitLab Geo dashboard](img/geo-node-dashboard.png)
After the backfill is completed you can continue to monitor geo node health and
replication delays from the dashboard.
The two most obvious issues that can become apparent in the dashboard are:
1. Database replication not working well
1. Instance to instance notification not working. In that case, it can be
something of the following:
- You are using a custom certificate or custom CA (see the
### Step 4. Enable Git access over HTTP/HTTPS
- Instance is firewalled (check your firewall rules)
Please note that disabling a secondary node will stop the sync process.
Please note that if `git_data_dirs` is customized on the primary for multiple
repository shards you must duplicate the same configuration on the secondary.
GitLab Geo synchronizes repositories over HTTP/HTTPS, and so requires this clone
method to be enabled. Navigate to **Admin Area ➔ Settings**
......
This diff is collapsed.
......@@ -229,11 +229,14 @@ primary before the database is replicated.
(`/admin/geo_nodes`) in your browser.
1. Add the secondary node by providing its full URL. **Do NOT** check the box
'This is a primary node'.
1. Added in GitLab 9.5: Choose which namespaces should be replicated by the
1. Optionally, choose which namespaces should be replicated by the
secondary node. Leave blank to replicate all. Read more in
[selective replication](#selective-replication).
1. Click the **Add node** button.
The new secondary geo node will have the status **Unhealthy**. This is expected
because we have not yet configured the secondary server. This is the next step.
### Step 3. Configure the secondary server
1. SSH into your GitLab **secondary** server and login as root:
......@@ -273,10 +276,10 @@ primary before the database is replicated.
```
# Certificate and key currently used by GitLab, and connecting by FQDN
sudo -u postgres psql -h primary.geo.example.com -U gitlab_replicator -d "dbname=gitlabhq_production sslmode=verify-ca" -W
sudo -u postgres psql -U gitlab_replicator -d "dbname=gitlabhq_production sslmode=verify-ca" -W -h primary.geo.example.com
# Self-signed certificate and key, or connecting by IP address
sudo -u postgres psql -h 1.2.3.4 -U gitlab_replicator -d "dbname=gitlabhq_production sslmode=verify-ca" -W
sudo -u postgres psql -U gitlab_replicator -d "dbname=gitlabhq_production sslmode=verify-ca" -W -h 1.2.3.4
```
When prompted enter the password you set in the first step for the
......@@ -473,5 +476,4 @@ We don't support MySQL replication for GitLab Geo.
Read the [troubleshooting document](troubleshooting.md).
[pgback]: http://www.postgresql.org/docs/9.6/static/app-pgbasebackup.html
[reconfigure GitLab]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure
[toc]: README.md#using-gitlab-installed-from-source
# GitLab Geo High Availability
This document describes a possible configuration on how to set up Geo
in a Highly Available environment. If your HA setup differs from the one
described in this document, you still can use the instructions and adapt them
to your needs.
## Architecture overview
![Active/Active HA Diagram](../administration/img/high_availability/active-active-diagram.png)
This documentation assumes all machines used in this HA setup can
communicate over the network using internal IP addresses.
NOTE: **Note:**
`external_url` must be the same for every machine, and `https` should be used.
## Services machine
One machine, called the Services machine will be used to run:
- NFS shares
- PostgreSQL
- Redis
- HAProxy
### Prerequisites
Make sure you have GitLab EE installed using the
[Omnibus package](https://about.gitlab.com/installation).
The following steps should be performed in the Services machine. SSH to it
and login as root:
```sh
sudo -i
```
### Step 1: Set up NFS share
1. Install the required NFS packages:
```sh
apt-get install nfs-kernel-server
```
1. Create the required directories:
```sh
mkdir -p /var/opt/gitlab/nfs/builds/ \
/var/opt/gitlab/nfs/git-data/ \
/var/opt/gitlab/nfs/shared/ \
/var/opt/gitlab/nfs/uploads/
```
1. Make the directories available through NFS, by adding this to
`/etc/exports` (see also the [NFS HA recommended options](../administration/high_availability/nfs.md#recommended-options)):
```
/var/opt/gitlab/nfs *(rw,sync,no_root_squash)
```
1. Start the NFS service:
```sh
systemctl start nfs-kernel-server.service
```
1. Apply the settings to take effect:
```sh
exportfs -a
```
### Step 2: Set up PostgreSQL server
1. Edit `/etc/gitlab/gitlab.rb` and add the following:
```ruby
postgresql['enable'] = true
##
## Replace 1.2.3.4 with the internal IP address of the current machine and
## 2.3.4.5 and 3.4.5.6 with the internal IP addresses of the machines
## running the Application server(s).
##
postgresql['listen_address'] = '1.2.3.4'
postgresql['trust_auth_cidr_addresses'] = ['1.2.3.4/32', '2.3.4.5/32', '3.4.5.6/32']
gitlab_rails['auto_migrate'] = true
gitlab_rails['db_password'] = 'DB password'
```
1. **Only for secondary nodes** Also add this to `/etc/gitlab/gitlab.rb`:
```ruby
geo_postgresql['enable'] = true
##
## Replace 1.2.3.4 with the internal IP address of the current machine and
## 2.3.4.5 and 3.4.5.6 with the internal IP addresses of the machines
## running the Application server(s).
##
geo_postgresql['listen_address'] = '1.2.3.4'
geo_postgresql['trust_auth_cidr_addresses'] = ['1.2.3.4/32', '2.3.4.5/32', '3.4.5.6/32']
geo_secondary['auto_migrate'] = true
geo_secondary['db_host'] = '1.2.3.4'
geo_secondary['db_password'] = 'Geo tracking DB password'
```
### Step 3: Set up Redis
Edit `/etc/gitlab/gitlab.rb` and add the following:
```ruby
redis['enable'] = true
redis['password'] = 'Redis password'
##
## Replace 1.2.3.4 with the internal IP address of the current machine
##
redis['bind'] = '1.2.3.4'
redis['port'] = 6379
##
## Needed because 'gitlab-ctl reconfigure' runs 'rake cache:clear:redis'
##
gitlab_rails['redis_password'] = 'Redis password'
```
### Step 4: HAProxy
We'll be using HAProxy to balance the load between the Application machines.
1. Manually stop Nginx (we will disable it in `/etc/gitlab/gitlab.rb` later):
```sh
gitlab-ctl stop nginx
```
1. Install the HAProxy package:
```sh
apt-get install haproxy
```
1. Make sure you have a single SSL `pem` file containing the
certificate and the private key.
```sh
cat /etc/ssl/cert.pem /etc/ssl/privkey.pem > /etc/ssl/aio.pem
```
1. Edit `/etc/haproxy/haproxy.cfg` and overwrite it with the following:
```
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
maxconn 4096
user haproxy
group haproxy
daemon
defaults
log global
mode http
option httplog
option dontlognull
option forwardfor
option http-server-close
stats enable
stats uri /haproxy?stats
frontend www-http
bind *:80
reqadd X-Forwarded-Proto:\ http
default_backend www-backend
frontend www-https
bind 0.0.0.0:443 ssl crt /etc/ssl/aio.pem
reqadd X-Forwarded-Proto:\ https
default_backend www-backend
backend www-backend
redirect scheme https if !{ ssl_fc }
balance leastconn
option httpclose
option forwardfor
cookie JSESSIONID prefix
##
## Enter the IPs of your Application servers here
##
server nodeA 2.3.4.5:80 cookie A check
server nodeB 3.4.5.6:80 cookie A check
```
1. Start the HAProxy service:
```sh
service haproxy start
```
### Step 5: Apply settings
1. Edit `/etc/gitlab/gitlab.rb` and add the following:
```ruby
nginx['enable'] = false
sidekiq['enable'] = false
unicorn['enable'] = false
##
## These are optional/untested/irrelevant
##
gitaly['enable'] = false
gitlab_workhorse['enable'] = false
mailroom['enable'] = false
prometheus['enable'] = false
```
1. [Reconfigure GitLab][] for the changes to take effect.
1. Until [Omnibus#2797](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/2797)
gets fixed, you will need to manually restart PostgreSQL:
```sh
gitlab-ctl restart postgresql geo-postgresql
```
### Step 6: Step up database replication
Database replication will operate between the Services machines.
Follow the [Setup the database replication](database.md) instructions
to set up.
## Application machine
Repeat these steps for every machine running `gitlab-rails`.
The following steps should be performed in the Application machine. SSH to it
and login as root:
```sh
sudo -i
```
### Step 1: Add NFS mount
1. Install the required NFS packages:
```sh
apt-get install nfs-common
```
1. Create the mount point directory:
```sh
mkdir -p /mnt/nfs
```
1. Edit `/etc/fstab` and add the following lines
(where `1.2.3.4` is the internal IP of the Services machine):
```
1.2.3.4:/var/opt/gitlab/nfs /mnt/nfs nfs defaults,nfsvers=4,soft,rsize=1048576,wsize=1048576,noatime
```
1. Mount the share:
```sh
mount -a -t nfs
```
You can check if the mount is working by checking the existence of the
directories `builds/`, `git-data/`, `shared/`, and `uploads/` in
`/mnt/nfs`:
```
ls /mnt/nfs
```
### Step 2: Configure proxied SSL
The load balancer will take care of the SSL termination, so configure nginx to
work with proxied SSL.
Follow the instructions to [configure proxied SSL](https://docs.gitlab.com/omnibus/settings/nginx.html#supporting-proxied-ssl).
### Step 3: Configure connections to the Services machine
1. Edit `/etc/gitlab/gitlab.rb` and add the following:
```ruby
##
## Use the NFS mount to store data
##
gitlab_rails['uploads_directory'] = '/mnt/nfs/uploads'
gitlab_rails['shared_path'] = '/mnt/nfs/shared'
gitlab_ci['builds_directory'] = '/mnt/nfs/builds'
git_data_dirs({
'default': {
'path': '/mnt/nfs/git-data'
}
})
high_availability['mountpoint'] = '/mnt/nfs'
##
## Disable PostgreSQL on the local machine and connect to the remote
##
postgresql['enable'] = false
gitlab_rails['auto_migrate'] = false
gitlab_rails['db_host'] = '1.2.3.4'
gitlab_rails['db_password'] = 'DB password'
##
## Disable Redis on the local machine and connect to the remote
##
redis['enable'] = false
gitlab_rails['redis_host'] = '1.2.3.4'
gitlab_rails['redis_password'] = 'Redis password'
```
1. **[Only for primary nodes]** Add the following to `/etc/gitlab/gitlab.rb`:
```ruby
geo_primary_role['enable'] = true
```
1. **[Only for secondary nodes]** Add the following to `/etc/gitlab/gitlab.rb`:
```ruby
geo_secondary_role['enable'] = true
geo_postgresql['enable'] = true # TODO set to false when https://gitlab.com/gitlab-org/omnibus-gitlab/issues/2980 is fixed
geo_secondary['auto_migrate'] = false
geo_secondary['db_host'] = '1.2.3.4'
geo_secondary['db_password'] = 'Geo tracking DB password'
```
1. Copy the database encryption key. Follow the instructions of
[Step 1. Copying the database encryption key](configuration.md#step-1-copying-the-database-encryption-key)
1. [Reconfigure GitLab][] for the changes to take effect (if you haven't done
this yet in previous step).
1. [Restart GitLab][] to start the processes with the correct connections.
## Troubleshooting
### HAProxy
You can connect to `https://example.com/haproxy?stats` to monitor the
load balancing between the Application machines.
[reconfigure GitLab]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure
[restart GitLab]: ../administration/restart_gitlab.md#omnibus-gitlab-restart
# Tuning GitLab Geo
## Changing the sync capacity values
In the Geo admin page (`/admin/geo_nodes`), there are several variables that
can be tuned to improve performance of Geo:
* Repository sync capacity
* File sync capacity
Increasing these values will increase the number of jobs that are scheduled,
but this may not lead to a more downloads in parallel unless the number of
available Sidekiq threads is also increased. For example, if repository sync
capacity is increased from 25 to 50, you may also want to increase the number
of Sidekiq threads from 25 to 50. See the [Sidekiq concurrency
documentation](../administration/operations/extra_sidekiq_processes.html#concurrency)
for more details.
[//]: # (Please update EE::GitLab::GeoGitAccess::GEO_SERVER_DOCS_URL if this file is moved)
# Using a Geo Server
After you set up the [database replication and configure the GitLab Geo nodes][req],
......
# GitLab Pages documentation
With GitLab Pages you can create static websites for your GitLab projects,
groups, or user accounts. You can use any static website generator: Jekyll,
Middleman, Hexo, Hugo, Pelican, you name it! Connect as many customs domains
as you like and bring your own TLS certificate to secure them.
Here's some info we've gathered to get you started.
## General info
- [Product webpage](https://pages.gitlab.io)
- ["We're bringing GitLab Pages to CE"](https://about.gitlab.com/2016/12/24/were-bringing-gitlab-pages-to-community-edition/)
- [Pages group - templates](https://gitlab.com/pages)
- [General user documentation](introduction.md)
- [Admin documentation - Set GitLab Pages on your own GitLab instance](../../../administration/pages/index.md)
- ["We are changing the IP of GitLab Pages on GitLab.com"](https://about.gitlab.com/2017/03/06/we-are-changing-the-ip-of-gitlab-pages-on-gitlab-com/)
## Getting started
- **GitLab Pages from A to Z**
- [Part 1: Static sites and GitLab Pages domains](getting_started_part_one.md)
- [Part 2: Quick start guide - Setting up GitLab Pages](getting_started_part_two.md)
- [Part 3: Setting Up Custom Domains - DNS Records and SSL/TLS Certificates](getting_started_part_three.md)
- [Part 4: Creating and tweaking `.gitlab-ci.yml` for GitLab Pages](getting_started_part_four.md)
- **Static Site Generators - Blog posts series**
- [SSGs part 1: Static vs dynamic websites](https://about.gitlab.com/2016/06/03/ssg-overview-gitlab-pages-part-1-dynamic-x-static/)
- [SSGs part 2: Modern static site generators](https://about.gitlab.com/2016/06/10/ssg-overview-gitlab-pages-part-2/)
- [SSGs part 3: Build any SSG site with GitLab Pages](https://about.gitlab.com/2016/06/17/ssg-overview-gitlab-pages-part-3-examples-ci/)
- **Secure GitLab Pages custom domain with SSL/TLS certificates**
- [Let's Encrypt](https://about.gitlab.com/2016/04/11/tutorial-securing-your-gitlab-pages-with-tls-and-letsencrypt/)
- [CloudFlare](https://about.gitlab.com/2017/02/07/setting-up-gitlab-pages-with-cloudflare-certificates/)
- [StartSSL](https://about.gitlab.com/2016/06/24/secure-gitlab-pages-with-startssl/)
- **General**
- [Hosting on GitLab.com with GitLab Pages](https://about.gitlab.com/2016/04/07/gitlab-pages-setup/) a comprehensive step-by-step guide
- [Posting to your GitLab Pages blog from iOS](https://about.gitlab.com/2016/08/19/posting-to-your-gitlab-pages-blog-from-ios/)
## Video tutorials
- [How to publish a website with GitLab Pages on GitLab.com: from a forked project](https://youtu.be/TWqh9MtT4Bg)
- [How to Enable GitLab Pages for GitLab CE and EE (for Admins only)](https://youtu.be/dD8c7WNcc6s)
# GitLab Pages
With GitLab Pages you can host your website at no cost.
Your files live in a GitLab project's [repository](../repository/index.md),
from which you can deploy [static websites](#explore-gitlab-pages).
GitLab Pages supports all static site generators (SSGs).
## Getting Started
Follow the steps below to get your website live. They shouldn't take more than
5 minutes to complete:
- 1. [Fork](../../../gitlab-basics/fork-project.md#how-to-fork-a-project) an [example project](https://gitlab.com/pages)
- 2. Change a file to trigger a GitLab CI/CD pipeline
- 3. Visit your project's **Settings > Pages** to see your **website link**, and click on it. Bam! Your website is live.
_Further steps (optional):_
- 4. Remove the [fork relationship](getting_started_part_two.md#fork-a-project-to-get-started-from) (_You don't need the relationship unless you intent to contribute back to the example project you forked from_).
- 5. Make it a [user/group website](getting_started_part_one.md#user-and-group-websites)
**Watch a video with the steps above: https://www.youtube.com/watch?v=TWqh9MtT4Bg**
_Advanced options:_
- [Use a custom domain](getting_started_part_three.md#adding-your-custom-domain-to-gitlab-pages)
- Apply [SSL/TLS certification](getting_started_part_three.md#ssl-tls-certificates) to your custom domain
## Explore GitLab Pages
With GitLab Pages you can create [static websites](getting_started_part_one.md#what-you-need-to-know-before-getting-started)
for your GitLab projects, groups, or user accounts. You can use any static
website generator: Jekyll, Middleman, Hexo, Hugo, Pelican, you name it!
Connect as many custom domains as you like and bring your own TLS certificate
to secure them.
Read the following tutorials to know more about:
- [Static websites and GitLab Pages domains](getting_started_part_one.md)
- [Forking projects and creating new ones from scratch, URLs and baseurls](getting_started_part_two.md)
- [Custom domains and subdomains, DNS records, SSL/TLS certificates](getting_started_part_three.md)
- [How to create your own `.gitlab-ci.yml` for your site](getting_started_part_four.md)
- [Technical aspects, custom 404 pages, limitations](introduction.md)
- [Hosting on GitLab.com with GitLab Pages](https://about.gitlab.com/2016/04/07/gitlab-pages-setup/) (outdated)
_Blog posts series about Static Site Generators (SSGs):_
- [SSGs part 1: Static vs dynamic websites](https://about.gitlab.com/2016/06/03/ssg-overview-gitlab-pages-part-1-dynamic-x-static/)
- [SSGs part 2: Modern static site generators](https://about.gitlab.com/2016/06/10/ssg-overview-gitlab-pages-part-2/)
- [SSGs part 3: Build any SSG site with GitLab Pages](https://about.gitlab.com/2016/06/17/ssg-overview-gitlab-pages-part-3-examples-ci/)
_Blog posts for securing GitLab Pages custom domains with SSL/TLS certificates:_
- [CloudFlare](https://about.gitlab.com/2017/02/07/setting-up-gitlab-pages-with-cloudflare-certificates/)
- [Let's Encrypt](https://about.gitlab.com/2016/04/11/tutorial-securing-your-gitlab-pages-with-tls-and-letsencrypt/) (outdated)
- [StartSSL](https://about.gitlab.com/2016/06/24/secure-gitlab-pages-with-startssl/) (deprecated)
## Advanced use
- **Blog Posts**
- [GitLab CI: Run jobs sequentially, in parallel, or build a custom pipeline](https://about.gitlab.com/2016/07/29/the-basics-of-gitlab-ci/)
- [GitLab CI: Deployment & environments](https://about.gitlab.com/2016/08/26/ci-deployment-and-environments/)
- [Building a new GitLab docs site with Nanoc, GitLab CI, and GitLab Pages](https://about.gitlab.com/2016/12/07/building-a-new-gitlab-docs-site-with-nanoc-gitlab-ci-and-gitlab-pages/)
- [Publish code coverage reports with GitLab Pages](https://about.gitlab.com/2016/11/03/publish-code-coverage-report-with-gitlab-pages/)
- [Posting to your GitLab Pages blog from iOS](https://about.gitlab.com/2016/08/19/posting-to-your-gitlab-pages-blog-from-ios/)
- [GitLab CI: Run jobs sequentially, in parallel, or build a custom pipeline](https://about.gitlab.com/2016/07/29/the-basics-of-gitlab-ci/)
- [GitLab CI: Deployment & environments](https://about.gitlab.com/2016/08/26/ci-deployment-and-environments/)
- [Building a new GitLab docs site with Nanoc, GitLab CI, and GitLab Pages](https://about.gitlab.com/2016/12/07/building-a-new-gitlab-docs-site-with-nanoc-gitlab-ci-and-gitlab-pages/)
- [Publish code coverage reports with GitLab Pages](https://about.gitlab.com/2016/11/03/publish-code-coverage-report-with-gitlab-pages/)
## Admin GitLab Pages for CE and EE
Enable and configure GitLab Pages on your own instance (GitLab Community Edition and Enterprise Editions) with
the [admin guide](../../../administration/pages/index.md).
**Watch the video: https://www.youtube.com/watch?v=dD8c7WNcc6s**
## More information about GitLab Pages
- For an overview, visit the [feature webpage](https://about.gitlab.com/features/pages/)
- Announcement (2016-12-24): ["We're bringing GitLab Pages to CE"](https://about.gitlab.com/2016/12/24/were-bringing-gitlab-pages-to-community-edition/)
- Announcement (2017-03-06): ["We are changing the IP of GitLab Pages on GitLab.com"](https://about.gitlab.com/2017/03/06/we-are-changing-the-ip-of-gitlab-pages-on-gitlab-com/)
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment