Add latest changes from gitlab-org/gitlab@master
This commit is contained in:
GitLab Bot
parent
4535b534a8
commit
5169d58a9d
|
|
@ -349,14 +349,6 @@ export default {
|
|||
refetchDiffData() {
|
||||
this.fetchData(false);
|
||||
},
|
||||
startDiffRendering() {
|
||||
requestIdleCallback(
|
||||
() => {
|
||||
this.startRenderDiffsQueue();
|
||||
},
|
||||
{ timeout: 1000 },
|
||||
);
|
||||
},
|
||||
needsReload() {
|
||||
return this.diffFiles.length && isSingleViewStyle(this.diffFiles[0]);
|
||||
},
|
||||
|
|
@ -368,8 +360,6 @@ export default {
|
|||
.then(({ real_size }) => {
|
||||
this.diffFilesLength = parseInt(real_size, 10);
|
||||
if (toggleTree) this.setTreeDisplay();
|
||||
|
||||
this.startDiffRendering();
|
||||
})
|
||||
.catch(() => {
|
||||
createFlash(__('Something went wrong on our end. Please try again!'));
|
||||
|
|
@ -384,7 +374,6 @@ export default {
|
|||
// change when loading the other half of the diff files.
|
||||
this.setDiscussions();
|
||||
})
|
||||
.then(() => this.startDiffRendering())
|
||||
.catch(() => {
|
||||
createFlash(__('Something went wrong on our end. Please try again!'));
|
||||
});
|
||||
|
|
|
|||
|
|
@ -49,7 +49,6 @@ import {
|
|||
convertExpandLines,
|
||||
idleCallback,
|
||||
allDiscussionWrappersExpanded,
|
||||
prepareDiffData,
|
||||
prepareLineForRenamedFile,
|
||||
} from './utils';
|
||||
|
||||
|
|
@ -163,7 +162,15 @@ export const fetchDiffFilesBatch = ({ commit, state, dispatch }) => {
|
|||
|
||||
return pagination.next_page;
|
||||
})
|
||||
.then((nextPage) => nextPage && getBatch(nextPage))
|
||||
.then((nextPage) => {
|
||||
dispatch('startRenderDiffsQueue');
|
||||
|
||||
if (nextPage) {
|
||||
return getBatch(nextPage);
|
||||
}
|
||||
|
||||
return null;
|
||||
})
|
||||
.catch(() => commit(types.SET_RETRIEVING_BATCHES, false));
|
||||
|
||||
return getBatch()
|
||||
|
|
@ -197,13 +204,7 @@ export const fetchDiffFilesMeta = ({ commit, state }) => {
|
|||
commit(types.SET_MERGE_REQUEST_DIFFS, data.merge_request_diffs || []);
|
||||
commit(types.SET_DIFF_METADATA, strippedData);
|
||||
|
||||
worker.postMessage(
|
||||
prepareDiffData({
|
||||
diff: data,
|
||||
priorFiles: state.diffFiles,
|
||||
meta: true,
|
||||
}),
|
||||
);
|
||||
worker.postMessage(data.diff_files);
|
||||
|
||||
return data;
|
||||
})
|
||||
|
|
@ -304,33 +305,38 @@ export const renderFileForDiscussionId = ({ commit, rootState, state }, discussi
|
|||
};
|
||||
|
||||
export const startRenderDiffsQueue = ({ state, commit }) => {
|
||||
const checkItem = () =>
|
||||
new Promise((resolve) => {
|
||||
const nextFile = state.diffFiles.find(
|
||||
(file) =>
|
||||
!file.renderIt &&
|
||||
file.viewer &&
|
||||
(!isCollapsed(file) || file.viewer.name !== diffViewerModes.text),
|
||||
);
|
||||
const diffFilesToRender = state.diffFiles.filter(
|
||||
(file) =>
|
||||
!file.renderIt &&
|
||||
file.viewer &&
|
||||
(!isCollapsed(file) || file.viewer.name !== diffViewerModes.text),
|
||||
);
|
||||
let currentDiffFileIndex = 0;
|
||||
|
||||
if (nextFile) {
|
||||
requestAnimationFrame(() => {
|
||||
commit(types.RENDER_FILE, nextFile);
|
||||
const checkItem = () => {
|
||||
const nextFile = diffFilesToRender[currentDiffFileIndex];
|
||||
|
||||
if (nextFile) {
|
||||
currentDiffFileIndex += 1;
|
||||
commit(types.RENDER_FILE, nextFile);
|
||||
|
||||
const requestIdle = () =>
|
||||
requestIdleCallback((idleDeadline) => {
|
||||
// Wait for at least 5ms before trying to render
|
||||
if (idleDeadline.timeRemaining() >= 6) {
|
||||
checkItem();
|
||||
} else {
|
||||
requestIdle();
|
||||
}
|
||||
});
|
||||
requestIdleCallback(
|
||||
() => {
|
||||
checkItem()
|
||||
.then(resolve)
|
||||
.catch(() => {});
|
||||
},
|
||||
{ timeout: 1000 },
|
||||
);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
|
||||
return checkItem();
|
||||
requestIdle();
|
||||
}
|
||||
};
|
||||
|
||||
if (diffFilesToRender.length) {
|
||||
checkItem();
|
||||
}
|
||||
};
|
||||
|
||||
export const setRenderIt = ({ commit }, file) => commit(types.RENDER_FILE, file);
|
||||
|
|
|
|||
|
|
@ -381,22 +381,13 @@ function prepareDiffFileLines(file) {
|
|||
|
||||
inlineLines.forEach((line) => prepareLine(line, file)); // WARNING: In-Place Mutations!
|
||||
|
||||
Object.assign(file, {
|
||||
inlineLinesCount: inlineLines.length,
|
||||
});
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
function getVisibleDiffLines(file) {
|
||||
return file.inlineLinesCount;
|
||||
}
|
||||
|
||||
function finalizeDiffFile(file) {
|
||||
const lines = getVisibleDiffLines(file);
|
||||
|
||||
function finalizeDiffFile(file, index) {
|
||||
Object.assign(file, {
|
||||
renderIt: lines < LINES_TO_BE_RENDERED_DIRECTLY,
|
||||
renderIt:
|
||||
index < 3 ? file[INLINE_DIFF_LINES_KEY].length < LINES_TO_BE_RENDERED_DIRECTLY : false,
|
||||
isShowingFullFile: false,
|
||||
isLoadingFullFile: false,
|
||||
discussions: [],
|
||||
|
|
@ -424,7 +415,7 @@ export function prepareDiffData({ diff, priorFiles = [], meta = false }) {
|
|||
.map((file, index, allFiles) => prepareRawDiffFile({ file, allFiles, meta }))
|
||||
.map(ensureBasicDiffFileLines)
|
||||
.map(prepareDiffFileLines)
|
||||
.map(finalizeDiffFile);
|
||||
.map((file, index) => finalizeDiffFile(file, priorFiles.length + index));
|
||||
|
||||
return deduplicateFilesList([...priorFiles, ...cleanedFiles]);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,43 +1,24 @@
|
|||
<script>
|
||||
import { s__, __, sprintf } from '~/locale';
|
||||
import ClipboardButton from '~/vue_shared/components/clipboard_button.vue';
|
||||
import CopyableField from '../../vue_shared/components/sidebar/copyable_field.vue';
|
||||
|
||||
export default {
|
||||
i18n: {
|
||||
copyEmail: __('Copy email address'),
|
||||
},
|
||||
components: {
|
||||
ClipboardButton,
|
||||
CopyableField,
|
||||
},
|
||||
props: {
|
||||
copyText: {
|
||||
issueEmailAddress: {
|
||||
type: String,
|
||||
required: true,
|
||||
},
|
||||
},
|
||||
computed: {
|
||||
emailText() {
|
||||
return sprintf(s__('RightSidebar|Issue email: %{copyText}'), { copyText: this.copyText });
|
||||
},
|
||||
},
|
||||
};
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div
|
||||
<copyable-field
|
||||
data-qa-selector="copy-forward-email"
|
||||
class="copy-email-address gl-display-flex gl-align-items-center gl-justify-content-space-between"
|
||||
>
|
||||
<span
|
||||
class="gl-overflow-hidden gl-text-overflow-ellipsis gl-white-space-nowrap hide-collapsed gl-w-85p"
|
||||
>{{ emailText }}</span
|
||||
>
|
||||
<clipboard-button
|
||||
class="copy-email-button gl-bg-none!"
|
||||
category="tertiary"
|
||||
:title="$options.i18n.copyEmail"
|
||||
:text="copyText"
|
||||
tooltip-placement="left"
|
||||
/>
|
||||
</div>
|
||||
:name="s__('RightSidebar|Issue email')"
|
||||
:clipboard-tooltip-text="s__('RightSidebar|Copy email address')"
|
||||
:value="issueEmailAddress"
|
||||
/>
|
||||
</template>
|
||||
|
|
|
|||
|
|
@ -337,7 +337,7 @@ function mountCopyEmailComponent() {
|
|||
new Vue({
|
||||
el,
|
||||
render: (createElement) =>
|
||||
createElement(CopyEmailToClipboard, { props: { copyText: createNoteEmail } }),
|
||||
createElement(CopyEmailToClipboard, { props: { issueEmailAddress: createNoteEmail } }),
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,11 @@ export default {
|
|||
required: false,
|
||||
default: false,
|
||||
},
|
||||
clipboardTooltipText: {
|
||||
type: String,
|
||||
required: false,
|
||||
default: undefined,
|
||||
},
|
||||
},
|
||||
computed: {
|
||||
clipboardProps() {
|
||||
|
|
@ -35,7 +40,9 @@ export default {
|
|||
tooltipBoundary: 'viewport',
|
||||
tooltipPlacement: 'left',
|
||||
text: this.value,
|
||||
title: sprintf(this.$options.i18n.clipboardTooltip, { name: this.name }),
|
||||
title:
|
||||
this.clipboardTooltipText ||
|
||||
sprintf(this.$options.i18n.clipboardTooltip, { name: this.name }),
|
||||
};
|
||||
},
|
||||
loadingIconLabel() {
|
||||
|
|
|
|||
|
|
@ -58,19 +58,6 @@
|
|||
height: $gl-padding;
|
||||
}
|
||||
}
|
||||
|
||||
.copy-email-button { // TODO: replace with utility
|
||||
@include gl-w-full;
|
||||
@include gl-h-full;
|
||||
}
|
||||
|
||||
.copy-email-address {
|
||||
height: 60px;
|
||||
|
||||
&:hover {
|
||||
background: $gray-100;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.right-sidebar-expanded {
|
||||
|
|
|
|||
|
|
@ -359,7 +359,7 @@ class MergeRequest < ApplicationRecord
|
|||
scope :preload_metrics, -> (relation) { preload(metrics: relation) }
|
||||
scope :preload_project_and_latest_diff, -> { preload(:source_project, :latest_merge_request_diff) }
|
||||
scope :preload_latest_diff_commit, -> { preload(latest_merge_request_diff: :merge_request_diff_commits) }
|
||||
scope :with_web_entity_associations, -> { preload(:author, :target_project) }
|
||||
scope :with_web_entity_associations, -> { preload(:author, target_project: [:project_feature, group: [:route, :parent], namespace: :route]) }
|
||||
|
||||
scope :with_auto_merge_enabled, -> do
|
||||
with_state(:opened).where(auto_merge_enabled: true)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
title: Preload additional data to fix N+1 queries for merge request search
|
||||
merge_request: 57284
|
||||
author:
|
||||
type: performance
|
||||
|
|
@ -54,7 +54,7 @@ Geo provides:
|
|||
### Gitaly Cluster
|
||||
|
||||
Geo should not be confused with [Gitaly Cluster](../gitaly/praefect.md). For more information about
|
||||
the difference between Geo and Gitaly Cluster, see [Gitaly Cluster compared to Geo](../gitaly/praefect.md#gitaly-cluster-compared-to-geo).
|
||||
the difference between Geo and Gitaly Cluster, see [Gitaly Cluster compared to Geo](../gitaly/index.md#gitaly-cluster-compared-to-geo).
|
||||
|
||||
## How it works
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
|
|||
type: reference
|
||||
---
|
||||
|
||||
# Configure Gitaly
|
||||
# Configure Gitaly **(FREE SELF)**
|
||||
|
||||
The Gitaly service itself is configured by using a [TOML configuration file](reference.md).
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
|
|||
type: reference
|
||||
---
|
||||
|
||||
# Gitaly
|
||||
# Gitaly and Gitaly Cluster **(FREE SELF)**
|
||||
|
||||
[Gitaly](https://gitlab.com/gitlab-org/gitaly) provides high-level RPC access to Git repositories.
|
||||
It is used by GitLab to read and write Git data.
|
||||
|
|
@ -68,8 +68,202 @@ GitLab installations for more than 2000 users should use Gitaly Cluster.
|
|||
|
||||
## Gitaly Cluster
|
||||
|
||||
Gitaly can run in a clustered configuration to scale Gitaly and increase fault tolerance. For more
|
||||
information, see [Gitaly Cluster](praefect.md).
|
||||
Gitaly, the service that provides storage for Git repositories, can
|
||||
be run in a clustered configuration to scale the Gitaly service and increase
|
||||
fault tolerance. In this configuration, every Git repository is stored on every
|
||||
Gitaly node in the cluster.
|
||||
|
||||
Using a Gitaly Cluster increases fault tolerance by:
|
||||
|
||||
- Replicating write operations to warm standby Gitaly nodes.
|
||||
- Detecting Gitaly node failures.
|
||||
- Automatically routing Git requests to an available Gitaly node.
|
||||
|
||||
NOTE:
|
||||
Technical support for Gitaly clusters is limited to GitLab Premium and Ultimate
|
||||
customers.
|
||||
|
||||
The availability objectives for Gitaly clusters are:
|
||||
|
||||
- **Recovery Point Objective (RPO):** Less than 1 minute.
|
||||
|
||||
Writes are replicated asynchronously. Any writes that have not been replicated
|
||||
to the newly promoted primary are lost.
|
||||
|
||||
[Strong consistency](praefect.md#strong-consistency) can be used to avoid loss in some
|
||||
circumstances.
|
||||
|
||||
- **Recovery Time Objective (RTO):** Less than 10 seconds.
|
||||
Outages are detected by a health check run by each Praefect node every
|
||||
second. Failover requires ten consecutive failed health checks on each
|
||||
Praefect node.
|
||||
|
||||
[Faster outage detection](https://gitlab.com/gitlab-org/gitaly/-/issues/2608)
|
||||
is planned to improve this to less than 1 second.
|
||||
|
||||
Gitaly Cluster supports:
|
||||
|
||||
- [Strong consistency](praefect.md#strong-consistency) of the secondary replicas.
|
||||
- [Automatic failover](praefect.md#automatic-failover-and-leader-election) from the primary to the secondary.
|
||||
- Reporting of possible data loss if replication queue is non-empty.
|
||||
- Marking repositories as [read only](praefect.md#read-only-mode) if data loss is detected to prevent data inconsistencies.
|
||||
|
||||
Follow the [Gitaly Cluster epic](https://gitlab.com/groups/gitlab-org/-/epics/1489)
|
||||
for improvements including
|
||||
[horizontally distributing reads](https://gitlab.com/groups/gitlab-org/-/epics/2013).
|
||||
|
||||
### Overview
|
||||
|
||||
Git storage is provided through the Gitaly service in GitLab, and is essential
|
||||
to the operation of the GitLab application. When the number of
|
||||
users, repositories, and activity grows, it is important to scale Gitaly
|
||||
appropriately by:
|
||||
|
||||
- Increasing the available CPU and memory resources available to Git before
|
||||
resource exhaustion degrades Git, Gitaly, and GitLab application performance.
|
||||
- Increase available storage before storage limits are reached causing write
|
||||
operations to fail.
|
||||
- Improve fault tolerance by removing single points of failure. Git should be
|
||||
considered mission critical if a service degradation would prevent you from
|
||||
deploying changes to production.
|
||||
|
||||
### Moving beyond NFS
|
||||
|
||||
WARNING:
|
||||
From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab 14.0,
|
||||
support for NFS for Git repositories is scheduled to be removed. Upgrade to
|
||||
Gitaly Cluster as soon as possible.
|
||||
|
||||
[Network File System (NFS)](https://en.wikipedia.org/wiki/Network_File_System)
|
||||
is not well suited to Git workloads which are CPU and IOPS sensitive.
|
||||
Specifically:
|
||||
|
||||
- Git is sensitive to file system latency. Even simple operations require many
|
||||
read operations. Operations that are fast on block storage can become an order of
|
||||
magnitude slower. This significantly impacts GitLab application performance.
|
||||
- NFS performance optimizations that prevent the performance gap between
|
||||
block storage and NFS being even wider are vulnerable to race conditions. We have observed
|
||||
[data inconsistencies](https://gitlab.com/gitlab-org/gitaly/-/issues/2589)
|
||||
in production environments caused by simultaneous writes to different NFS
|
||||
clients. Data corruption is not an acceptable risk.
|
||||
|
||||
Gitaly Cluster is purpose built to provide reliable, high performance, fault
|
||||
tolerant Git storage.
|
||||
|
||||
Further reading:
|
||||
|
||||
- Blog post: [The road to Gitaly v1.0 (aka, why GitLab doesn't require NFS for storing Git data anymore)](https://about.gitlab.com/blog/2018/09/12/the-road-to-gitaly-1-0/)
|
||||
- Blog post: [How we spent two weeks hunting an NFS bug in the Linux kernel](https://about.gitlab.com/blog/2018/11/14/how-we-spent-two-weeks-hunting-an-nfs-bug/)
|
||||
|
||||
### Where Gitaly Cluster fits
|
||||
|
||||
GitLab accesses [repositories](../../user/project/repository/index.md) through the configured
|
||||
[repository storages](../repository_storage_paths.md). Each new repository is stored on one of the
|
||||
repository storages based on their configured weights. Each repository storage is either:
|
||||
|
||||
- A Gitaly storage served directly by Gitaly. These map to a directory on the file system of a
|
||||
Gitaly node.
|
||||
- A [virtual storage](#virtual-storage-or-direct-gitaly-storage) served by Praefect. A virtual
|
||||
storage is a cluster of Gitaly storages that appear as a single repository storage.
|
||||
|
||||
Virtual storages are a feature of Gitaly Cluster. They support replicating the repositories to
|
||||
multiple storages for fault tolerance. Virtual storages can improve performance by distributing
|
||||
requests across Gitaly nodes. Their distributed nature makes it viable to have a single repository
|
||||
storage in GitLab to simplify repository management.
|
||||
|
||||
### Components of Gitaly Cluster
|
||||
|
||||
Gitaly Cluster consists of multiple components:
|
||||
|
||||
- [Load balancer](praefect.md#load-balancer) for distributing requests and providing fault-tolerant access to
|
||||
Praefect nodes.
|
||||
- [Praefect](praefect.md#praefect) nodes for managing the cluster and routing requests to Gitaly nodes.
|
||||
- [PostgreSQL database](praefect.md#postgresql) for persisting cluster metadata and [PgBouncer](praefect.md#pgbouncer),
|
||||
recommended for pooling Praefect's database connections.
|
||||
- Gitaly nodes to provide repository storage and Git access.
|
||||
|
||||
|
||||
|
||||
In this example:
|
||||
|
||||
- Repositories are stored on a virtual storage called `storage-1`.
|
||||
- Three Gitaly nodes provide `storage-1` access: `gitaly-1`, `gitaly-2`, and `gitaly-3`.
|
||||
- The three Gitaly nodes store data on their file systems.
|
||||
|
||||
### Virtual storage or direct Gitaly storage
|
||||
|
||||
Gitaly supports multiple models of scaling:
|
||||
|
||||
- Clustering using Gitaly Cluster, where each repository is stored on multiple Gitaly nodes in the
|
||||
cluster. Read requests are distributed between repository replicas and write requests are
|
||||
broadcast to repository replicas. GitLab accesses virtual storage.
|
||||
- Direct access to Gitaly storage using [repository storage paths](../repository_storage_paths.md),
|
||||
where each repository is stored on the assigned Gitaly node. All requests are routed to this node.
|
||||
|
||||
The following is Gitaly set up to use direct access to Gitaly instead of Gitaly Cluster:
|
||||
|
||||
|
||||
|
||||
In this example:
|
||||
|
||||
- Each repository is stored on one of three Gitaly storages: `storage-1`, `storage-2`,
|
||||
or `storage-3`.
|
||||
- Each storage is serviced by a Gitaly node.
|
||||
- The three Gitaly nodes store data in three separate hashed storage locations.
|
||||
|
||||
Generally, virtual storage with Gitaly Cluster can replace direct Gitaly storage configurations, at
|
||||
the expense of additional storage needed to store each repository on multiple Gitaly nodes. The
|
||||
benefit of using Gitaly Cluster over direct Gitaly storage is:
|
||||
|
||||
- Improved fault tolerance, because each Gitaly node has a copy of every repository.
|
||||
- Improved resource utilization, reducing the need for over-provisioning for shard-specific peak
|
||||
loads, because read loads are distributed across replicas.
|
||||
- Manual rebalancing for performance is not required, because read loads are distributed across
|
||||
replicas.
|
||||
- Simpler management, because all Gitaly nodes are identical.
|
||||
|
||||
Under some workloads, CPU and memory requirements may require a large fleet of Gitaly nodes. It
|
||||
can be uneconomical to have one to one replication factor.
|
||||
|
||||
A hybrid approach can be used in these instances, where each shard is configured as a smaller
|
||||
cluster. [Variable replication factor](https://gitlab.com/groups/gitlab-org/-/epics/3372) is planned
|
||||
to provide greater flexibility for extremely large GitLab instances.
|
||||
|
||||
### Gitaly Cluster compared to Geo
|
||||
|
||||
Gitaly Cluster and [Geo](../geo/index.md) both provide redundancy. However the redundancy of:
|
||||
|
||||
- Gitaly Cluster provides fault tolerance for data storage and is invisible to the user. Users are
|
||||
not aware when Gitaly Cluster is used.
|
||||
- Geo provides [replication](../geo/index.md) and [disaster recovery](../geo/disaster_recovery/index.md) for
|
||||
an entire instance of GitLab. Users know when they are using Geo for
|
||||
[replication](../geo/index.md). Geo [replicates multiple data types](../geo/replication/datatypes.md#limitations-on-replicationverification),
|
||||
including Git data.
|
||||
|
||||
The following table outlines the major differences between Gitaly Cluster and Geo:
|
||||
|
||||
| Tool | Nodes | Locations | Latency tolerance | Failover | Consistency | Provides redundancy for |
|
||||
|:---------------|:---------|:----------|:-------------------|:----------------------------------------------------------------|:-----------------------------------------|:------------------------|
|
||||
| Gitaly Cluster | Multiple | Single | Approximately 1 ms | [Automatic](praefect.md#automatic-failover-and-leader-election) | [Strong](praefect.md#strong-consistency) | Data storage in Git |
|
||||
| Geo | Multiple | Multiple | Up to one minute | [Manual](../geo/disaster_recovery/index.md) | Eventual | Entire GitLab instance |
|
||||
|
||||
For more information, see:
|
||||
|
||||
- Geo [use cases](../geo/index.md#use-cases).
|
||||
- Geo [architecture](../geo/index.md#architecture).
|
||||
|
||||
### Architecture
|
||||
|
||||
Praefect is a router and transaction manager for Gitaly, and a required
|
||||
component for running a Gitaly Cluster.
|
||||
|
||||
|
||||
|
||||
For more information, see [Gitaly High Availability (HA) Design](https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/design_ha.md).
|
||||
|
||||
### Configure Gitaly Cluster
|
||||
|
||||
For more information on configuring Gitaly Cluster, see [Configure Gitaly Cluster](praefect.md).
|
||||
|
||||
## Do not bypass Gitaly
|
||||
|
||||
|
|
@ -180,7 +374,7 @@ There are two facets to our efforts to remove direct Git access in GitLab:
|
|||
NFS.
|
||||
|
||||
The second facet presents the only real solution. For this, we developed
|
||||
[Gitaly Cluster](praefect.md).
|
||||
[Gitaly Cluster](#gitaly-cluster).
|
||||
|
||||
## NFS deprecation notice
|
||||
|
||||
|
|
@ -198,7 +392,7 @@ Additional information:
|
|||
|
||||
GitLab recommends:
|
||||
|
||||
- Creating a [Gitaly Cluster](praefect.md) as soon as possible.
|
||||
- Creating a [Gitaly Cluster](#gitaly-cluster) as soon as possible.
|
||||
- [Moving your projects](praefect.md#migrate-existing-repositories-to-gitaly-cluster) from NFS-based
|
||||
storage to the Gitaly Cluster.
|
||||
|
||||
|
|
|
|||
|
|
@ -5,201 +5,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w
|
|||
type: reference
|
||||
---
|
||||
|
||||
# Gitaly Cluster **(FREE SELF)**
|
||||
# Configure Gitaly Cluster **(FREE SELF)**
|
||||
|
||||
[Gitaly](index.md), the service that provides storage for Git repositories, can
|
||||
be run in a clustered configuration to scale the Gitaly service and increase
|
||||
fault tolerance. In this configuration, every Git repository is stored on every
|
||||
Gitaly node in the cluster.
|
||||
|
||||
Using a Gitaly Cluster increases fault tolerance by:
|
||||
|
||||
- Replicating write operations to warm standby Gitaly nodes.
|
||||
- Detecting Gitaly node failures.
|
||||
- Automatically routing Git requests to an available Gitaly node.
|
||||
|
||||
NOTE:
|
||||
Technical support for Gitaly clusters is limited to GitLab Premium and Ultimate
|
||||
customers.
|
||||
|
||||
The availability objectives for Gitaly clusters are:
|
||||
|
||||
- **Recovery Point Objective (RPO):** Less than 1 minute.
|
||||
|
||||
Writes are replicated asynchronously. Any writes that have not been replicated
|
||||
to the newly promoted primary are lost.
|
||||
|
||||
[Strong consistency](#strong-consistency) can be used to avoid loss in some
|
||||
circumstances.
|
||||
|
||||
- **Recovery Time Objective (RTO):** Less than 10 seconds.
|
||||
|
||||
Outages are detected by a health checks run by each Praefect node every
|
||||
second. Failover requires ten consecutive failed health checks on each
|
||||
Praefect node.
|
||||
|
||||
[Faster outage detection](https://gitlab.com/gitlab-org/gitaly/-/issues/2608)
|
||||
is planned to improve this to less than 1 second.
|
||||
|
||||
Gitaly Cluster supports:
|
||||
|
||||
- [Strong consistency](#strong-consistency) of the secondary replicas.
|
||||
- [Automatic failover](#automatic-failover-and-leader-election) from the primary to the secondary.
|
||||
- Reporting of possible data loss if replication queue is non-empty.
|
||||
- Marking repositories as [read only](#read-only-mode) if data loss is detected to prevent data inconsistencies.
|
||||
|
||||
Follow the [Gitaly Cluster epic](https://gitlab.com/groups/gitlab-org/-/epics/1489)
|
||||
for improvements including
|
||||
[horizontally distributing reads](https://gitlab.com/groups/gitlab-org/-/epics/2013).
|
||||
|
||||
## Overview
|
||||
|
||||
Git storage is provided through the Gitaly service in GitLab, and is essential
|
||||
to correct proper operation of the GitLab application. When the number of
|
||||
users, repositories, and activity grows, it is important to scale Gitaly
|
||||
appropriately by:
|
||||
|
||||
- Increasing the available CPU and memory resources available to Git before
|
||||
resource exhaustion degrades Git, Gitaly, and GitLab application performance.
|
||||
- Increase available storage before storage limits are reached causing write
|
||||
operations to fail.
|
||||
- Improve fault tolerance by removing single points of failure. Git should be
|
||||
considered mission critical if a service degradation would prevent you from
|
||||
deploying changes to production.
|
||||
|
||||
### Moving beyond NFS
|
||||
|
||||
WARNING:
|
||||
From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab 14.0,
|
||||
support for NFS for Git repositories is scheduled to be removed. Upgrade to
|
||||
Gitaly Cluster as soon as possible.
|
||||
|
||||
[Network File System (NFS)](https://en.wikipedia.org/wiki/Network_File_System)
|
||||
is not well suited to Git workloads which are CPU and IOPS sensitive.
|
||||
Specifically:
|
||||
|
||||
- Git is sensitive to file system latency. Even simple operations require many
|
||||
read operations. Operations that are fast on block storage can become an order of
|
||||
magnitude slower. This significantly impacts GitLab application performance.
|
||||
- NFS performance optimizations that prevent the performance gap between
|
||||
block storage and NFS being even wider are vulnerable to race conditions. We have observed
|
||||
[data inconsistencies](https://gitlab.com/gitlab-org/gitaly/-/issues/2589)
|
||||
in production environments caused by simultaneous writes to different NFS
|
||||
clients. Data corruption is not an acceptable risk.
|
||||
|
||||
Gitaly Cluster is purpose built to provide reliable, high performance, fault
|
||||
tolerant Git storage.
|
||||
|
||||
Further reading:
|
||||
|
||||
- Blog post: [The road to Gitaly v1.0 (aka, why GitLab doesn't require NFS for storing Git data anymore)](https://about.gitlab.com/blog/2018/09/12/the-road-to-gitaly-1-0/)
|
||||
- Blog post: [How we spent two weeks hunting an NFS bug in the Linux kernel](https://about.gitlab.com/blog/2018/11/14/how-we-spent-two-weeks-hunting-an-nfs-bug/)
|
||||
|
||||
## Where Gitaly Cluster fits
|
||||
|
||||
GitLab accesses [repositories](../../user/project/repository/index.md) through the configured
|
||||
[repository storages](../repository_storage_paths.md). Each new repository is stored on one of the
|
||||
repository storages based on their configured weights. Each repository storage is either:
|
||||
|
||||
- A Gitaly storage served directly by Gitaly. These map to a directory on the file system of a
|
||||
Gitaly node.
|
||||
- A [virtual storage](#virtual-storage-or-direct-gitaly-storage) served by Praefect. A virtual
|
||||
storage is a cluster of Gitaly storages that appear as a single repository storage.
|
||||
|
||||
Virtual storages are a feature of Gitaly Cluster. They support replicating the repositories to
|
||||
multiple storages for fault tolerance. Virtual storages can improve performance by distributing
|
||||
requests across Gitaly nodes. Their distributed nature makes it viable to have a single repository
|
||||
storage in GitLab to simplify repository management.
|
||||
|
||||
## Components of Gitaly Cluster
|
||||
|
||||
Gitaly Cluster consists of multiple components:
|
||||
|
||||
- [Load balancer](#load-balancer) for distributing requests and providing fault-tolerant access to
|
||||
Praefect nodes.
|
||||
- [Praefect](#praefect) nodes for managing the cluster and routing requests to Gitaly nodes.
|
||||
- [PostgreSQL database](#postgresql) for persisting cluster metadata and [PgBouncer](#pgbouncer),
|
||||
recommended for pooling Praefect's database connections.
|
||||
- [Gitaly](index.md) nodes to provide repository storage and Git access.
|
||||
|
||||
|
||||
|
||||
In this example:
|
||||
|
||||
- Repositories are stored on a virtual storage called `storage-1`.
|
||||
- Three Gitaly nodes provide `storage-1` access: `gitaly-1`, `gitaly-2`, and `gitaly-3`.
|
||||
- The three Gitaly nodes store data on their file systems.
|
||||
|
||||
### Virtual storage or direct Gitaly storage
|
||||
|
||||
Gitaly supports multiple models of scaling:
|
||||
|
||||
- Clustering using Gitaly Cluster, where each repository is stored on multiple Gitaly nodes in the
|
||||
cluster. Read requests are distributed between repository replicas and write requests are
|
||||
broadcast to repository replicas. GitLab accesses virtual storage.
|
||||
- Direct access to Gitaly storage using [repository storage paths](../repository_storage_paths.md),
|
||||
where each repository is stored on the assigned Gitaly node. All requests are routed to this node.
|
||||
|
||||
The following is Gitaly set up to use direct access to Gitaly instead of Gitaly Cluster:
|
||||
|
||||
|
||||
|
||||
In this example:
|
||||
|
||||
- Each repository is stored on one of three Gitaly storages: `storage-1`, `storage-2`,
|
||||
or `storage-3`.
|
||||
- Each storage is serviced by a Gitaly node.
|
||||
- The three Gitaly nodes store data in three separate hashed storage locations.
|
||||
|
||||
Generally, virtual storage with Gitaly Cluster can replace direct Gitaly storage configurations, at
|
||||
the expense of additional storage needed to store each repository on multiple Gitaly nodes. The
|
||||
benefit of using Gitaly Cluster over direct Gitaly storage is:
|
||||
|
||||
- Improved fault tolerance, because each Gitaly node has a copy of every repository.
|
||||
- Improved resource utilization, reducing the need for over-provisioning for shard-specific peak
|
||||
loads, because read loads are distributed across replicas.
|
||||
- Manual rebalancing for performance is not required, because read loads are distributed across
|
||||
replicas.
|
||||
- Simpler management, because all Gitaly nodes are identical.
|
||||
|
||||
Under some workloads, CPU and memory requirements may require a large fleet of Gitaly nodes. It
|
||||
can be uneconomical to have one to one replication factor.
|
||||
|
||||
A hybrid approach can be used in these instances, where each shard is configured as a smaller
|
||||
cluster. [Variable replication factor](https://gitlab.com/groups/gitlab-org/-/epics/3372) is planned
|
||||
to provide greater flexibility for extremely large GitLab instances.
|
||||
|
||||
### Gitaly Cluster compared to Geo
|
||||
|
||||
Gitaly Cluster and [Geo](../geo/index.md) both provide redundancy. However the redundancy of:
|
||||
|
||||
- Gitaly Cluster provides fault tolerance for data storage and is invisible to the user. Users are
|
||||
not aware when Gitaly Cluster is used.
|
||||
- Geo provides [replication](../geo/index.md) and [disaster recovery](../geo/disaster_recovery/index.md) for
|
||||
an entire instance of GitLab. Users know when they are using Geo for
|
||||
[replication](../geo/index.md). Geo [replicates multiple datatypes](../geo/replication/datatypes.md#limitations-on-replicationverification),
|
||||
including Git data.
|
||||
|
||||
The following table outlines the major differences between Gitaly Cluster and Geo:
|
||||
|
||||
| Tool | Nodes | Locations | Latency tolerance | Failover | Consistency | Provides redundancy for |
|
||||
|:---------------|:---------|:----------|:-------------------|:-----------------------------------------------------|:------------------------------|:------------------------|
|
||||
| Gitaly Cluster | Multiple | Single | Approximately 1 ms | [Automatic](#automatic-failover-and-leader-election) | [Strong](#strong-consistency) | Data storage in Git |
|
||||
| Geo | Multiple | Multiple | Up to one minute | [Manual](../geo/disaster_recovery/index.md) | Eventual | Entire GitLab instance |
|
||||
|
||||
For more information, see:
|
||||
|
||||
- [Gitaly](index.md).
|
||||
- Geo [use cases](../geo/index.md#use-cases) and [architecture](../geo/index.md#architecture).
|
||||
|
||||
## Architecture
|
||||
|
||||
Praefect is a router and transaction manager for Gitaly, and a required
|
||||
component for running a Gitaly Cluster.
|
||||
|
||||
|
||||
|
||||
For more information, see [Gitaly HA Design](https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/design_ha.md)
|
||||
In addition to Gitaly Cluster configuration instructions available as part of
|
||||
[reference architectures](../reference_architectures/index.md) for installations for more than
|
||||
2000 users, advanced configuration instructions are available below.
|
||||
|
||||
## Requirements for configuring a Gitaly Cluster
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
|
|||
type: reference
|
||||
---
|
||||
|
||||
# Gitaly reference
|
||||
# Gitaly reference **(FREE SELF)**
|
||||
|
||||
Gitaly is configured via a [TOML](https://github.com/toml-lang/toml)
|
||||
configuration file. Unlike installations from source, in Omnibus GitLab, you
|
||||
|
|
|
|||
|
|
@ -8582,9 +8582,6 @@ msgstr ""
|
|||
msgid "Copy commit SHA"
|
||||
msgstr ""
|
||||
|
||||
msgid "Copy email address"
|
||||
msgstr ""
|
||||
|
||||
msgid "Copy environment"
|
||||
msgstr ""
|
||||
|
||||
|
|
@ -20749,6 +20746,9 @@ msgstr ""
|
|||
msgid "No authentication methods configured."
|
||||
msgstr ""
|
||||
|
||||
msgid "No available branches"
|
||||
msgstr ""
|
||||
|
||||
msgid "No available groups to fork the project."
|
||||
msgstr ""
|
||||
|
||||
|
|
@ -21540,6 +21540,9 @@ msgstr ""
|
|||
msgid "OnDemandScans|You cannot run an active scan against an unvalidated site."
|
||||
msgstr ""
|
||||
|
||||
msgid "OnDemandScans|You must create a repository within your project to run an on-demand scan."
|
||||
msgstr ""
|
||||
|
||||
msgid "Once a project is permanently deleted, it %{strongStart}cannot be recovered%{strongEnd}. Permanently deleting this project will %{strongStart}immediately delete%{strongEnd} its repositories and %{strongStart}all related resources%{strongEnd}, including issues, merge requests etc."
|
||||
msgstr ""
|
||||
|
||||
|
|
@ -26365,7 +26368,10 @@ msgstr ""
|
|||
msgid "Revoked project access token %{project_access_token_name}!"
|
||||
msgstr ""
|
||||
|
||||
msgid "RightSidebar|Issue email: %{copyText}"
|
||||
msgid "RightSidebar|Copy email address"
|
||||
msgstr ""
|
||||
|
||||
msgid "RightSidebar|Issue email"
|
||||
msgstr ""
|
||||
|
||||
msgid "RightSidebar|adding a"
|
||||
|
|
|
|||
|
|
@ -105,7 +105,6 @@ describe('diffs/components/app', () => {
|
|||
jest.spyOn(wrapper.vm, 'fetchDiffFilesBatch').mockImplementation(fetchResolver);
|
||||
jest.spyOn(wrapper.vm, 'fetchCoverageFiles').mockImplementation(fetchResolver);
|
||||
jest.spyOn(wrapper.vm, 'setDiscussions').mockImplementation(() => {});
|
||||
jest.spyOn(wrapper.vm, 'startRenderDiffsQueue').mockImplementation(() => {});
|
||||
jest.spyOn(wrapper.vm, 'unwatchDiscussions').mockImplementation(() => {});
|
||||
jest.spyOn(wrapper.vm, 'unwatchRetrievingBatches').mockImplementation(() => {});
|
||||
store.state.diffs.retrievingBatches = true;
|
||||
|
|
@ -119,7 +118,6 @@ describe('diffs/components/app', () => {
|
|||
|
||||
await nextTick();
|
||||
|
||||
expect(wrapper.vm.startRenderDiffsQueue).toHaveBeenCalled();
|
||||
expect(wrapper.vm.fetchDiffFilesMeta).toHaveBeenCalled();
|
||||
expect(wrapper.vm.fetchDiffFilesBatch).toHaveBeenCalled();
|
||||
expect(wrapper.vm.fetchCoverageFiles).toHaveBeenCalled();
|
||||
|
|
@ -134,7 +132,6 @@ describe('diffs/components/app', () => {
|
|||
|
||||
await nextTick();
|
||||
|
||||
expect(wrapper.vm.startRenderDiffsQueue).toHaveBeenCalled();
|
||||
expect(wrapper.vm.fetchDiffFilesMeta).toHaveBeenCalled();
|
||||
expect(wrapper.vm.fetchDiffFilesBatch).toHaveBeenCalled();
|
||||
expect(wrapper.vm.fetchCoverageFiles).toHaveBeenCalled();
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ describe('DiffsStoreActions', () => {
|
|||
jest.spyOn(utils, 'idleCallback').mockImplementation(() => null);
|
||||
['requestAnimationFrame', 'requestIdleCallback'].forEach((method) => {
|
||||
global[method] = (cb) => {
|
||||
cb();
|
||||
cb({ timeRemaining: () => 10 });
|
||||
};
|
||||
});
|
||||
});
|
||||
|
|
@ -198,7 +198,7 @@ describe('DiffsStoreActions', () => {
|
|||
{ type: types.VIEW_DIFF_FILE, payload: 'test2' },
|
||||
{ type: types.SET_RETRIEVING_BATCHES, payload: false },
|
||||
],
|
||||
[],
|
||||
[{ type: 'startRenderDiffsQueue' }, { type: 'startRenderDiffsQueue' }],
|
||||
done,
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,22 +1,17 @@
|
|||
import { getByText } from '@testing-library/dom';
|
||||
import { mount } from '@vue/test-utils';
|
||||
import { shallowMount } from '@vue/test-utils';
|
||||
import CopyEmailToClipboard from '~/sidebar/components/copy_email_to_clipboard.vue';
|
||||
import ClipboardButton from '~/vue_shared/components/clipboard_button.vue';
|
||||
import CopyableField from '~/vue_shared/components/sidebar/copyable_field.vue';
|
||||
|
||||
describe('CopyEmailToClipboard component', () => {
|
||||
const sampleEmail = 'sample+email@test.com';
|
||||
const mockIssueEmailAddress = 'sample+email@test.com';
|
||||
|
||||
const wrapper = mount(CopyEmailToClipboard, {
|
||||
const wrapper = shallowMount(CopyEmailToClipboard, {
|
||||
propsData: {
|
||||
copyText: sampleEmail,
|
||||
issueEmailAddress: mockIssueEmailAddress,
|
||||
},
|
||||
});
|
||||
|
||||
it('renders the Issue email text with the forwardable email', () => {
|
||||
expect(getByText(wrapper.element, `Issue email: ${sampleEmail}`)).not.toBeNull();
|
||||
});
|
||||
|
||||
it('finds ClipboardButton with the correct props', () => {
|
||||
expect(wrapper.find(ClipboardButton).props('text')).toBe(sampleEmail);
|
||||
it('sets CopyableField `value` prop to issueEmailAddress', () => {
|
||||
expect(wrapper.find(CopyableField).props('value')).toBe(mockIssueEmailAddress);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -61,5 +61,14 @@ describe('SidebarCopyableField', () => {
|
|||
expect(findClipboardButton().exists()).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('with `clipboardTooltipText` prop', () => {
|
||||
it('sets ClipboardButton `title` prop to `clipboardTooltipText` value', () => {
|
||||
const mockClipboardTooltipText = 'Copy my custom value';
|
||||
createComponent({ ...defaultProps, clipboardTooltipText: mockClipboardTooltipText });
|
||||
|
||||
expect(findClipboardButton().props('title')).toBe(mockClipboardTooltipText);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue