Add latest changes from gitlab-org/gitlab@master
This commit is contained in:
parent
8bdfdd49b3
commit
9474789110
18 changed files with 356 additions and 57 deletions
|
@ -178,8 +178,8 @@ export default {
|
||||||
serverErrorMessage: '',
|
serverErrorMessage: '',
|
||||||
isErrorAlertDismissed: false,
|
isErrorAlertDismissed: false,
|
||||||
sort: 'STARTED_AT_DESC',
|
sort: 'STARTED_AT_DESC',
|
||||||
statusFilter: [],
|
statusFilter: ALERTS_STATUS_TABS[0].filters,
|
||||||
filteredByStatus: '',
|
filteredByStatus: ALERTS_STATUS_TABS[0].status,
|
||||||
alerts: {},
|
alerts: {},
|
||||||
alertsCount: {},
|
alertsCount: {},
|
||||||
sortBy: 'startedAt',
|
sortBy: 'startedAt',
|
||||||
|
|
|
@ -0,0 +1,18 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
class AddTemporaryIndexForBackfillIntegrationsEnableSslVerification < Gitlab::Database::Migration[2.0]
|
||||||
|
disable_ddl_transaction!
|
||||||
|
|
||||||
|
INDEX_NAME = 'tmp_index_integrations_on_id_where_type_droneci_or_teamcity'
|
||||||
|
INDEX_CONDITION = "type_new IN ('Integrations::DroneCi', 'Integrations::Teamcity') " \
|
||||||
|
"AND encrypted_properties IS NOT NULL"
|
||||||
|
|
||||||
|
def up
|
||||||
|
# this index is used in 20220209121435_backfill_integrations_enable_ssl_verification
|
||||||
|
add_concurrent_index :integrations, :id, where: INDEX_CONDITION, name: INDEX_NAME
|
||||||
|
end
|
||||||
|
|
||||||
|
def down
|
||||||
|
remove_concurrent_index_by_name :integrations, INDEX_NAME
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,57 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
class BackfillIntegrationsEnableSslVerification < Gitlab::Database::Migration[2.0]
|
||||||
|
disable_ddl_transaction!
|
||||||
|
restrict_gitlab_migration gitlab_schema: :gitlab_main
|
||||||
|
|
||||||
|
MIGRATION = 'BackfillIntegrationsEnableSslVerification'
|
||||||
|
INTERVAL = 5.minutes
|
||||||
|
BATCH_SIZE = 1_000
|
||||||
|
|
||||||
|
class Integration < MigrationRecord
|
||||||
|
include EachBatch
|
||||||
|
include IgnorableColumns
|
||||||
|
|
||||||
|
self.table_name = :integrations
|
||||||
|
self.inheritance_column = :_type_disabled
|
||||||
|
|
||||||
|
ignore_column :template, remove_with: '15.0', remove_after: '2022-04-22'
|
||||||
|
ignore_column :type, remove_with: '15.0', remove_after: '2022-04-22'
|
||||||
|
ignore_column :properties, remove_with: '15.1', remove_after: '2022-05-22'
|
||||||
|
|
||||||
|
scope :affected, -> do
|
||||||
|
where(type_new: %w[Integrations::DroneCi Integrations::Teamcity]).where.not(encrypted_properties: nil)
|
||||||
|
end
|
||||||
|
|
||||||
|
attr_encrypted :properties,
|
||||||
|
mode: :per_attribute_iv,
|
||||||
|
key: Settings.attr_encrypted_db_key_base_32,
|
||||||
|
algorithm: 'aes-256-gcm',
|
||||||
|
marshal: true,
|
||||||
|
marshaler: ::Gitlab::Json,
|
||||||
|
encode: false,
|
||||||
|
encode_iv: false
|
||||||
|
|
||||||
|
# Handle assignment of props with symbol keys.
|
||||||
|
# To do this correctly, we need to call the method generated by attr_encrypted.
|
||||||
|
alias_method :attr_encrypted_props=, :properties=
|
||||||
|
private :attr_encrypted_props=
|
||||||
|
|
||||||
|
def properties=(props)
|
||||||
|
self.attr_encrypted_props = props&.with_indifferent_access&.freeze
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def up
|
||||||
|
queue_background_migration_jobs_by_range_at_intervals(
|
||||||
|
Integration.affected,
|
||||||
|
MIGRATION,
|
||||||
|
INTERVAL,
|
||||||
|
batch_size: BATCH_SIZE,
|
||||||
|
track_jobs: true
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
def down
|
||||||
|
end
|
||||||
|
end
|
1
db/schema_migrations/20220425121410
Normal file
1
db/schema_migrations/20220425121410
Normal file
|
@ -0,0 +1 @@
|
||||||
|
73ab85c5ef724f6aba4a964f42e586db0a198affd134ba598189629fb95989a4
|
1
db/schema_migrations/20220425121435
Normal file
1
db/schema_migrations/20220425121435
Normal file
|
@ -0,0 +1 @@
|
||||||
|
fd00d3d8104cba09134853976cf4f1dea7abb5e1dd2c4ea8b46adc4742c71030
|
|
@ -29742,6 +29742,8 @@ CREATE INDEX tmp_index_for_null_project_namespace_id ON projects USING btree (id
|
||||||
|
|
||||||
CREATE INDEX tmp_index_for_project_namespace_id_migration_on_routes ON routes USING btree (id) WHERE ((namespace_id IS NULL) AND ((source_type)::text = 'Project'::text));
|
CREATE INDEX tmp_index_for_project_namespace_id_migration_on_routes ON routes USING btree (id) WHERE ((namespace_id IS NULL) AND ((source_type)::text = 'Project'::text));
|
||||||
|
|
||||||
|
CREATE INDEX tmp_index_integrations_on_id_where_type_droneci_or_teamcity ON integrations USING btree (id) WHERE ((type_new = ANY (ARRAY['Integrations::DroneCi'::text, 'Integrations::Teamcity'::text])) AND (encrypted_properties IS NOT NULL));
|
||||||
|
|
||||||
CREATE INDEX tmp_index_issues_on_issue_type_and_id ON issues USING btree (issue_type, id);
|
CREATE INDEX tmp_index_issues_on_issue_type_and_id ON issues USING btree (issue_type, id);
|
||||||
|
|
||||||
CREATE INDEX tmp_index_members_on_state ON members USING btree (state) WHERE (state = 2);
|
CREATE INDEX tmp_index_members_on_state ON members USING btree (state) WHERE (state = 2);
|
||||||
|
|
|
@ -42,7 +42,7 @@ To bring the former **primary** site up to date:
|
||||||
NOTE:
|
NOTE:
|
||||||
If you [changed the DNS records](index.md#step-4-optional-updating-the-primary-domain-dns-record)
|
If you [changed the DNS records](index.md#step-4-optional-updating-the-primary-domain-dns-record)
|
||||||
for this site during disaster recovery procedure you may need to [block
|
for this site during disaster recovery procedure you may need to [block
|
||||||
all the writes to this site](planned_failover.md#prevent-updates-to-the-primary-node)
|
all the writes to this site](planned_failover.md#prevent-updates-to-the-primary-site)
|
||||||
during this procedure.
|
during this procedure.
|
||||||
|
|
||||||
1. [Set up database replication](../setup/database.md). In this case, the **secondary** site
|
1. [Set up database replication](../setup/database.md). In this case, the **secondary** site
|
||||||
|
|
|
@ -12,10 +12,10 @@ the event of unplanned outage, but it can be used in conjunction with a planned
|
||||||
failover to migrate your GitLab instance between regions without extended
|
failover to migrate your GitLab instance between regions without extended
|
||||||
downtime.
|
downtime.
|
||||||
|
|
||||||
As replication between Geo nodes is asynchronous, a planned failover requires
|
As replication between Geo sites is asynchronous, a planned failover requires
|
||||||
a maintenance window in which updates to the **primary** node are blocked. The
|
a maintenance window in which updates to the **primary** site are blocked. The
|
||||||
length of this window is determined by your replication capacity - once the
|
length of this window is determined by your replication capacity - once the
|
||||||
**secondary** node is completely synchronized with the **primary** node, the failover can occur without
|
**secondary** site is completely synchronized with the **primary** site, the failover can occur without
|
||||||
data loss.
|
data loss.
|
||||||
|
|
||||||
This document assumes you already have a fully configured, working Geo setup.
|
This document assumes you already have a fully configured, working Geo setup.
|
||||||
|
@ -28,7 +28,7 @@ have a high degree of confidence in being able to perform them accurately.
|
||||||
## Not all data is automatically replicated
|
## Not all data is automatically replicated
|
||||||
|
|
||||||
If you are using any GitLab features that Geo [doesn't support](../replication/datatypes.md#limitations-on-replicationverification),
|
If you are using any GitLab features that Geo [doesn't support](../replication/datatypes.md#limitations-on-replicationverification),
|
||||||
you must make separate provisions to ensure that the **secondary** node has an
|
you must make separate provisions to ensure that the **secondary** site has an
|
||||||
up-to-date copy of any data associated with that feature. This may extend the
|
up-to-date copy of any data associated with that feature. This may extend the
|
||||||
required scheduled maintenance period significantly.
|
required scheduled maintenance period significantly.
|
||||||
|
|
||||||
|
@ -36,7 +36,7 @@ A common strategy for keeping this period as short as possible for data stored
|
||||||
in files is to use `rsync` to transfer the data. An initial `rsync` can be
|
in files is to use `rsync` to transfer the data. An initial `rsync` can be
|
||||||
performed ahead of the maintenance window; subsequent `rsync`s (including a
|
performed ahead of the maintenance window; subsequent `rsync`s (including a
|
||||||
final transfer inside the maintenance window) then transfers only the
|
final transfer inside the maintenance window) then transfers only the
|
||||||
*changes* between the **primary** node and the **secondary** nodes.
|
*changes* between the **primary** site and the **secondary** sites.
|
||||||
|
|
||||||
Repository-centric strategies for using `rsync` effectively can be found in the
|
Repository-centric strategies for using `rsync` effectively can be found in the
|
||||||
[moving repositories](../../operations/moving_repositories.md) documentation; these strategies can
|
[moving repositories](../../operations/moving_repositories.md) documentation; these strategies can
|
||||||
|
@ -98,42 +98,42 @@ Doing so reduces both the length of the maintenance window, and the risk of data
|
||||||
loss as a result of a poorly executed planned failover.
|
loss as a result of a poorly executed planned failover.
|
||||||
|
|
||||||
In GitLab 12.4, you can optionally allow GitLab to manage replication of Object Storage for
|
In GitLab 12.4, you can optionally allow GitLab to manage replication of Object Storage for
|
||||||
**secondary** nodes. For more information, see [Object Storage replication](../replication/object_storage.md).
|
**secondary** sites. For more information, see [Object Storage replication](../replication/object_storage.md).
|
||||||
|
|
||||||
### Review the configuration of each **secondary** node
|
### Review the configuration of each **secondary** site
|
||||||
|
|
||||||
Database settings are automatically replicated to the **secondary** node, but the
|
Database settings are automatically replicated to the **secondary** site, but the
|
||||||
`/etc/gitlab/gitlab.rb` file must be set up manually, and differs between
|
`/etc/gitlab/gitlab.rb` file must be set up manually, and differs between
|
||||||
nodes. If features such as Mattermost, OAuth or LDAP integration are enabled
|
sites. If features such as Mattermost, OAuth or LDAP integration are enabled
|
||||||
on the **primary** node but not the **secondary** node, they are lost during failover.
|
on the **primary** site but not the **secondary** site, they are lost during failover.
|
||||||
|
|
||||||
Review the `/etc/gitlab/gitlab.rb` file for both nodes and ensure the **secondary** node
|
Review the `/etc/gitlab/gitlab.rb` file for both sites and ensure the **secondary** site
|
||||||
supports everything the **primary** node does **before** scheduling a planned failover.
|
supports everything the **primary** site does **before** scheduling a planned failover.
|
||||||
|
|
||||||
### Run system checks
|
### Run system checks
|
||||||
|
|
||||||
Run the following on both **primary** and **secondary** nodes:
|
Run the following on both **primary** and **secondary** sites:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
gitlab-rake gitlab:check
|
gitlab-rake gitlab:check
|
||||||
gitlab-rake gitlab:geo:check
|
gitlab-rake gitlab:geo:check
|
||||||
```
|
```
|
||||||
|
|
||||||
If any failures are reported on either node, they should be resolved **before**
|
If any failures are reported on either site, they should be resolved **before**
|
||||||
scheduling a planned failover.
|
scheduling a planned failover.
|
||||||
|
|
||||||
### Check that secrets match between nodes
|
### Check that secrets match between sites
|
||||||
|
|
||||||
The SSH host keys and `/etc/gitlab/gitlab-secrets.json` files should be
|
The SSH host keys and `/etc/gitlab/gitlab-secrets.json` files should be
|
||||||
identical on all nodes. Check this by running the following on all nodes and
|
identical on all sites. Check this by running the following on all sites and
|
||||||
comparing the output:
|
comparing the output:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
sudo sha256sum /etc/ssh/ssh_host* /etc/gitlab/gitlab-secrets.json
|
sudo sha256sum /etc/ssh/ssh_host* /etc/gitlab/gitlab-secrets.json
|
||||||
```
|
```
|
||||||
|
|
||||||
If any files differ, replace the content on the **secondary** node with the
|
If any files differ, replace the content on the **secondary** site with the
|
||||||
content from the **primary** node.
|
content from the **primary** site.
|
||||||
|
|
||||||
### Ensure Geo replication is up-to-date
|
### Ensure Geo replication is up-to-date
|
||||||
|
|
||||||
|
@ -141,13 +141,13 @@ The maintenance window won't end until Geo replication and verification is
|
||||||
completely finished. To keep the window as short as possible, you should
|
completely finished. To keep the window as short as possible, you should
|
||||||
ensure these processes are close to 100% as possible during active use.
|
ensure these processes are close to 100% as possible during active use.
|
||||||
|
|
||||||
On the **secondary** node:
|
On the **secondary** site:
|
||||||
|
|
||||||
1. On the top bar, select **Menu > Admin**.
|
1. On the top bar, select **Menu > Admin**.
|
||||||
1. On the left sidebar, select **Geo > Sites**.
|
1. On the left sidebar, select **Geo > Sites**.
|
||||||
Replicated objects (shown in green) should be close to 100%,
|
Replicated objects (shown in green) should be close to 100%,
|
||||||
and there should be no failures (shown in red). If a large proportion of
|
and there should be no failures (shown in red). If a large proportion of
|
||||||
objects aren't yet replicated (shown in gray), consider giving the node more
|
objects aren't yet replicated (shown in gray), consider giving the site more
|
||||||
time to complete
|
time to complete
|
||||||
|
|
||||||
![Replication status](../replication/img/geo_dashboard_v14_0.png)
|
![Replication status](../replication/img/geo_dashboard_v14_0.png)
|
||||||
|
@ -160,7 +160,7 @@ You can use the [Geo status API](../../../api/geo_nodes.md#retrieve-project-sync
|
||||||
the reasons for failure.
|
the reasons for failure.
|
||||||
|
|
||||||
A common cause of replication failures is the data being missing on the
|
A common cause of replication failures is the data being missing on the
|
||||||
**primary** node - you can resolve these failures by restoring the data from backup,
|
**primary** site - you can resolve these failures by restoring the data from backup,
|
||||||
or removing references to the missing data.
|
or removing references to the missing data.
|
||||||
|
|
||||||
### Verify the integrity of replicated data
|
### Verify the integrity of replicated data
|
||||||
|
@ -169,7 +169,7 @@ This [content was moved to another location](background_verification.md).
|
||||||
|
|
||||||
### Notify users of scheduled maintenance
|
### Notify users of scheduled maintenance
|
||||||
|
|
||||||
On the **primary** node:
|
On the **primary** site:
|
||||||
|
|
||||||
1. On the top bar, select **Menu > Admin**.
|
1. On the top bar, select **Menu > Admin**.
|
||||||
1. On the left sidebar, select **Messages**.
|
1. On the left sidebar, select **Messages**.
|
||||||
|
@ -178,12 +178,12 @@ On the **primary** node:
|
||||||
takes to finish syncing.
|
takes to finish syncing.
|
||||||
1. Select **Add broadcast message**.
|
1. Select **Add broadcast message**.
|
||||||
|
|
||||||
## Prevent updates to the **primary** node
|
## Prevent updates to the **primary** site
|
||||||
|
|
||||||
To ensure that all data is replicated to a secondary site, updates (write requests) need to
|
To ensure that all data is replicated to a secondary site, updates (write requests) need to
|
||||||
be disabled on the **primary** site:
|
be disabled on the **primary** site:
|
||||||
|
|
||||||
1. Enable [maintenance mode](../../maintenance_mode/index.md) on the **primary** node.
|
1. Enable [maintenance mode](../../maintenance_mode/index.md) on the **primary** site.
|
||||||
1. On the top bar, select **Menu > Admin**.
|
1. On the top bar, select **Menu > Admin**.
|
||||||
1. On the left sidebar, select **Monitoring > Background Jobs**.
|
1. On the left sidebar, select **Monitoring > Background Jobs**.
|
||||||
1. On the Sidekiq dashboard, select **Cron**.
|
1. On the Sidekiq dashboard, select **Cron**.
|
||||||
|
@ -199,7 +199,7 @@ GitLab 13.9 through GitLab 14.3 are affected by a bug in which the Geo secondary
|
||||||
|
|
||||||
1. If you are manually replicating any data not managed by Geo, trigger the
|
1. If you are manually replicating any data not managed by Geo, trigger the
|
||||||
final replication process now.
|
final replication process now.
|
||||||
1. On the **primary** node:
|
1. On the **primary** site:
|
||||||
1. On the top bar, select **Menu > Admin**.
|
1. On the top bar, select **Menu > Admin**.
|
||||||
1. On the left sidebar, select **Monitoring > Background Jobs**.
|
1. On the left sidebar, select **Monitoring > Background Jobs**.
|
||||||
1. On the Sidekiq dashboard, select **Queues**, and wait for all queues except
|
1. On the Sidekiq dashboard, select **Queues**, and wait for all queues except
|
||||||
|
@ -207,14 +207,14 @@ GitLab 13.9 through GitLab 14.3 are affected by a bug in which the Geo secondary
|
||||||
These queues contain work that has been submitted by your users; failing over
|
These queues contain work that has been submitted by your users; failing over
|
||||||
before it is completed, causes the work to be lost.
|
before it is completed, causes the work to be lost.
|
||||||
1. On the left sidebar, select **Geo > Sites** and wait for the
|
1. On the left sidebar, select **Geo > Sites** and wait for the
|
||||||
following conditions to be true of the **secondary** node you are failing over to:
|
following conditions to be true of the **secondary** site you are failing over to:
|
||||||
|
|
||||||
- All replication meters reach 100% replicated, 0% failures.
|
- All replication meters reach 100% replicated, 0% failures.
|
||||||
- All verification meters reach 100% verified, 0% failures.
|
- All verification meters reach 100% verified, 0% failures.
|
||||||
- Database replication lag is 0ms.
|
- Database replication lag is 0ms.
|
||||||
- The Geo log cursor is up to date (0 events behind).
|
- The Geo log cursor is up to date (0 events behind).
|
||||||
|
|
||||||
1. On the **secondary** node:
|
1. On the **secondary** site:
|
||||||
1. On the top bar, select **Menu > Admin**.
|
1. On the top bar, select **Menu > Admin**.
|
||||||
1. On the left sidebar, select **Monitoring > Background Jobs**.
|
1. On the left sidebar, select **Monitoring > Background Jobs**.
|
||||||
1. On the Sidekiq dashboard, select **Queues**, and wait for all the `geo`
|
1. On the Sidekiq dashboard, select **Queues**, and wait for all the `geo`
|
||||||
|
@ -222,16 +222,16 @@ GitLab 13.9 through GitLab 14.3 are affected by a bug in which the Geo secondary
|
||||||
1. [Run an integrity check](../../raketasks/check.md) to verify the integrity
|
1. [Run an integrity check](../../raketasks/check.md) to verify the integrity
|
||||||
of CI artifacts, LFS objects, and uploads in file storage.
|
of CI artifacts, LFS objects, and uploads in file storage.
|
||||||
|
|
||||||
At this point, your **secondary** node contains an up-to-date copy of everything the
|
At this point, your **secondary** site contains an up-to-date copy of everything the
|
||||||
**primary** node has, meaning nothing was lost when you fail over.
|
**primary** site has, meaning nothing was lost when you fail over.
|
||||||
|
|
||||||
## Promote the **secondary** node
|
## Promote the **secondary** site
|
||||||
|
|
||||||
After the replication is finished, [promote the **secondary** node to a **primary** node](index.md). This process causes a brief outage on the **secondary** node, and users may need to log in again. If you follow the steps correctly, the old primary Geo site should still be disabled and user traffic should go to the newly-promoted site instead.
|
After the replication is finished, [promote the **secondary** site to a **primary** site](index.md). This process causes a brief outage on the **secondary** site, and users may need to log in again. If you follow the steps correctly, the old primary Geo site should still be disabled and user traffic should go to the newly-promoted site instead.
|
||||||
|
|
||||||
When the promotion is completed, the maintenance window is over, and your new **primary** node now
|
When the promotion is completed, the maintenance window is over, and your new **primary** site now
|
||||||
begins to diverge from the old one. If problems do arise at this point, failing
|
begins to diverge from the old one. If problems do arise at this point, failing
|
||||||
back to the old **primary** node [is possible](bring_primary_back.md), but likely to result
|
back to the old **primary** site [is possible](bring_primary_back.md), but likely to result
|
||||||
in the loss of any data uploaded to the new **primary** in the meantime.
|
in the loss of any data uploaded to the new **primary** in the meantime.
|
||||||
|
|
||||||
Don't forget to remove the broadcast message after the failover is complete.
|
Don't forget to remove the broadcast message after the failover is complete.
|
||||||
|
|
|
@ -69,7 +69,7 @@ GitLab 13.9 through GitLab 14.3 are affected by a bug in which the Geo secondary
|
||||||
On the **secondary** site:
|
On the **secondary** site:
|
||||||
|
|
||||||
1. On the top bar, select **Menu > Admin**.
|
1. On the top bar, select **Menu > Admin**.
|
||||||
1. On the left sidebar, select **Geo > Nodes** to see its status.
|
1. On the left sidebar, select **Geo > Sites** to see its status.
|
||||||
Replicated objects (shown in green) should be close to 100%,
|
Replicated objects (shown in green) should be close to 100%,
|
||||||
and there should be no failures (shown in red). If a large proportion of
|
and there should be no failures (shown in red). If a large proportion of
|
||||||
objects aren't yet replicated (shown in gray), consider giving the site more
|
objects aren't yet replicated (shown in gray), consider giving the site more
|
||||||
|
@ -100,22 +100,22 @@ follow these steps to avoid unnecessary data loss:
|
||||||
**primary**. Your **secondary** site still needs read-only
|
**primary**. Your **secondary** site still needs read-only
|
||||||
access to the **primary** site during the maintenance window:
|
access to the **primary** site during the maintenance window:
|
||||||
|
|
||||||
1. At the scheduled time, using your cloud provider or your node's firewall, block
|
1. At the scheduled time, using your cloud provider or your site's firewall, block
|
||||||
all HTTP, HTTPS and SSH traffic to/from the **primary** node, **except** for your IP and
|
all HTTP, HTTPS and SSH traffic to/from the **primary** site, **except** for your IP and
|
||||||
the **secondary** node's IP.
|
the **secondary** site's IP.
|
||||||
|
|
||||||
For instance, you can run the following commands on the **primary** node:
|
For instance, you can run the following commands on the **primary** site:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 22 -j ACCEPT
|
sudo iptables -A INPUT -p tcp -s <secondary_site_ip> --destination-port 22 -j ACCEPT
|
||||||
sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 22 -j ACCEPT
|
sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 22 -j ACCEPT
|
||||||
sudo iptables -A INPUT --destination-port 22 -j REJECT
|
sudo iptables -A INPUT --destination-port 22 -j REJECT
|
||||||
|
|
||||||
sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 80 -j ACCEPT
|
sudo iptables -A INPUT -p tcp -s <secondary_site_ip> --destination-port 80 -j ACCEPT
|
||||||
sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 80 -j ACCEPT
|
sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 80 -j ACCEPT
|
||||||
sudo iptables -A INPUT --tcp-dport 80 -j REJECT
|
sudo iptables -A INPUT --tcp-dport 80 -j REJECT
|
||||||
|
|
||||||
sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 443 -j ACCEPT
|
sudo iptables -A INPUT -p tcp -s <secondary_site_ip> --destination-port 443 -j ACCEPT
|
||||||
sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 443 -j ACCEPT
|
sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 443 -j ACCEPT
|
||||||
sudo iptables -A INPUT --tcp-dport 443 -j REJECT
|
sudo iptables -A INPUT --tcp-dport 443 -j REJECT
|
||||||
```
|
```
|
||||||
|
@ -157,8 +157,8 @@ follow these steps to avoid unnecessary data loss:
|
||||||
those with `geo` in the name to drop to 0.
|
those with `geo` in the name to drop to 0.
|
||||||
These queues contain work that has been submitted by your users; failing over
|
These queues contain work that has been submitted by your users; failing over
|
||||||
before it is completed, causes the work to be lost.
|
before it is completed, causes the work to be lost.
|
||||||
1. On the left sidebar, select **Geo > Nodes** and wait for the
|
1. On the left sidebar, select **Geo > Sites** and wait for the
|
||||||
following conditions to be true of the **secondary** node you are failing over to:
|
following conditions to be true of the **secondary** site you are failing over to:
|
||||||
|
|
||||||
- All replication meters reach 100% replicated, 0% failures.
|
- All replication meters reach 100% replicated, 0% failures.
|
||||||
- All verification meters reach 100% verified, 0% failures.
|
- All verification meters reach 100% verified, 0% failures.
|
||||||
|
@ -230,13 +230,13 @@ follow these steps to avoid unnecessary data loss:
|
||||||
|
|
||||||
1. SSH to every Sidekiq, PostgresSQL, and Gitaly node in the **secondary** site and run one of the following commands:
|
1. SSH to every Sidekiq, PostgresSQL, and Gitaly node in the **secondary** site and run one of the following commands:
|
||||||
|
|
||||||
- To promote the secondary node to primary:
|
- To promote the secondary site to primary:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
sudo gitlab-ctl geo promote
|
sudo gitlab-ctl geo promote
|
||||||
```
|
```
|
||||||
|
|
||||||
- To promote the secondary node to primary **without any further confirmation**:
|
- To promote the secondary site to primary **without any further confirmation**:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
sudo gitlab-ctl geo promote --force
|
sudo gitlab-ctl geo promote --force
|
||||||
|
@ -244,13 +244,13 @@ follow these steps to avoid unnecessary data loss:
|
||||||
|
|
||||||
1. SSH into each Rails node on your **secondary** site and run one of the following commands:
|
1. SSH into each Rails node on your **secondary** site and run one of the following commands:
|
||||||
|
|
||||||
- To promote the secondary node to primary:
|
- To promote the secondary site to primary:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
sudo gitlab-ctl geo promote
|
sudo gitlab-ctl geo promote
|
||||||
```
|
```
|
||||||
|
|
||||||
- To promote the secondary node to primary **without any further confirmation**:
|
- To promote the secondary site to primary **without any further confirmation**:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
sudo gitlab-ctl geo promote --force
|
sudo gitlab-ctl geo promote --force
|
||||||
|
|
|
@ -876,6 +876,10 @@ See the limits in the [Add a design to an issue](../user/project/issues/design_m
|
||||||
|
|
||||||
## Push Event Limits
|
## Push Event Limits
|
||||||
|
|
||||||
|
### Max push size
|
||||||
|
|
||||||
|
The maximum allowed [push size](../user/admin_area/settings/account_and_limit_settings.md#max-push-size) is set to 5 GB.
|
||||||
|
|
||||||
### Webhooks and Project Services
|
### Webhooks and Project Services
|
||||||
|
|
||||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31009) in GitLab 12.4.
|
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31009) in GitLab 12.4.
|
||||||
|
|
|
@ -166,7 +166,7 @@ Package Registry allows you to install but not publish packages.
|
||||||
|
|
||||||
Background jobs (cron jobs, Sidekiq) continue running as is, because background jobs are not automatically disabled.
|
Background jobs (cron jobs, Sidekiq) continue running as is, because background jobs are not automatically disabled.
|
||||||
|
|
||||||
[During a planned Geo failover](../geo/disaster_recovery/planned_failover.md#prevent-updates-to-the-primary-node),
|
[During a planned Geo failover](../geo/disaster_recovery/planned_failover.md#prevent-updates-to-the-primary-site),
|
||||||
it is recommended that you disable all cron jobs except for those related to Geo.
|
it is recommended that you disable all cron jobs except for those related to Geo.
|
||||||
|
|
||||||
To monitor queues and disable jobs:
|
To monitor queues and disable jobs:
|
||||||
|
@ -210,4 +210,4 @@ For the same reason we don't automatically block background jobs when Maintenanc
|
||||||
|
|
||||||
The resulting database writes are acceptable. Here, the trade-off is between more service degradation and the completion of replication.
|
The resulting database writes are acceptable. Here, the trade-off is between more service degradation and the completion of replication.
|
||||||
|
|
||||||
However, during a planned failover, we [ask users to turn off cron jobs that are not related to Geo, manually](../geo/disaster_recovery/planned_failover.md#prevent-updates-to-the-primary-node). In the absence of new database writes and non-Geo cron jobs, new background jobs would either not be created at all or be minimal.
|
However, during a planned failover, we [ask users to turn off cron jobs that are not related to Geo, manually](../geo/disaster_recovery/planned_failover.md#prevent-updates-to-the-primary-site). In the absence of new database writes and non-Geo cron jobs, new background jobs would either not be created at all or be minimal.
|
||||||
|
|
|
@ -52,6 +52,8 @@ gitlab/gitlab-ee:11.5.3-ee.0
|
||||||
|
|
||||||
#### SAML for Authentication
|
#### SAML for Authentication
|
||||||
|
|
||||||
|
In the following examples, when replacing `<GITLAB_IP_OR_DOMAIN>` and `<SAML_IP_OR_DOMAIN>` it is important to prepend your IP or domain name, with the protocol (`http://` or `https://`) being used.
|
||||||
|
|
||||||
We can use the [`test-saml-idp` Docker image](https://hub.docker.com/r/jamedjo/test-saml-idp)
|
We can use the [`test-saml-idp` Docker image](https://hub.docker.com/r/jamedjo/test-saml-idp)
|
||||||
to do the work for us:
|
to do the work for us:
|
||||||
|
|
||||||
|
|
|
@ -124,8 +124,11 @@ To remove a page:
|
||||||
|
|
||||||
1. Leave the page title. Remove all other content, including the version history bullets and the word `WARNING:`.
|
1. Leave the page title. Remove all other content, including the version history bullets and the word `WARNING:`.
|
||||||
1. After the title, change `(deprecated)` to `(removed)`.
|
1. After the title, change `(deprecated)` to `(removed)`.
|
||||||
1. Add `remove_date` in the YAML metadata. Set the value to a date three months after
|
1. Update the YAML metadata:
|
||||||
the release when the feature was removed. For example:
|
- For `remove_date`, set the value to a date three months after
|
||||||
|
the release when the feature was removed.
|
||||||
|
- For the `redirect_to`, set a path to a file that makes sense. If no obvious
|
||||||
|
page exists, use the docs home page.
|
||||||
|
|
||||||
```markdown
|
```markdown
|
||||||
---
|
---
|
||||||
|
@ -133,6 +136,7 @@ To remove a page:
|
||||||
group: Global Search
|
group: Global Search
|
||||||
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
|
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
|
||||||
remove_date: '2022-08-02'
|
remove_date: '2022-08-02'
|
||||||
|
redirect_to: '../newpath/to/file/index.md'
|
||||||
---
|
---
|
||||||
|
|
||||||
# Title (removed) **(ULTIMATE SELF)**
|
# Title (removed) **(ULTIMATE SELF)**
|
||||||
|
@ -158,7 +162,7 @@ To remove a topic:
|
||||||
For the `remove_date`, set a date three months after the release where it was removed.
|
For the `remove_date`, set a date three months after the release where it was removed.
|
||||||
|
|
||||||
```markdown
|
```markdown
|
||||||
<!--- The following content will be removed on remove_date: '2023-08-22' -->
|
<!--- start_remove The following content will be removed on remove_date: '2023-08-22' -->
|
||||||
|
|
||||||
## Title (removed) **(ULTIMATE SELF)**
|
## Title (removed) **(ULTIMATE SELF)**
|
||||||
|
|
||||||
|
|
|
@ -51,12 +51,14 @@ For GitLab.com repository size limits, read [accounts and limit settings](../../
|
||||||
|
|
||||||
## Max push size
|
## Max push size
|
||||||
|
|
||||||
You can change the maximum push size for your repository:
|
You can change the maximum push size for your instance:
|
||||||
|
|
||||||
1. On the top bar, select **Menu > Admin**.
|
1. On the top bar, select **Menu > Admin**.
|
||||||
1. On the left sidebar, select **Settings > General**, then expand **Account and limit**.
|
1. On the left sidebar, select **Settings > General**, then expand **Account and limit**.
|
||||||
1. Increase or decrease by changing the value in **Maximum push size (MB)**.
|
1. Increase or decrease by changing the value in **Maximum push size (MB)**.
|
||||||
|
|
||||||
|
For GitLab.com appliation limits, read [GitLab application limits](../../../administration/instance_limits.md#max-push-size).
|
||||||
|
|
||||||
NOTE:
|
NOTE:
|
||||||
When you [add files to a repository](../../project/repository/web_editor.md#create-a-file)
|
When you [add files to a repository](../../project/repository/web_editor.md#create-a-file)
|
||||||
through the web UI, the maximum **attachment** size is the limiting factor,
|
through the web UI, the maximum **attachment** size is the limiting factor,
|
||||||
|
|
|
@ -0,0 +1,104 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
module Gitlab
|
||||||
|
module BackgroundMigration
|
||||||
|
# Enable SSL verification for CI integrations with known-good hostnames.
|
||||||
|
class BackfillIntegrationsEnableSslVerification
|
||||||
|
INTEGRATIONS = {
|
||||||
|
# This matches the logic in `Integrations::DroneCi#url_is_saas?`
|
||||||
|
# - https://gitlab.com/gitlab-org/gitlab/blob/65b7fc1ad1ad33247890324e9a3396993b7718a1/app/models/integrations/drone_ci.rb#L122-127
|
||||||
|
# - https://docs.drone.io/pipeline/environment/reference/drone-system-hostname/
|
||||||
|
'Integrations::DroneCi' => [
|
||||||
|
:drone_url,
|
||||||
|
/\Acloud\.drone\.io\z/i.freeze
|
||||||
|
],
|
||||||
|
# This matches the logic in `Integrations::Teamcity#url_is_saas?`
|
||||||
|
# - https://gitlab.com/gitlab-org/gitlab/blob/65b7fc1ad1ad33247890324e9a3396993b7718a1/app/models/integrations/teamcity.rb#L117-122
|
||||||
|
# - https://www.jetbrains.com/help/teamcity/cloud/migrate-from-teamcity-on-premises-to-teamcity-cloud.html#Migration+Process
|
||||||
|
'Integrations::Teamcity' => [
|
||||||
|
:teamcity_url,
|
||||||
|
/\A[^\.]+\.teamcity\.com\z/i.freeze
|
||||||
|
]
|
||||||
|
|
||||||
|
# Other CI integrations which don't seem to have a SaaS offering:
|
||||||
|
# - Atlassian Bamboo (the SaaS offering is Bitbucket Pipelines)
|
||||||
|
# - Jenkins (self-hosted only)
|
||||||
|
# - MockCi (development only)
|
||||||
|
}.freeze
|
||||||
|
|
||||||
|
# Define the `Integration` model
|
||||||
|
class Integration < ActiveRecord::Base
|
||||||
|
include IgnorableColumns
|
||||||
|
|
||||||
|
self.table_name = :integrations
|
||||||
|
self.inheritance_column = :_type_disabled
|
||||||
|
|
||||||
|
ignore_column :template, remove_with: '15.0', remove_after: '2022-04-22'
|
||||||
|
ignore_column :type, remove_with: '15.0', remove_after: '2022-04-22'
|
||||||
|
ignore_column :properties, remove_with: '15.1', remove_after: '2022-05-22'
|
||||||
|
|
||||||
|
scope :affected, -> { where(type_new: INTEGRATIONS.keys).where.not(encrypted_properties: nil) }
|
||||||
|
|
||||||
|
attr_encrypted :properties,
|
||||||
|
mode: :per_attribute_iv,
|
||||||
|
key: Settings.attr_encrypted_db_key_base_32,
|
||||||
|
algorithm: 'aes-256-gcm',
|
||||||
|
marshal: true,
|
||||||
|
marshaler: ::Gitlab::Json,
|
||||||
|
encode: false,
|
||||||
|
encode_iv: false
|
||||||
|
|
||||||
|
# Handle assignment of props with symbol keys.
|
||||||
|
# To do this correctly, we need to call the method generated by attr_encrypted.
|
||||||
|
alias_method :attr_encrypted_props=, :properties=
|
||||||
|
private :attr_encrypted_props=
|
||||||
|
|
||||||
|
def properties=(props)
|
||||||
|
self.attr_encrypted_props = props&.with_indifferent_access&.freeze
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def perform(start_id, stop_id)
|
||||||
|
integration_ids = Integration
|
||||||
|
.affected
|
||||||
|
.where(id: (start_id..stop_id))
|
||||||
|
.pluck(:id)
|
||||||
|
|
||||||
|
integration_ids.each do |id|
|
||||||
|
Integration.transaction do
|
||||||
|
integration = Integration.lock.find(id)
|
||||||
|
process_integration(integration)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
mark_job_as_succeeded(start_id, stop_id)
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def process_integration(integration)
|
||||||
|
url_field, known_hostnames = INTEGRATIONS.fetch(integration.type_new)
|
||||||
|
|
||||||
|
url = integration.properties[url_field.to_s] if integration.properties.present?
|
||||||
|
return unless url.present?
|
||||||
|
|
||||||
|
parsed_url = Addressable::URI.parse(url)
|
||||||
|
return unless parsed_url.scheme == 'https' && parsed_url.hostname =~ known_hostnames
|
||||||
|
|
||||||
|
integration.properties = integration.properties.merge('enable_ssl_verification' => true)
|
||||||
|
|
||||||
|
integration.save!(touch: false)
|
||||||
|
rescue Addressable::URI::InvalidURIError, ActiveRecord::RecordInvalid
|
||||||
|
# Don't change the configuration if the record is invalid, in this case
|
||||||
|
# they will just keep having SSL verification disabled.
|
||||||
|
end
|
||||||
|
|
||||||
|
def mark_job_as_succeeded(*arguments)
|
||||||
|
Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
|
||||||
|
self.class.name.demodulize,
|
||||||
|
arguments
|
||||||
|
)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -3,4 +3,3 @@
|
||||||
--format documentation
|
--format documentation
|
||||||
--default-path qa/specs
|
--default-path qa/specs
|
||||||
--require spec_helper
|
--require spec_helper
|
||||||
--tag ~orchestrated
|
|
||||||
|
|
|
@ -0,0 +1,73 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
require 'spec_helper'
|
||||||
|
|
||||||
|
RSpec.describe Gitlab::BackgroundMigration::BackfillIntegrationsEnableSslVerification, schema: 20220425121410 do
|
||||||
|
let(:migration) { described_class.new }
|
||||||
|
let(:integrations) { described_class::Integration }
|
||||||
|
|
||||||
|
before do
|
||||||
|
integrations.create!(id: 1, type_new: 'Integrations::Bamboo') # unaffected integration
|
||||||
|
integrations.create!(id: 2, type_new: 'Integrations::DroneCi') # no properties
|
||||||
|
integrations.create!(id: 3, type_new: 'Integrations::DroneCi',
|
||||||
|
properties: {}) # no URL
|
||||||
|
integrations.create!(id: 4, type_new: 'Integrations::DroneCi',
|
||||||
|
properties: { 'drone_url' => '' }) # blank URL
|
||||||
|
integrations.create!(id: 5, type_new: 'Integrations::DroneCi',
|
||||||
|
properties: { 'drone_url' => 'https://example.com:foo' }) # invalid URL
|
||||||
|
integrations.create!(id: 6, type_new: 'Integrations::DroneCi',
|
||||||
|
properties: { 'drone_url' => 'https://example.com' }) # unknown URL
|
||||||
|
integrations.create!(id: 7, type_new: 'Integrations::DroneCi',
|
||||||
|
properties: { 'drone_url' => 'http://cloud.drone.io' }) # no HTTPS
|
||||||
|
integrations.create!(id: 8, type_new: 'Integrations::DroneCi',
|
||||||
|
properties: { 'drone_url' => 'https://cloud.drone.io' }) # known URL
|
||||||
|
integrations.create!(id: 9, type_new: 'Integrations::Teamcity',
|
||||||
|
properties: { 'teamcity_url' => 'https://example.com' }) # unknown URL
|
||||||
|
integrations.create!(id: 10, type_new: 'Integrations::Teamcity',
|
||||||
|
properties: { 'teamcity_url' => 'https://foo.bar.teamcity.com' }) # unknown URL
|
||||||
|
integrations.create!(id: 11, type_new: 'Integrations::Teamcity',
|
||||||
|
properties: { 'teamcity_url' => 'https://teamcity.com' }) # unknown URL
|
||||||
|
integrations.create!(id: 12, type_new: 'Integrations::Teamcity',
|
||||||
|
properties: { 'teamcity_url' => 'https://customer.teamcity.com' }) # known URL
|
||||||
|
end
|
||||||
|
|
||||||
|
def properties(id)
|
||||||
|
integrations.find(id).properties
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'enables SSL verification for known-good hostnames', :aggregate_failures do
|
||||||
|
migration.perform(1, 12)
|
||||||
|
|
||||||
|
# Bamboo
|
||||||
|
expect(properties(1)).to be_nil
|
||||||
|
|
||||||
|
# DroneCi
|
||||||
|
expect(properties(2)).to be_nil
|
||||||
|
expect(properties(3)).not_to include('enable_ssl_verification')
|
||||||
|
expect(properties(4)).not_to include('enable_ssl_verification')
|
||||||
|
expect(properties(5)).not_to include('enable_ssl_verification')
|
||||||
|
expect(properties(6)).not_to include('enable_ssl_verification')
|
||||||
|
expect(properties(7)).not_to include('enable_ssl_verification')
|
||||||
|
expect(properties(8)).to include('enable_ssl_verification' => true)
|
||||||
|
|
||||||
|
# Teamcity
|
||||||
|
expect(properties(9)).not_to include('enable_ssl_verification')
|
||||||
|
expect(properties(10)).not_to include('enable_ssl_verification')
|
||||||
|
expect(properties(11)).not_to include('enable_ssl_verification')
|
||||||
|
expect(properties(12)).to include('enable_ssl_verification' => true)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'only updates records within the given ID range', :aggregate_failures do
|
||||||
|
migration.perform(1, 8)
|
||||||
|
|
||||||
|
expect(properties(8)).to include('enable_ssl_verification' => true)
|
||||||
|
expect(properties(12)).not_to include('enable_ssl_verification')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'marks the job as succeeded' do
|
||||||
|
expect(Gitlab::Database::BackgroundMigrationJob).to receive(:mark_all_as_succeeded)
|
||||||
|
.with('BackfillIntegrationsEnableSslVerification', [1, 10])
|
||||||
|
|
||||||
|
migration.perform(1, 10)
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,32 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
require 'spec_helper'
|
||||||
|
require_migration!
|
||||||
|
|
||||||
|
RSpec.describe BackfillIntegrationsEnableSslVerification do
|
||||||
|
let_it_be(:migration) { described_class::MIGRATION }
|
||||||
|
let_it_be(:integrations) { described_class::Integration }
|
||||||
|
|
||||||
|
before do
|
||||||
|
stub_const("#{described_class.name}::BATCH_SIZE", 2)
|
||||||
|
|
||||||
|
integrations.create!(id: 1, type_new: 'Integrations::DroneCi')
|
||||||
|
integrations.create!(id: 2, type_new: 'Integrations::DroneCi', properties: {})
|
||||||
|
integrations.create!(id: 3, type_new: 'Integrations::Bamboo', properties: {})
|
||||||
|
integrations.create!(id: 4, type_new: 'Integrations::Teamcity', properties: {})
|
||||||
|
integrations.create!(id: 5, type_new: 'Integrations::DroneCi', properties: {})
|
||||||
|
integrations.create!(id: 6, type_new: 'Integrations::Teamcity', properties: {})
|
||||||
|
end
|
||||||
|
|
||||||
|
describe '#up' do
|
||||||
|
it 'schedules background jobs for each batch of integrations', :freeze_time do
|
||||||
|
Sidekiq::Testing.fake! do
|
||||||
|
migrate!
|
||||||
|
|
||||||
|
expect(BackgroundMigrationWorker.jobs.size).to eq(2)
|
||||||
|
expect(migration).to be_scheduled_delayed_migration(5.minutes, 2, 4)
|
||||||
|
expect(migration).to be_scheduled_delayed_migration(10.minutes, 5, 6)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
Loading…
Reference in a new issue