kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

Browse source
This commit is contained in:
GitLab Bot 2022-05-03 21:08:54 +00:00
parent 8bdfdd49b3
commit 9474789110
18 changed files with 356 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/assets/javascripts/alert_management/components/alert_management_table.vue
View file

@ -178,8 +178,8 @@ export default {
serverErrorMessage: '',
isErrorAlertDismissed: false,
sort: 'STARTED_AT_DESC',
statusFilter: [],
filteredByStatus: '',
statusFilter: ALERTS_STATUS_TABS[0].filters,
filteredByStatus: ALERTS_STATUS_TABS[0].status,
alerts: {},
alertsCount: {},
sortBy: 'startedAt',

18
db/post_migrate/20220425121410_add_temporary_index_for_backfill_integrations_enable_ssl_verification.rb Normal file
View file

@ -0,0 +1,18 @@
# frozen_string_literal: true
class AddTemporaryIndexForBackfillIntegrationsEnableSslVerification < Gitlab::Database::Migration[2.0]
disable_ddl_transaction!
INDEX_NAME = 'tmp_index_integrations_on_id_where_type_droneci_or_teamcity'
INDEX_CONDITION = "type_new IN ('Integrations::DroneCi', 'Integrations::Teamcity') " \
"AND encrypted_properties IS NOT NULL"
def up
# this index is used in 20220209121435_backfill_integrations_enable_ssl_verification
add_concurrent_index :integrations, :id, where: INDEX_CONDITION, name: INDEX_NAME
end
def down
remove_concurrent_index_by_name :integrations, INDEX_NAME
end
end

57
db/post_migrate/20220425121435_backfill_integrations_enable_ssl_verification.rb Normal file
View file

@ -0,0 +1,57 @@
# frozen_string_literal: true
class BackfillIntegrationsEnableSslVerification < Gitlab::Database::Migration[2.0]
disable_ddl_transaction!
restrict_gitlab_migration gitlab_schema: :gitlab_main
MIGRATION = 'BackfillIntegrationsEnableSslVerification'
INTERVAL = 5.minutes
BATCH_SIZE = 1_000
class Integration < MigrationRecord
include EachBatch
include IgnorableColumns
self.table_name = :integrations
self.inheritance_column = :_type_disabled
ignore_column :template, remove_with: '15.0', remove_after: '2022-04-22'
ignore_column :type, remove_with: '15.0', remove_after: '2022-04-22'
ignore_column :properties, remove_with: '15.1', remove_after: '2022-05-22'
scope :affected, -> do
where(type_new: %w[Integrations::DroneCi Integrations::Teamcity]).where.not(encrypted_properties: nil)
end
attr_encrypted :properties,
mode: :per_attribute_iv,
key: Settings.attr_encrypted_db_key_base_32,
algorithm: 'aes-256-gcm',
marshal: true,
marshaler: ::Gitlab::Json,
encode: false,
encode_iv: false
# Handle assignment of props with symbol keys.
# To do this correctly, we need to call the method generated by attr_encrypted.
alias_method :attr_encrypted_props=, :properties=
private :attr_encrypted_props=
def properties=(props)
self.attr_encrypted_props = props&.with_indifferent_access&.freeze
end
end
def up
queue_background_migration_jobs_by_range_at_intervals(
Integration.affected,
MIGRATION,
INTERVAL,
batch_size: BATCH_SIZE,
track_jobs: true
)
end
def down
end
end

1
db/schema_migrations/20220425121410 Normal file
View file

@ -0,0 +1 @@
73ab85c5ef724f6aba4a964f42e586db0a198affd134ba598189629fb95989a4

1
db/schema_migrations/20220425121435 Normal file
View file

@ -0,0 +1 @@
fd00d3d8104cba09134853976cf4f1dea7abb5e1dd2c4ea8b46adc4742c71030

2
db/structure.sql
View file

@ -29742,6 +29742,8 @@ CREATE INDEX tmp_index_for_null_project_namespace_id ON projects USING btree (id
CREATE INDEX tmp_index_for_project_namespace_id_migration_on_routes ON routes USING btree (id) WHERE ((namespace_id IS NULL) AND ((source_type)::text = 'Project'::text));
CREATE INDEX tmp_index_integrations_on_id_where_type_droneci_or_teamcity ON integrations USING btree (id) WHERE ((type_new = ANY (ARRAY['Integrations::DroneCi'::text, 'Integrations::Teamcity'::text])) AND (encrypted_properties IS NOT NULL));
CREATE INDEX tmp_index_issues_on_issue_type_and_id ON issues USING btree (issue_type, id);
CREATE INDEX tmp_index_members_on_state ON members USING btree (state) WHERE (state = 2);

2
doc/administration/geo/disaster_recovery/bring_primary_back.md
View file

@ -42,7 +42,7 @@ To bring the former **primary** site up to date:
NOTE:
If you [changed the DNS records](index.md#step-4-optional-updating-the-primary-domain-dns-record)
for this site during disaster recovery procedure you may need to [block
all the writes to this site](planned_failover.md#prevent-updates-to-the-primary-node)
all the writes to this site](planned_failover.md#prevent-updates-to-the-primary-site)
during this procedure.
1. [Set up database replication](../setup/database.md). In this case, the **secondary** site

66
doc/administration/geo/disaster_recovery/planned_failover.md
View file

@ -12,10 +12,10 @@ the event of unplanned outage, but it can be used in conjunction with a planned
failover to migrate your GitLab instance between regions without extended
downtime.
As replication between Geo nodes is asynchronous, a planned failover requires
a maintenance window in which updates to the **primary** node are blocked. The
As replication between Geo sites is asynchronous, a planned failover requires
a maintenance window in which updates to the **primary** site are blocked. The
length of this window is determined by your replication capacity - once the
**secondary** node is completely synchronized with the **primary** node, the failover can occur without
**secondary** site is completely synchronized with the **primary** site, the failover can occur without
data loss.
This document assumes you already have a fully configured, working Geo setup.
@ -28,7 +28,7 @@ have a high degree of confidence in being able to perform them accurately.
## Not all data is automatically replicated
If you are using any GitLab features that Geo [doesn't support](../replication/datatypes.md#limitations-on-replicationverification),
you must make separate provisions to ensure that the **secondary** node has an
you must make separate provisions to ensure that the **secondary** site has an
up-to-date copy of any data associated with that feature. This may extend the
required scheduled maintenance period significantly.
@ -36,7 +36,7 @@ A common strategy for keeping this period as short as possible for data stored
in files is to use `rsync` to transfer the data. An initial `rsync` can be
performed ahead of the maintenance window; subsequent `rsync`s (including a
final transfer inside the maintenance window) then transfers only the
*changes* between the **primary** node and the **secondary** nodes.
*changes* between the **primary** site and the **secondary** sites.
Repository-centric strategies for using `rsync` effectively can be found in the
[moving repositories](../../operations/moving_repositories.md) documentation; these strategies can
@ -98,42 +98,42 @@ Doing so reduces both the length of the maintenance window, and the risk of data
loss as a result of a poorly executed planned failover.
In GitLab 12.4, you can optionally allow GitLab to manage replication of Object Storage for
**secondary** nodes. For more information, see [Object Storage replication](../replication/object_storage.md).
**secondary** sites. For more information, see [Object Storage replication](../replication/object_storage.md).
### Review the configuration of each **secondary** node
### Review the configuration of each **secondary** site
Database settings are automatically replicated to the **secondary** node, but the
Database settings are automatically replicated to the **secondary** site, but the
`/etc/gitlab/gitlab.rb` file must be set up manually, and differs between
nodes. If features such as Mattermost, OAuth or LDAP integration are enabled
on the **primary** node but not the **secondary** node, they are lost during failover.
sites. If features such as Mattermost, OAuth or LDAP integration are enabled
on the **primary** site but not the **secondary** site, they are lost during failover.
Review the `/etc/gitlab/gitlab.rb` file for both nodes and ensure the **secondary** node
supports everything the **primary** node does **before** scheduling a planned failover.
Review the `/etc/gitlab/gitlab.rb` file for both sites and ensure the **secondary** site
supports everything the **primary** site does **before** scheduling a planned failover.
### Run system checks
Run the following on both **primary** and **secondary** nodes:
Run the following on both **primary** and **secondary** sites:
```shell
gitlab-rake gitlab:check
gitlab-rake gitlab:geo:check
```
If any failures are reported on either node, they should be resolved **before**
If any failures are reported on either site, they should be resolved **before**
scheduling a planned failover.
### Check that secrets match between nodes
### Check that secrets match between sites
The SSH host keys and `/etc/gitlab/gitlab-secrets.json` files should be
identical on all nodes. Check this by running the following on all nodes and
identical on all sites. Check this by running the following on all sites and
comparing the output:
```shell
sudo sha256sum /etc/ssh/ssh_host* /etc/gitlab/gitlab-secrets.json
```
If any files differ, replace the content on the **secondary** node with the
content from the **primary** node.
If any files differ, replace the content on the **secondary** site with the
content from the **primary** site.
### Ensure Geo replication is up-to-date
@ -141,13 +141,13 @@ The maintenance window won't end until Geo replication and verification is
completely finished. To keep the window as short as possible, you should
ensure these processes are close to 100% as possible during active use.
On the **secondary** node:
On the **secondary** site:
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Geo > Sites**.
Replicated objects (shown in green) should be close to 100%,
and there should be no failures (shown in red). If a large proportion of
objects aren't yet replicated (shown in gray), consider giving the node more
objects aren't yet replicated (shown in gray), consider giving the site more
time to complete
![Replication status](../replication/img/geo_dashboard_v14_0.png)
@ -160,7 +160,7 @@ You can use the [Geo status API](../../../api/geo_nodes.md#retrieve-project-sync
the reasons for failure.
A common cause of replication failures is the data being missing on the
**primary** node - you can resolve these failures by restoring the data from backup,
**primary** site - you can resolve these failures by restoring the data from backup,
or removing references to the missing data.
### Verify the integrity of replicated data
@ -169,7 +169,7 @@ This [content was moved to another location](background_verification.md).
### Notify users of scheduled maintenance
On the **primary** node:
On the **primary** site:
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Messages**.
@ -178,12 +178,12 @@ On the **primary** node:
takes to finish syncing.
1. Select **Add broadcast message**.
## Prevent updates to the **primary** node
## Prevent updates to the **primary** site
To ensure that all data is replicated to a secondary site, updates (write requests) need to
be disabled on the **primary** site:
1. Enable [maintenance mode](../../maintenance_mode/index.md) on the **primary** node.
1. Enable [maintenance mode](../../maintenance_mode/index.md) on the **primary** site.
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Monitoring > Background Jobs**.
1. On the Sidekiq dashboard, select **Cron**.
@ -199,7 +199,7 @@ GitLab 13.9 through GitLab 14.3 are affected by a bug in which the Geo secondary
1. If you are manually replicating any data not managed by Geo, trigger the
final replication process now.
1. On the **primary** node:
1. On the **primary** site:
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Monitoring > Background Jobs**.
1. On the Sidekiq dashboard, select **Queues**, and wait for all queues except
@ -207,14 +207,14 @@ GitLab 13.9 through GitLab 14.3 are affected by a bug in which the Geo secondary
These queues contain work that has been submitted by your users; failing over
before it is completed, causes the work to be lost.
1. On the left sidebar, select **Geo > Sites** and wait for the
following conditions to be true of the **secondary** node you are failing over to:
following conditions to be true of the **secondary** site you are failing over to:
- All replication meters reach 100% replicated, 0% failures.
- All verification meters reach 100% verified, 0% failures.
- Database replication lag is 0ms.
- The Geo log cursor is up to date (0 events behind).
1. On the **secondary** node:
1. On the **secondary** site:
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Monitoring > Background Jobs**.
1. On the Sidekiq dashboard, select **Queues**, and wait for all the `geo`
@ -222,16 +222,16 @@ GitLab 13.9 through GitLab 14.3 are affected by a bug in which the Geo secondary
1. [Run an integrity check](../../raketasks/check.md) to verify the integrity
of CI artifacts, LFS objects, and uploads in file storage.
At this point, your **secondary** node contains an up-to-date copy of everything the
**primary** node has, meaning nothing was lost when you fail over.
At this point, your **secondary** site contains an up-to-date copy of everything the
**primary** site has, meaning nothing was lost when you fail over.
## Promote the **secondary** node
## Promote the **secondary** site
After the replication is finished, [promote the **secondary** node to a **primary** node](index.md). This process causes a brief outage on the **secondary** node, and users may need to log in again. If you follow the steps correctly, the old primary Geo site should still be disabled and user traffic should go to the newly-promoted site instead.
After the replication is finished, [promote the **secondary** site to a **primary** site](index.md). This process causes a brief outage on the **secondary** site, and users may need to log in again. If you follow the steps correctly, the old primary Geo site should still be disabled and user traffic should go to the newly-promoted site instead.
When the promotion is completed, the maintenance window is over, and your new **primary** node now
When the promotion is completed, the maintenance window is over, and your new **primary** site now
begins to diverge from the old one. If problems do arise at this point, failing
back to the old **primary** node [is possible](bring_primary_back.md), but likely to result
back to the old **primary** site [is possible](bring_primary_back.md), but likely to result
in the loss of any data uploaded to the new **primary** in the meantime.
Don't forget to remove the broadcast message after the failover is complete.

28
doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md
View file

@ -69,7 +69,7 @@ GitLab 13.9 through GitLab 14.3 are affected by a bug in which the Geo secondary
On the **secondary** site:
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Geo > Nodes** to see its status.
1. On the left sidebar, select **Geo > Sites** to see its status.
Replicated objects (shown in green) should be close to 100%,
and there should be no failures (shown in red). If a large proportion of
objects aren't yet replicated (shown in gray), consider giving the site more
@ -100,22 +100,22 @@ follow these steps to avoid unnecessary data loss:
**primary**. Your **secondary** site still needs read-only
access to the **primary** site during the maintenance window:
1. At the scheduled time, using your cloud provider or your node's firewall, block
all HTTP, HTTPS and SSH traffic to/from the **primary** node, **except** for your IP and
the **secondary** node's IP.
1. At the scheduled time, using your cloud provider or your site's firewall, block
all HTTP, HTTPS and SSH traffic to/from the **primary** site, **except** for your IP and
the **secondary** site's IP.
For instance, you can run the following commands on the **primary** node:
For instance, you can run the following commands on the **primary** site:
```shell
sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 22 -j ACCEPT
sudo iptables -A INPUT -p tcp -s <secondary_site_ip> --destination-port 22 -j ACCEPT
sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 22 -j ACCEPT
sudo iptables -A INPUT --destination-port 22 -j REJECT
sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 80 -j ACCEPT
sudo iptables -A INPUT -p tcp -s <secondary_site_ip> --destination-port 80 -j ACCEPT
sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 80 -j ACCEPT
sudo iptables -A INPUT --tcp-dport 80 -j REJECT
sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 443 -j ACCEPT
sudo iptables -A INPUT -p tcp -s <secondary_site_ip> --destination-port 443 -j ACCEPT
sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 443 -j ACCEPT
sudo iptables -A INPUT --tcp-dport 443 -j REJECT
```
@ -157,8 +157,8 @@ follow these steps to avoid unnecessary data loss:
those with `geo` in the name to drop to 0.
These queues contain work that has been submitted by your users; failing over
before it is completed, causes the work to be lost.
1. On the left sidebar, select **Geo > Nodes** and wait for the
following conditions to be true of the **secondary** node you are failing over to:
1. On the left sidebar, select **Geo > Sites** and wait for the
following conditions to be true of the **secondary** site you are failing over to:
- All replication meters reach 100% replicated, 0% failures.
- All verification meters reach 100% verified, 0% failures.
@ -230,13 +230,13 @@ follow these steps to avoid unnecessary data loss:
1. SSH to every Sidekiq, PostgresSQL, and Gitaly node in the **secondary** site and run one of the following commands:
- To promote the secondary node to primary:
- To promote the secondary site to primary:
```shell
sudo gitlab-ctl geo promote
```
- To promote the secondary node to primary **without any further confirmation**:
- To promote the secondary site to primary **without any further confirmation**:
```shell
sudo gitlab-ctl geo promote --force
@ -244,13 +244,13 @@ follow these steps to avoid unnecessary data loss:
1. SSH into each Rails node on your **secondary** site and run one of the following commands:
- To promote the secondary node to primary:
- To promote the secondary site to primary:
```shell
sudo gitlab-ctl geo promote
```
- To promote the secondary node to primary **without any further confirmation**:
- To promote the secondary site to primary **without any further confirmation**:
```shell
sudo gitlab-ctl geo promote --force

4
doc/administration/instance_limits.md
View file

@ -876,6 +876,10 @@ See the limits in the [Add a design to an issue](../user/project/issues/design_m
## Push Event Limits
### Max push size
The maximum allowed [push size](../user/admin_area/settings/account_and_limit_settings.md#max-push-size) is set to 5 GB.
### Webhooks and Project Services
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31009) in GitLab 12.4.

4
doc/administration/maintenance_mode/index.md
View file

@ -166,7 +166,7 @@ Package Registry allows you to install but not publish packages.
Background jobs (cron jobs, Sidekiq) continue running as is, because background jobs are not automatically disabled.
[During a planned Geo failover](../geo/disaster_recovery/planned_failover.md#prevent-updates-to-the-primary-node),
[During a planned Geo failover](../geo/disaster_recovery/planned_failover.md#prevent-updates-to-the-primary-site),
it is recommended that you disable all cron jobs except for those related to Geo.
To monitor queues and disable jobs:
@ -210,4 +210,4 @@ For the same reason we don't automatically block background jobs when Maintenanc
The resulting database writes are acceptable. Here, the trade-off is between more service degradation and the completion of replication.
However, during a planned failover, we [ask users to turn off cron jobs that are not related to Geo, manually](../geo/disaster_recovery/planned_failover.md#prevent-updates-to-the-primary-node). In the absence of new database writes and non-Geo cron jobs, new background jobs would either not be created at all or be minimal.
However, during a planned failover, we [ask users to turn off cron jobs that are not related to Geo, manually](../geo/disaster_recovery/planned_failover.md#prevent-updates-to-the-primary-site). In the absence of new database writes and non-Geo cron jobs, new background jobs would either not be created at all or be minimal.

2
doc/administration/troubleshooting/test_environments.md
View file

@ -52,6 +52,8 @@ gitlab/gitlab-ee:11.5.3-ee.0
#### SAML for Authentication
In the following examples, when replacing `<GITLAB_IP_OR_DOMAIN>` and `<SAML_IP_OR_DOMAIN>` it is important to prepend your IP or domain name, with the protocol (`http://` or `https://`) being used.
We can use the [`test-saml-idp` Docker image](https://hub.docker.com/r/jamedjo/test-saml-idp)
to do the work for us:

10
doc/development/documentation/versions.md
View file

@ -124,8 +124,11 @@ To remove a page:
1. Leave the page title. Remove all other content, including the version history bullets and the word `WARNING:`.
1. After the title, change `(deprecated)` to `(removed)`.
1. Add `remove_date` in the YAML metadata. Set the value to a date three months after
the release when the feature was removed. For example:
1. Update the YAML metadata:
- For `remove_date`, set the value to a date three months after
the release when the feature was removed.
- For the `redirect_to`, set a path to a file that makes sense. If no obvious
page exists, use the docs home page.
```markdown
---
@ -133,6 +136,7 @@ To remove a page:
group: Global Search
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
remove_date: '2022-08-02'
redirect_to: '../newpath/to/file/index.md'
---
# Title (removed) **(ULTIMATE SELF)**
@ -158,7 +162,7 @@ To remove a topic:
For the `remove_date`, set a date three months after the release where it was removed.
```markdown
<!--- The following content will be removed on remove_date: '2023-08-22' -->
<!--- start_remove The following content will be removed on remove_date: '2023-08-22' -->
## Title (removed) **(ULTIMATE SELF)**

4
doc/user/admin_area/settings/account_and_limit_settings.md
View file

@ -51,12 +51,14 @@ For GitLab.com repository size limits, read [accounts and limit settings](../../
## Max push size
You can change the maximum push size for your repository:
You can change the maximum push size for your instance:
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > General**, then expand **Account and limit**.
1. Increase or decrease by changing the value in **Maximum push size (MB)**.
For GitLab.com appliation limits, read [GitLab application limits](../../../administration/instance_limits.md#max-push-size).
NOTE:
When you [add files to a repository](../../project/repository/web_editor.md#create-a-file)
through the web UI, the maximum **attachment** size is the limiting factor,

104
lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb Normal file
View file

@ -0,0 +1,104 @@
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# Enable SSL verification for CI integrations with known-good hostnames.
class BackfillIntegrationsEnableSslVerification
INTEGRATIONS = {
# This matches the logic in `Integrations::DroneCi#url_is_saas?`
# - https://gitlab.com/gitlab-org/gitlab/blob/65b7fc1ad1ad33247890324e9a3396993b7718a1/app/models/integrations/drone_ci.rb#L122-127
# - https://docs.drone.io/pipeline/environment/reference/drone-system-hostname/
'Integrations::DroneCi' => [
:drone_url,
/\Acloud\.drone\.io\z/i.freeze
],
# This matches the logic in `Integrations::Teamcity#url_is_saas?`
# - https://gitlab.com/gitlab-org/gitlab/blob/65b7fc1ad1ad33247890324e9a3396993b7718a1/app/models/integrations/teamcity.rb#L117-122
# - https://www.jetbrains.com/help/teamcity/cloud/migrate-from-teamcity-on-premises-to-teamcity-cloud.html#Migration+Process
'Integrations::Teamcity' => [
:teamcity_url,
/\A[^\.]+\.teamcity\.com\z/i.freeze
]
# Other CI integrations which don't seem to have a SaaS offering:
# - Atlassian Bamboo (the SaaS offering is Bitbucket Pipelines)
# - Jenkins (self-hosted only)
# - MockCi (development only)
}.freeze
# Define the `Integration` model
class Integration < ActiveRecord::Base
include IgnorableColumns
self.table_name = :integrations
self.inheritance_column = :_type_disabled
ignore_column :template, remove_with: '15.0', remove_after: '2022-04-22'
ignore_column :type, remove_with: '15.0', remove_after: '2022-04-22'
ignore_column :properties, remove_with: '15.1', remove_after: '2022-05-22'
scope :affected, -> { where(type_new: INTEGRATIONS.keys).where.not(encrypted_properties: nil) }
attr_encrypted :properties,
mode: :per_attribute_iv,
key: Settings.attr_encrypted_db_key_base_32,
algorithm: 'aes-256-gcm',
marshal: true,
marshaler: ::Gitlab::Json,
encode: false,
encode_iv: false
# Handle assignment of props with symbol keys.
# To do this correctly, we need to call the method generated by attr_encrypted.
alias_method :attr_encrypted_props=, :properties=
private :attr_encrypted_props=
def properties=(props)
self.attr_encrypted_props = props&.with_indifferent_access&.freeze
end
end
def perform(start_id, stop_id)
integration_ids = Integration
.affected
.where(id: (start_id..stop_id))
.pluck(:id)
integration_ids.each do |id|
Integration.transaction do
integration = Integration.lock.find(id)
process_integration(integration)
end
end
mark_job_as_succeeded(start_id, stop_id)
end
private
def process_integration(integration)
url_field, known_hostnames = INTEGRATIONS.fetch(integration.type_new)
url = integration.properties[url_field.to_s] if integration.properties.present?
return unless url.present?
parsed_url = Addressable::URI.parse(url)
return unless parsed_url.scheme == 'https' && parsed_url.hostname =~ known_hostnames
integration.properties = integration.properties.merge('enable_ssl_verification' => true)
integration.save!(touch: false)
rescue Addressable::URI::InvalidURIError, ActiveRecord::RecordInvalid
# Don't change the configuration if the record is invalid, in this case
# they will just keep having SSL verification disabled.
end
def mark_job_as_succeeded(*arguments)
Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
self.class.name.demodulize,
arguments
)
end
end
end
end

1
qa/.rspec
View file

@ -3,4 +3,3 @@
--format documentation
--default-path qa/specs
--require spec_helper
--tag ~orchestrated

73
spec/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification_spec.rb Normal file
View file

@ -0,0 +1,73 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillIntegrationsEnableSslVerification, schema: 20220425121410 do
let(:migration) { described_class.new }
let(:integrations) { described_class::Integration }
before do
integrations.create!(id: 1, type_new: 'Integrations::Bamboo') # unaffected integration
integrations.create!(id: 2, type_new: 'Integrations::DroneCi') # no properties
integrations.create!(id: 3, type_new: 'Integrations::DroneCi',
properties: {}) # no URL
integrations.create!(id: 4, type_new: 'Integrations::DroneCi',
properties: { 'drone_url' => '' }) # blank URL
integrations.create!(id: 5, type_new: 'Integrations::DroneCi',
properties: { 'drone_url' => 'https://example.com:foo' }) # invalid URL
integrations.create!(id: 6, type_new: 'Integrations::DroneCi',
properties: { 'drone_url' => 'https://example.com' }) # unknown URL
integrations.create!(id: 7, type_new: 'Integrations::DroneCi',
properties: { 'drone_url' => 'http://cloud.drone.io' }) # no HTTPS
integrations.create!(id: 8, type_new: 'Integrations::DroneCi',
properties: { 'drone_url' => 'https://cloud.drone.io' }) # known URL
integrations.create!(id: 9, type_new: 'Integrations::Teamcity',
properties: { 'teamcity_url' => 'https://example.com' }) # unknown URL
integrations.create!(id: 10, type_new: 'Integrations::Teamcity',
properties: { 'teamcity_url' => 'https://foo.bar.teamcity.com' }) # unknown URL
integrations.create!(id: 11, type_new: 'Integrations::Teamcity',
properties: { 'teamcity_url' => 'https://teamcity.com' }) # unknown URL
integrations.create!(id: 12, type_new: 'Integrations::Teamcity',
properties: { 'teamcity_url' => 'https://customer.teamcity.com' }) # known URL
end
def properties(id)
integrations.find(id).properties
end
it 'enables SSL verification for known-good hostnames', :aggregate_failures do
migration.perform(1, 12)
# Bamboo
expect(properties(1)).to be_nil
# DroneCi
expect(properties(2)).to be_nil
expect(properties(3)).not_to include('enable_ssl_verification')
expect(properties(4)).not_to include('enable_ssl_verification')
expect(properties(5)).not_to include('enable_ssl_verification')
expect(properties(6)).not_to include('enable_ssl_verification')
expect(properties(7)).not_to include('enable_ssl_verification')
expect(properties(8)).to include('enable_ssl_verification' => true)
# Teamcity
expect(properties(9)).not_to include('enable_ssl_verification')
expect(properties(10)).not_to include('enable_ssl_verification')
expect(properties(11)).not_to include('enable_ssl_verification')
expect(properties(12)).to include('enable_ssl_verification' => true)
end
it 'only updates records within the given ID range', :aggregate_failures do
migration.perform(1, 8)
expect(properties(8)).to include('enable_ssl_verification' => true)
expect(properties(12)).not_to include('enable_ssl_verification')
end
it 'marks the job as succeeded' do
expect(Gitlab::Database::BackgroundMigrationJob).to receive(:mark_all_as_succeeded)
.with('BackfillIntegrationsEnableSslVerification', [1, 10])
migration.perform(1, 10)
end
end

32
spec/migrations/backfill_integrations_enable_ssl_verification_spec.rb Normal file
View file

@ -0,0 +1,32 @@
# frozen_string_literal: true
require 'spec_helper'
require_migration!
RSpec.describe BackfillIntegrationsEnableSslVerification do
let_it_be(:migration) { described_class::MIGRATION }
let_it_be(:integrations) { described_class::Integration }
before do
stub_const("#{described_class.name}::BATCH_SIZE", 2)
integrations.create!(id: 1, type_new: 'Integrations::DroneCi')
integrations.create!(id: 2, type_new: 'Integrations::DroneCi', properties: {})
integrations.create!(id: 3, type_new: 'Integrations::Bamboo', properties: {})
integrations.create!(id: 4, type_new: 'Integrations::Teamcity', properties: {})
integrations.create!(id: 5, type_new: 'Integrations::DroneCi', properties: {})
integrations.create!(id: 6, type_new: 'Integrations::Teamcity', properties: {})
end
describe '#up' do
it 'schedules background jobs for each batch of integrations', :freeze_time do
Sidekiq::Testing.fake! do
migrate!
expect(BackgroundMigrationWorker.jobs.size).to eq(2)
expect(migration).to be_scheduled_delayed_migration(5.minutes, 2, 4)
expect(migration).to be_scheduled_delayed_migration(10.minutes, 5, 6)
end
end
end
end