From a2c2225d61e60c5fe534e9455b8f60bf6ae22423 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 1 Nov 2022 03:09:22 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- app/models/ci/build_metadata.rb | 6 +- config/initializers/types.rb | 3 + .../auth/ldap/ldap-troubleshooting.md | 6 +- .../geo/disaster_recovery/planned_failover.md | 2 +- doc/administration/geo/index.md | 2 +- .../geo/replication/configuration.md | 5 +- .../geo/replication/remove_geo_site.md | 5 +- doc/administration/geo/setup/database.md | 2 +- .../geo/setup/external_database.md | 2 +- doc/administration/git_protocol.md | 2 +- doc/administration/gitaly/configure_gitaly.md | 48 +++++++++++++- doc/administration/gitaly/praefect.md | 2 +- .../logs/tracing_correlation_id.md | 2 +- doc/administration/maintenance_mode/index.md | 4 +- .../gitlab_self_monitoring_project/index.md | 4 +- doc/administration/nfs.md | 2 +- .../sec/analyzer_development_guide.md | 6 ++ lib/gitlab/database/type/symbolized_jsonb.rb | 28 ++++++++ lib/serializers/symbolized_json.rb | 18 ------ .../database/type/symbolized_jsonb_spec.rb | 64 +++++++++++++++++++ spec/lib/serializers/symbolized_json_spec.rb | 42 ------------ spec/models/ci/build_metadata_spec.rb | 10 +++ 22 files changed, 181 insertions(+), 84 deletions(-) create mode 100644 config/initializers/types.rb create mode 100644 lib/gitlab/database/type/symbolized_jsonb.rb delete mode 100644 lib/serializers/symbolized_json.rb create mode 100644 spec/lib/gitlab/database/type/symbolized_jsonb_spec.rb delete mode 100644 spec/lib/serializers/symbolized_json_spec.rb diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 879ff0321e3..2f28509f812 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -30,9 +30,9 @@ module Ci validates :id_tokens, json_schema: { filename: 'build_metadata_id_tokens' } validates :secrets, json_schema: { filename: 'build_metadata_secrets' } - serialize :config_options, Serializers::SymbolizedJson # rubocop:disable Cop/ActiveRecordSerialize - serialize :config_variables, Serializers::SymbolizedJson # rubocop:disable Cop/ActiveRecordSerialize - serialize :runtime_runner_features, Serializers::SymbolizedJson # rubocop:disable Cop/ActiveRecordSerialize + attribute :config_options, :sym_jsonb + attribute :config_variables, :sym_jsonb + attribute :runtime_runner_features, :sym_jsonb chronic_duration_attr_reader :timeout_human_readable, :timeout diff --git a/config/initializers/types.rb b/config/initializers/types.rb new file mode 100644 index 00000000000..4a20e257469 --- /dev/null +++ b/config/initializers/types.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +ActiveRecord::Type.register(:sym_jsonb, Gitlab::Database::Type::SymbolizedJsonb) diff --git a/doc/administration/auth/ldap/ldap-troubleshooting.md b/doc/administration/auth/ldap/ldap-troubleshooting.md index b0965818d76..6348dd7d4a4 100644 --- a/doc/administration/auth/ldap/ldap-troubleshooting.md +++ b/doc/administration/auth/ldap/ldap-troubleshooting.md @@ -196,7 +196,7 @@ same user) has the email `email@example.com` set as a secondary email, which is throwing this error. We can check where this conflicting email address is coming from using the -[rails console](#rails-console). Once in the console, run the following: +[rails console](#rails-console). In the console, run the following: ```ruby # This searches for an email among the primary AND secondary emails @@ -546,7 +546,7 @@ this entry, it could be due to a mismatched DN stored in GitLab. See ```shell User with DN `uid=john0,ou=people,dc=example,dc=com` should have access to 'my_group' group but there is no user in GitLab with that -identity. Membership will be updated once the user signs in for +identity. Membership will be updated when the user signs in for the first time. ``` @@ -556,7 +556,7 @@ Finally, the following entry says syncing has finished for this group: Finished syncing all providers for 'my_group' group ``` -Once all the configured group links have been synchronized, GitLab looks +When all the configured group links have been synchronized, GitLab looks for any Administrators or External users to sync: ```shell diff --git a/doc/administration/geo/disaster_recovery/planned_failover.md b/doc/administration/geo/disaster_recovery/planned_failover.md index 08b1a15522d..80707afacca 100644 --- a/doc/administration/geo/disaster_recovery/planned_failover.md +++ b/doc/administration/geo/disaster_recovery/planned_failover.md @@ -14,7 +14,7 @@ downtime. As replication between Geo sites is asynchronous, a planned failover requires a maintenance window in which updates to the **primary** site are blocked. The -length of this window is determined by your replication capacity - once the +length of this window is determined by your replication capacity - when the **secondary** site is completely synchronized with the **primary** site, the failover can occur without data loss. diff --git a/doc/administration/geo/index.md b/doc/administration/geo/index.md index df54ea402bd..e6fc7567ecd 100644 --- a/doc/administration/geo/index.md +++ b/doc/administration/geo/index.md @@ -306,7 +306,7 @@ For an example of how to set up a location-aware Git remote URL with AWS Route53 ### Backfill -Once a **secondary** site is set up, it starts replicating missing data from +When a **secondary** site is set up, it starts replicating missing data from the **primary** site in a process known as **backfill**. You can monitor the synchronization process on each Geo site from the **primary** site's **Geo Nodes** dashboard in your browser. diff --git a/doc/administration/geo/replication/configuration.md b/doc/administration/geo/replication/configuration.md index fa74f16cdc8..ee92d05c81e 100644 --- a/doc/administration/geo/replication/configuration.md +++ b/doc/administration/geo/replication/configuration.md @@ -239,8 +239,9 @@ keys must be manually replicated to the **secondary** site. If any of the checks fail, check the [troubleshooting documentation](troubleshooting.md). -Once added to the Geo administration page and restarted, the **secondary** site automatically starts -replicating missing data from the **primary** site in a process known as **backfill**. +After the **secondary** site is added to the Geo administration page and restarted, +the site automatically starts replicating missing data from the **primary** site +in a process known as **backfill**. Meanwhile, the **primary** site starts to notify each **secondary** site of any changes, so that the **secondary** site can act on those notifications immediately. diff --git a/doc/administration/geo/replication/remove_geo_site.md b/doc/administration/geo/replication/remove_geo_site.md index 62b1d9fdf7b..4b9f31dc08c 100644 --- a/doc/administration/geo/replication/remove_geo_site.md +++ b/doc/administration/geo/replication/remove_geo_site.md @@ -14,7 +14,8 @@ type: howto 1. Select the **Remove** button for the **secondary** site you want to remove. 1. Confirm by selecting **Remove** when the prompt appears. -Once removed from the Geo administration page, you must stop and uninstall the **secondary** site. For each node on your secondary Geo site: +After the **secondary** site is removed from the Geo administration page, you must +stop and uninstall this site. For each node on your secondary Geo site: 1. Stop GitLab: @@ -35,7 +36,7 @@ Once removed from the Geo administration page, you must stop and uninstall the * sudo rpm --erase gitlab-ee ``` -Once GitLab has been uninstalled from each node on the **secondary** site, the replication slot must be dropped from the **primary** site's database as follows: +When GitLab has been uninstalled from each node on the **secondary** site, the replication slot must be dropped from the **primary** site's database as follows: 1. On the **primary** site's database node, start a PostgreSQL console session: diff --git a/doc/administration/geo/setup/database.md b/doc/administration/geo/setup/database.md index 3a87d128792..fdb1c159709 100644 --- a/doc/administration/geo/setup/database.md +++ b/doc/administration/geo/setup/database.md @@ -619,7 +619,7 @@ If you still haven't [migrated from repmgr to Patroni](#migrating-from-repmgr-to 1. Before migrating, we recommend that there is no replication lag between the **primary** and **secondary** sites and that replication is paused. In GitLab 13.2 and later, you can pause and resume replication with `gitlab-ctl geo-replication-pause` and `gitlab-ctl geo-replication-resume` on a Geo secondary database node. 1. Follow the [instructions to migrate repmgr to Patroni](../../postgresql/replication_and_failover.md#switching-from-repmgr-to-patroni). When configuring Patroni on each **primary** site database node, add `patroni['replication_slots'] = { '' => 'physical' }` to `gitlab.rb` where `` is the name of the replication slot for your **secondary** site. This ensures that Patroni recognizes the replication slot as permanent and not drop it upon restarting. -1. If database replication to the **secondary** site was paused before migration, resume replication once Patroni is confirmed working on the **primary** site. +1. If database replication to the **secondary** site was paused before migration, resume replication after Patroni is confirmed working on the **primary** site. ### Migrating a single PostgreSQL node to Patroni diff --git a/doc/administration/geo/setup/external_database.md b/doc/administration/geo/setup/external_database.md index ab73e5c65f5..0fefc11f078 100644 --- a/doc/administration/geo/setup/external_database.md +++ b/doc/administration/geo/setup/external_database.md @@ -78,7 +78,7 @@ cloud providers: - Azure Database for PostgreSQL - [Create and manage read replicas in Azure Database for PostgreSQL](https://learn.microsoft.com/en-us/azure/postgresql/single-server/how-to-read-replicas-portal) - Google Cloud SQL - [Creating read replicas](https://cloud.google.com/sql/docs/postgres/replication/create-replica) -Once your read-only replica is set up, you can skip to [configure your secondary site](#configure-secondary-site-to-use-the-external-read-replica) +When your read-only replica is set up, you can skip to [configure your secondary site](#configure-secondary-site-to-use-the-external-read-replica) #### Manually configure the primary database for replication diff --git a/doc/administration/git_protocol.md b/doc/administration/git_protocol.md index 7a7f8229f4b..349a92de51e 100644 --- a/doc/administration/git_protocol.md +++ b/doc/administration/git_protocol.md @@ -36,7 +36,7 @@ the SSH configuration of your server manually by adding this line to the `/etc/s AcceptEnv GIT_PROTOCOL ``` -Once configured, restart the SSH daemon for the change to take effect: +When you have configured the SSH daemon, restart it for the change to take effect: ```shell # CentOS 6 / RHEL 6 diff --git a/doc/administration/gitaly/configure_gitaly.md b/doc/administration/gitaly/configure_gitaly.md index b59bce37b98..ec3bb47dcb3 100644 --- a/doc/administration/gitaly/configure_gitaly.md +++ b/doc/administration/gitaly/configure_gitaly.md @@ -1309,18 +1309,38 @@ following keys (in this example, to disable the `hasDotgit` consistency check): - In [GitLab 15.3](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6800) and later: ```ruby + ignored_blobs = "/etc/gitlab/instance_wide_ignored_git_blobs.txt" + gitaly['gitconfig'] = [ + + # Populate a file with one unabbreviated SHA-1 per line. + # See https://git-scm.com/docs/git-config#Documentation/git-config.txt-fsckskipList + { key: "fsck.skipList", value: ignored_blobs }, + { key: "fetch.fsck.skipList", value: ignored_blobs }, + { key: "receive.fsck.skipList", value: ignored_blobs }, + { key: "fsck.hasDotgit", value: "ignore" }, { key: "fetch.fsck.hasDotgit", value: "ignore" }, - { key: "receive.fsck.hasDotgit", value: "ignore "}, + { key: "receive.fsck.hasDotgit", value: "ignore" }, + { key: "fsck.missingSpaceBeforeEmail", value: "ignore" }, ] ``` - In GitLab 15.2 and earlier (legacy method): ```ruby - ignored_git_errors = ["hasDotgit = ignore"] + ignored_git_errors = [ + "hasDotgit = ignore", + "missingSpaceBeforeEmail = ignore", + ] omnibus_gitconfig['system'] = { + + # Populate a file with one unabbreviated SHA-1 per line. + # See https://git-scm.com/docs/git-config#Documentation/git-config.txt-fsckskipList + "fsck.skipList" => ignored_blobs + "fetch.fsck.skipList" => ignored_blobs, + "receive.fsck.skipList" => ignored_blobs, + "fsck" => ignored_git_errors, "fetch.fsck" => ignored_git_errors, "receive.fsck" => ignored_git_errors, @@ -1342,6 +1362,30 @@ value = "ignore" [[git.config]] key = "receive.fsck.hasDotgit" value = "ignore" + +[[git.config]] +key = "fsck.missingSpaceBeforeEmail" +value = "ignore" + +[[git.config]] +key = "fetch.fsck.missingSpaceBeforeEmail" +value = "ignore" + +[[git.config]] +key = "receive.fsck.missingSpaceBeforeEmail" +value = "ignore" + +[[git.config]] +key = "fsck.skipList" +value = "/etc/gitlab/instance_wide_ignored_git_blobs.txt" + +[[git.config]] +key = "fetch.fsck.skipList" +value = "/etc/gitlab/instance_wide_ignored_git_blobs.txt" + +[[git.config]] +key = "receive.fsck.skipList" +value = "/etc/gitlab/instance_wide_ignored_git_blobs.txt" ``` ## Configure commit signing for GitLab UI commits diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 66d4a740b27..45e6f89d62b 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -293,7 +293,7 @@ praefect['database_direct_dbname'] = 'praefect_production' #praefect['database_direct_sslrootcert'] = '...' ``` -Once configured, this connection is automatically used for the +When configured, this connection is automatically used for the [SQL LISTEN](https://www.postgresql.org/docs/11/sql-listen.html) feature and allows Praefect to receive notifications from PostgreSQL for cache invalidation. diff --git a/doc/administration/logs/tracing_correlation_id.md b/doc/administration/logs/tracing_correlation_id.md index f651455a088..906dcd3cea9 100644 --- a/doc/administration/logs/tracing_correlation_id.md +++ b/doc/administration/logs/tracing_correlation_id.md @@ -103,7 +103,7 @@ sudo gitlab-ctl tail gitlab-rails/production_json.log | grep '"username":"bob"' ## Searching your logs for the correlation ID -Once you have the correlation ID you can start searching for relevant log +When you have the correlation ID you can start searching for relevant log entries. You can filter the lines by the correlation ID itself. Combining a `find` and `grep` should be sufficient to find the entries you are looking for. diff --git a/doc/administration/maintenance_mode/index.md b/doc/administration/maintenance_mode/index.md index d7e7f6d79bd..9adb8ce2cd9 100644 --- a/doc/administration/maintenance_mode/index.md +++ b/doc/administration/maintenance_mode/index.md @@ -10,7 +10,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w Maintenance Mode allows administrators to reduce write operations to a minimum while maintenance tasks are performed. The main goal is to block all external actions that change the internal state, including the PostgreSQL database, but especially files, Git repositories, and Container repositories. -Once Maintenance Mode is enabled, in-progress actions finish relatively quickly since no new actions are coming in, and internal state changes are minimal. +When Maintenance Mode is enabled, in-progress actions finish relatively quickly since no new actions are coming in, and internal state changes are minimal. In that state, various maintenance tasks are easier, and services can be stopped completely or be further degraded for a much shorter period of time than might otherwise be needed. For example, stopping cron jobs and draining queues should be fairly quick. @@ -150,7 +150,7 @@ is turned off. Deployments don't go through because pipelines are unfinished. -It is recommended to disable auto deploys during Maintenance Mode, and enable them once it is disabled. +It is recommended to disable auto deploys during Maintenance Mode, and enable them when it is disabled. #### Terraform integration diff --git a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md index e1625ef8231..35dc64a0594 100644 --- a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md +++ b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md @@ -85,7 +85,7 @@ you [configure it manually](../../../user/project/integrations/prometheus.md#man You can [add a Prometheus integration](../../../operations/incident_management/integrations.md) to GitLab to receive notifications of any alerts. -Once the integration is setup, you can +When the integration is set up, you can [take action on incoming alerts](../../../operations/metrics/alerts.md#trigger-actions-from-alerts). ## Add custom metrics to the self-monitoring project @@ -118,4 +118,4 @@ If this returns true, the first administrator user is an external user. If you face this issue, you can temporarily [make the administrator user a non-external user](../../../user/permissions.md#external-users) and then try to create the project. -Once the project is created, the administrator user can be changed back to an external user. +After the project is created, the administrator user can be changed back to an external user. diff --git a/doc/administration/nfs.md b/doc/administration/nfs.md index 402d18fc9a7..85f35a1b188 100644 --- a/doc/administration/nfs.md +++ b/doc/administration/nfs.md @@ -352,7 +352,7 @@ are empty before attempting a restore. Read more about the ## Testing NFS -Once you've set up the NFS server and client, you can verify NFS is configured correctly +When you've set up the NFS server and client, you can verify NFS is configured correctly by testing the following commands: ```shell diff --git a/doc/development/sec/analyzer_development_guide.md b/doc/development/sec/analyzer_development_guide.md index a35bc2b7237..002c2fa3581 100644 --- a/doc/development/sec/analyzer_development_guide.md +++ b/doc/development/sec/analyzer_development_guide.md @@ -118,6 +118,12 @@ To use Docker with `replace` in the `go.mod` file: 1. Update the `replace` statement to make sure it matches the destination of the `COPY` statement in the step above: `replace gitlab.com/gitlab-org/security-products/analyzers/command/v3 => /command` +## Analyzer scripts + +The [analyzer-scripts](https://gitlab.com/gitlab-org/secure/tools/analyzer-scripts) repository contains scripts that can be used to interact with most analyzers. They enable you to build, run, and debug analyzers in a GitLab CI-like environment, and are particularly useful for locally validating changes to an analyzer. + +For more information, refer to the [project README](https://gitlab.com/gitlab-org/secure/tools/analyzer-scripts/-/blob/master/README.md). + ## Versioning and release process Analyzers are independent projects that follow their own versioning. `Patch` version bumps tend to correspond to a `Minor` version bump of the underlying tools (i.e. [`bandit`](https://wiki.openstack.org/wiki/Security/Projects/Bandit)), allowing us greater flexibility in reserving `Minor` bumps for more significant changes to our scanners. In case of breaking changes imposed by the wrapped scanner, creating a new analyzer on a separate repository must be considered. diff --git a/lib/gitlab/database/type/symbolized_jsonb.rb b/lib/gitlab/database/type/symbolized_jsonb.rb new file mode 100644 index 00000000000..5bec738ec9c --- /dev/null +++ b/lib/gitlab/database/type/symbolized_jsonb.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Type + # Extends Rails' Jsonb data type to deserialize it into symbolized Hash. + # + # Example: + # + # class SomeModel < ApplicationRecord + # # some_model.a_field is of type `jsonb` + # attribute :a_field, :sym_jsonb + # end + class SymbolizedJsonb < ::ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Jsonb + def type + :sym_jsonb + end + + def deserialize(value) + data = super + return unless data + + ::Gitlab::Utils.deep_symbolized_access(data) + end + end + end + end +end diff --git a/lib/serializers/symbolized_json.rb b/lib/serializers/symbolized_json.rb deleted file mode 100644 index 78192ce3132..00000000000 --- a/lib/serializers/symbolized_json.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Serializers - # Make the resulting hash have deep symbolized keys - class SymbolizedJson - class << self - def dump(obj) - obj - end - - def load(data) - return if data.nil? - - Gitlab::Utils.deep_symbolized_access(data) - end - end - end -end diff --git a/spec/lib/gitlab/database/type/symbolized_jsonb_spec.rb b/spec/lib/gitlab/database/type/symbolized_jsonb_spec.rb new file mode 100644 index 00000000000..a8401667b34 --- /dev/null +++ b/spec/lib/gitlab/database/type/symbolized_jsonb_spec.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Type::SymbolizedJsonb do + let(:type) { described_class.new } + + describe '#deserialize' do + using RSpec::Parameterized::TableSyntax + + subject { type.deserialize(json) } + + where(:json, :value) do + nil | nil + '{"key":"value"}' | { key: 'value' } + '{"key":[1,2,3]}' | { key: [1, 2, 3] } + '{"key":{"subkey":"value"}}' | { key: { subkey: 'value' } } + '{"key":{"a":[{"b":"c"},{"d":"e"}]}}' | { key: { a: [{ b: 'c' }, { d: 'e' }] } } + end + + with_them do + it { is_expected.to match(value) } + end + end + + context 'when used by a model' do + let(:model) do + Class.new(ApplicationRecord) do + self.table_name = :_test_symbolized_jsonb + + attribute :options, :sym_jsonb + end + end + + let(:record) do + model.create!(name: 'test', options: { key: 'value' }) + end + + before do + ApplicationRecord.connection.execute(<<~SQL) + CREATE TABLE _test_symbolized_jsonb( + id serial NOT NULL PRIMARY KEY, + name text, + options jsonb); + SQL + + model.reset_column_information + end + + it { expect(record.options).to match({ key: 'value' }) } + + it 'ignores changes to other attributes' do + record.name = 'other test' + + expect(record.changes).to match('name' => ['test', 'other test']) + end + + it 'tracks changes to options' do + record.options = { key: 'other value' } + + expect(record.changes).to match('options' => [{ 'key' => 'value' }, { 'key' => 'other value' }]) + end + end +end diff --git a/spec/lib/serializers/symbolized_json_spec.rb b/spec/lib/serializers/symbolized_json_spec.rb deleted file mode 100644 index b9217854d9a..00000000000 --- a/spec/lib/serializers/symbolized_json_spec.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -require 'fast_spec_helper' -require 'oj' - -RSpec.describe Serializers::SymbolizedJson do - describe '.dump' do - let(:obj) { { key: "value" } } - - subject { described_class.dump(obj) } - - it 'returns a hash' do - is_expected.to eq(obj) - end - end - - describe '.load' do - let(:data_string) { '{"key":"value","variables":[{"key":"VAR1","value":"VALUE1"}]}' } - let(:data_hash) { Gitlab::Json.parse(data_string) } - - context 'when loading a hash' do - subject { described_class.load(data_hash) } - - it 'decodes a string' do - is_expected.to be_a(Hash) - end - - it 'allows to access with symbols' do - expect(subject[:key]).to eq('value') - expect(subject[:variables].first[:key]).to eq('VAR1') - end - end - - context 'when loading a nil' do - subject { described_class.load(nil) } - - it 'returns nil' do - is_expected.to be_nil - end - end - end -end diff --git a/spec/models/ci/build_metadata_spec.rb b/spec/models/ci/build_metadata_spec.rb index b4c20637ce2..e728ce0f474 100644 --- a/spec/models/ci/build_metadata_spec.rb +++ b/spec/models/ci/build_metadata_spec.rb @@ -204,4 +204,14 @@ RSpec.describe Ci::BuildMetadata do end end end + + context 'jsonb fields serialization' do + it 'changing other fields does not change config_options' do + expect { metadata.id = metadata.id }.not_to change(metadata, :changes) + end + + it 'accessing config_options does not change it' do + expect { metadata.config_options }.not_to change(metadata, :changes) + end + end end