diff --git a/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.vue b/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.vue index 9aa4a524c30..fea94a0f7c7 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.vue +++ b/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.vue @@ -9,7 +9,6 @@ import { GlTooltipDirective, GlSafeHtmlDirective, } from '@gitlab/ui'; -import mrWidgetPipelineMixin from '~/vue_merge_request_widget/mixins/mr_widget_pipeline'; import { s__, n__ } from '~/locale'; import CiIcon from '~/vue_shared/components/ci_icon.vue'; import PipelineArtifacts from '~/pipelines/components/pipelines_list/pipelines_artifacts.vue'; @@ -36,7 +35,6 @@ export default { GlTooltip: GlTooltipDirective, SafeHtml: GlSafeHtmlDirective, }, - mixins: [mrWidgetPipelineMixin], props: { pipeline: { type: Object, @@ -277,10 +275,10 @@ export default { diff --git a/app/assets/javascripts/vue_merge_request_widget/mixins/mr_widget_pipeline.js b/app/assets/javascripts/vue_merge_request_widget/mixins/mr_widget_pipeline.js deleted file mode 100644 index 7b77d7475bc..00000000000 --- a/app/assets/javascripts/vue_merge_request_widget/mixins/mr_widget_pipeline.js +++ /dev/null @@ -1,10 +0,0 @@ -export default { - computed: { - triggered() { - return []; - }, - triggeredBy() { - return []; - }, - }, -}; diff --git a/db/post_migrate/20220606054503_fix_incorrect_job_artifacts_expire_at.rb b/db/post_migrate/20220606054503_fix_incorrect_job_artifacts_expire_at.rb new file mode 100644 index 00000000000..8fea22f5579 --- /dev/null +++ b/db/post_migrate/20220606054503_fix_incorrect_job_artifacts_expire_at.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class FixIncorrectJobArtifactsExpireAt < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_ci + + MIGRATION = 'RemoveBackfilledJobArtifactsExpireAt' + BATCH_CLASS = 'RemoveBackfilledJobArtifactsExpireAtBatchingStrategy' + BATCH_SIZE = 500 + INTERVAL = 2.minutes.freeze + + def up + return if Gitlab.com? + + queue_batched_background_migration( + MIGRATION, + :ci_job_artifacts, + :id, + job_interval: INTERVAL, + batch_class_name: BATCH_CLASS, + batch_size: BATCH_SIZE + ) + end + + def down + return if Gitlab.com? + + delete_batched_background_migration(MIGRATION, :ci_job_artifacts, :id, []) + end +end diff --git a/db/post_migrate/20220606080509_add_tmp_index_job_artifacts_id_and_expire_at.rb b/db/post_migrate/20220606080509_add_tmp_index_job_artifacts_id_and_expire_at.rb new file mode 100644 index 00000000000..28346eb1a97 --- /dev/null +++ b/db/post_migrate/20220606080509_add_tmp_index_job_artifacts_id_and_expire_at.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class AddTmpIndexJobArtifactsIdAndExpireAt < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + INDEX_NAME = 'tmp_index_ci_job_artifacts_on_id_expire_at_file_type_trace' + + EXPIRE_AT_ON_22_MIDNIGHT_IN_TIMEZONE_OR_TRACE = <<~SQL + (EXTRACT(day FROM timezone('UTC', expire_at)) IN (21, 22, 23) + AND EXTRACT(minute FROM timezone('UTC', expire_at)) IN (0, 30, 45) + AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0) + OR file_type = 3 + SQL + + def up + return if Gitlab.com? + return if index_exists_by_name?(:ci_job_artifacts, INDEX_NAME) + + add_concurrent_index :ci_job_artifacts, :id, + where: EXPIRE_AT_ON_22_MIDNIGHT_IN_TIMEZONE_OR_TRACE, name: INDEX_NAME + end + + def down + return if Gitlab.com? + return unless index_exists_by_name?(:ci_job_artifacts, INDEX_NAME) + + remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME + end +end diff --git a/db/schema_migrations/20220606054503 b/db/schema_migrations/20220606054503 new file mode 100644 index 00000000000..1ec25932ece --- /dev/null +++ b/db/schema_migrations/20220606054503 @@ -0,0 +1 @@ +ed815f5e2766913ded3479c2cdc8a885ab7164ae280e309cba34394668392a2b \ No newline at end of file diff --git a/db/schema_migrations/20220606080509 b/db/schema_migrations/20220606080509 new file mode 100644 index 00000000000..7403068a8da --- /dev/null +++ b/db/schema_migrations/20220606080509 @@ -0,0 +1 @@ +3afc50d92878da71453cfb23ad29d16123e4986e3304aff62013f4655b065d38 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 54535a95be5..cfb1e97ef00 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -30638,6 +30638,8 @@ CREATE UNIQUE INDEX term_agreements_unique_index ON term_agreements USING btree CREATE INDEX tmp_index_ci_job_artifacts_on_expire_at_where_locked_unknown ON ci_job_artifacts USING btree (expire_at, job_id) WHERE ((locked = 2) AND (expire_at IS NOT NULL)); +CREATE INDEX tmp_index_ci_job_artifacts_on_id_expire_at_file_type_trace ON ci_job_artifacts USING btree (id) WHERE (((date_part('day'::text, timezone('UTC'::text, expire_at)) = ANY (ARRAY[(21)::double precision, (22)::double precision, (23)::double precision])) AND (date_part('minute'::text, timezone('UTC'::text, expire_at)) = ANY (ARRAY[(0)::double precision, (30)::double precision, (45)::double precision])) AND (date_part('second'::text, timezone('UTC'::text, expire_at)) = (0)::double precision)) OR (file_type = 3)); + CREATE INDEX tmp_index_ci_job_artifacts_on_id_where_trace_and_expire_at ON ci_job_artifacts USING btree (id) WHERE ((file_type = 3) AND (expire_at = ANY (ARRAY['2021-04-22 00:00:00+00'::timestamp with time zone, '2021-05-22 00:00:00+00'::timestamp with time zone, '2021-06-22 00:00:00+00'::timestamp with time zone, '2022-01-22 00:00:00+00'::timestamp with time zone, '2022-02-22 00:00:00+00'::timestamp with time zone, '2022-03-22 00:00:00+00'::timestamp with time zone, '2022-04-22 00:00:00+00'::timestamp with time zone]))); CREATE INDEX tmp_index_cis_vulnerability_reads_on_id ON vulnerability_reads USING btree (id) WHERE (report_type = 7); diff --git a/doc/security/img/outbound_requests_section_v12_2.png b/doc/security/img/outbound_requests_section_v12_2.png deleted file mode 100644 index 3dc99868a35..00000000000 Binary files a/doc/security/img/outbound_requests_section_v12_2.png and /dev/null differ diff --git a/doc/security/webhooks.md b/doc/security/webhooks.md index f2066ee4a42..dcd4c76ab65 100644 --- a/doc/security/webhooks.md +++ b/doc/security/webhooks.md @@ -7,22 +7,14 @@ type: concepts, reference, howto # Webhooks and insecure internal web services **(FREE SELF)** -NOTE: -On GitLab.com, the [maximum number of webhooks and their size](../user/gitlab_com/index.md#webhooks) per project, and per group, is limited. +Users with at least the Maintainer role can set up [Webhooks](../user/project/integrations/webhooks.md) that are +triggered when specific changes occur in a project. When triggered, a `POST` HTTP request is sent to a URL. A webhook is +usually configured to send data to a specific external web service, which +processes the data in an appropriate way. -If you have non-GitLab web services running on your GitLab server or within its -local network, these may be vulnerable to exploitation via Webhooks. - -With [Webhooks](../user/project/integrations/webhooks.md), you and your project -maintainers and owners can set up URLs to be triggered when specific changes -occur in your projects. Normally, these requests are sent to external web -services specifically set up for this purpose, that process the request and its -attached data in some appropriate way. - -Things get hairy, however, when a Webhook is set up with a URL that doesn't -point to an external, but to an internal service, that may do something -completely unintended when the webhook is triggered and the POST request is -sent. +However, a Webhook can be configured with a URL for an internal web service instead of an external web service. +When the Webhook is triggered, +non-GitLab web services running on your GitLab server or in its local network could be exploited. Webhook requests are made by the GitLab server itself and use a single (optional) secret token per hook for authorization (instead of a user or @@ -38,6 +30,8 @@ If a web service does not require authentication, Webhooks can be used to trigger destructive commands by getting the GitLab server to make POST requests to endpoints like `http://localhost:123/some-resource/delete`. +## Allow requests to local network + To prevent this type of exploitation from happening, starting with GitLab 10.6, all Webhook requests to the current GitLab instance server address and/or in a private network are forbidden by default. That means that all requests made @@ -48,8 +42,7 @@ This behavior can be overridden: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > Network**. -1. Expand the **Outbound requests** section: - ![Outbound requests Admin Area settings](img/outbound_requests_section_v12_2.png) +1. Expand the **Outbound requests** section. 1. Select **Allow requests to the local network from web hooks and services**. NOTE: diff --git a/doc/update/index.md b/doc/update/index.md index 74e284aa76c..e95b1e1ba50 100644 --- a/doc/update/index.md +++ b/doc/update/index.md @@ -380,7 +380,7 @@ Find where your version sits in the upgrade path below, and upgrade GitLab accordingly, while also consulting the [version-specific upgrade instructions](#version-specific-upgrading-instructions): -`8.11.Z` -> `8.12.0` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> [`11.11.8`](#1200) -> `12.0.12` -> [`12.1.17`](#1210) -> [`12.10.14`](#12100) -> `13.0.14` -> [`13.1.11`](#1310) -> [`13.8.8`](#1388) -> [`13.12.15`](#13120) -> [`14.0.12`](#1400) -> [`14.3.6`](#1430) -> [`14.9.5`](#1490) -> [`14.10.Z`](#14100) -> [`15.0.Z`](#1500) -> [latest `15.Y.Z`](https://gitlab.com/gitlab-org/gitlab/-/releases) +`8.11.Z` -> `8.12.0` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> [`11.11.8`](#1200) -> `12.0.12` -> [`12.1.17`](#1210) -> [`12.10.14`](#12100) -> `13.0.14` -> [`13.1.11`](#1310) -> [`13.8.8`](#1388) -> [`13.12.15`](#13120) -> [`14.0.12`](#1400) -> [`14.3.6`](#1430) -> [`14.9.5`](#1490) -> [`14.10.Z`](#14100) -> [`15.0.Z`](#1500) -> [`15.4.0`](#1540) -> [latest `15.Y.Z`](https://gitlab.com/gitlab-org/gitlab/-/releases) NOTE: When not explicitly specified, upgrade GitLab to the latest available patch @@ -465,6 +465,11 @@ NOTE: Specific information that follow related to Ruby and Git versions do not apply to [Omnibus installations](https://docs.gitlab.com/omnibus/) and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with appropriate Ruby and Git versions and are not using system binaries for Ruby and Git. There is no need to install Ruby or Git when utilizing these two approaches. +### 15.4.0 + +- GitLab 15.4.0 includes a [batched background migration](#batched-background-migrations) to [remove incorrect values from `expire_at` in `ci_job_artifacts` table](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89318). + This migration might take hours or days to complete on larger GitLab instances. + ### 15.3.2 In GitLab 15.3.2, [SAML Group Links](../api/groups.md#saml-group-links) API `access_level` attribute type changed to `integer`. See diff --git a/doc/user/project/import/svn.md b/doc/user/project/import/svn.md index 97ea889e36f..4673b9c0ed3 100644 --- a/doc/user/project/import/svn.md +++ b/doc/user/project/import/svn.md @@ -8,98 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Migrate from Subversion to GitLab **(FREE)** GitLab uses Git as its version control system. If you're using Subversion (SVN) as your version control system, -you can migrate to using a Git repository in GitLab in two ways: - -- Using SubGit to set up a temporary mirror of the SVN repository for GitLab. You can use the SVN repository and the Git - repository at the same time, and check everything functions properly before removing access to the SVN repository and - shutting down the mirror. -- Using `svn2git` to migrate immediately from SVN to Git. You stop using SVN, migrate the SVN repository, then - start using the Git repository in GitLab. - -## Migrate using SubGit - -[SubGit](https://subgit.com) creates a writable Git mirror of a local or remote SVN repository. SubGit requires access -to your GitLab server because it accesses the Git repositories directly at the file-system level. - -### SubGit prerequisites - -1. Install Oracle JRE 1.8 or newer. On Debian-based Linux distributions you can - follow [this article](http://www.webupd8.org/2012/09/install-oracle-java-8-in-ubuntu-via-ppa.html). -1. Download SubGit from . -1. Unpack the downloaded SubGit zip archive to the `/opt` directory. The `subgit` - command is available at `/opt/subgit-VERSION/bin/subgit`. - -### SubGit configuration - -The first step to mirror you SVN repository in GitLab is to create a new empty -project that is used as a mirror. For Omnibus installations the path to -the repository is -`/var/opt/gitlab/git-data/repositories/USER/REPO.git` by default. For -installations from source, the default repository directory is -`/home/git/repositories/USER/REPO.git`. For convenience, assign this path to a -variable: - -```shell -GIT_REPO_PATH=/var/opt/gitlab/git-data/repositories/USER/REPOS.git -``` - -SubGit keeps this repository in sync with a remote SVN project. For -convenience, assign your remote SVN project URL to a variable: - -```shell -SVN_PROJECT_URL=http://svn.company.com/repos/project -``` - -Next you need to run SubGit to set up a Git/SVN mirror. Make sure the following -`subgit` command is ran on behalf of the same user that keeps ownership of -GitLab Git repositories (by default `git`): - -```shell -subgit configure --layout auto $SVN_PROJECT_URL $GIT_REPO_PATH -``` - -Adjust authors and branches mappings, if necessary. Open with your favorite -text editor: - -```shell -edit $GIT_REPO_PATH/subgit/authors.txt -edit $GIT_REPO_PATH/subgit/config -``` - -For more information regarding the SubGit configuration options, refer to -[SubGit's documentation](https://subgit.com/documentation/) website. - -### Initial translation - -Now that SubGit has configured the Git/SVN repositories, run `subgit` to perform the -initial translation of existing SVN revisions into the Git repository: - -```shell -subgit install $GIT_REPO_PATH -``` - -After the initial translation is completed, `subgit` keeps the Git repository and the SVN -project sync - new Git commits are translated to -SVN revisions and new SVN revisions are translated to Git commits. Mirror -works transparently and does not require any special commands. - -If you would prefer to perform one-time cut over migration with `subgit`, use -the `import` command instead of `install`: - -```shell -subgit import $GIT_REPO_PATH -``` - -### SubGit licensing - -Running SubGit in a mirror mode requires a -[registration](https://subgit.com/pricing). Registration is free for open -source, academic and startup projects. - -### SubGit support - -For any questions related to SVN to GitLab migration with SubGit, you can -contact the SubGit team directly at [support@subgit.com](mailto:support@subgit.com). +you can migrate to using a Git repository in GitLab using `svn2git`. ## Migrate using `svn2git` diff --git a/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb new file mode 100644 index 00000000000..ff90d2dbcac --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # Batching class to use for removing backfilled job artifact expire_at. + # Batches will be scoped to records where either: + # - expire_at is set to midnight on the 22nd of the month of the local timezone, + # - record that has file_type = 3 (trace) + # + # If no more batches exist in the table, returns nil. + class RemoveBackfilledJobArtifactsExpireAtBatchingStrategy < PrimaryKeyBatchingStrategy + EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE = <<~SQL + EXTRACT(day FROM timezone('UTC', expire_at)) IN (21, 22, 23) + AND EXTRACT(minute FROM timezone('UTC', expire_at)) IN (0, 30, 45) + AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0 + SQL + + def apply_additional_filters(relation, job_arguments: [], job_class: nil) + relation.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE) + .or(relation.where(file_type: 3)) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb new file mode 100644 index 00000000000..2fdd6f9e68c --- /dev/null +++ b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This detects and fixes job artifacts that have `expire_at` wrongly backfilled by the migration + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723. + # These job artifacts will not be deleted and will have their `expire_at` removed. + class RemoveBackfilledJobArtifactsExpireAt < BatchedMigrationJob + # The migration would have backfilled `expire_at` + # to midnight on the 22nd of the month of the local timezone, + # storing it as UTC time in the database. + # + # If the timezone setting has changed since the migration, + # the `expire_at` stored in the database could have changed to a different local time other than midnight. + # For example: + # - changing timezone from UTC+02:00 to UTC+02:30 would change the `expire_at` in local time 00:00:00 to 00:30:00. + # - changing timezone from UTC+00:00 to UTC-01:00 would change the `expire_at` in local time 00:00:00 to 23:00:00 + # on the previous day (21st). + # + # Therefore job artifacts that have `expire_at` exactly on the 00, 30 or 45 minute mark + # on the dates 21, 22, 23 of the month will not be deleted. + # https://en.wikipedia.org/wiki/List_of_UTC_time_offsets + EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE = <<~SQL + EXTRACT(day FROM timezone('UTC', expire_at)) IN (21, 22, 23) + AND EXTRACT(minute FROM timezone('UTC', expire_at)) IN (0, 30, 45) + AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0 + SQL + + def perform + each_sub_batch( + operation_name: :update_all + ) do |sub_batch| + sub_batch.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE) + .or(sub_batch.where(file_type: 3)) + .update_all(expire_at: nil) + end + end + end + end +end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index e646846face..d7eef722d6e 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -10,8 +10,8 @@ module Gitlab extend self PROC_STAT_PATH = '/proc/self/stat' - PROC_STATUS_PATH = '/proc/self/status' - PROC_SMAPS_ROLLUP_PATH = '/proc/self/smaps_rollup' + PROC_STATUS_PATH = '/proc/%s/status' + PROC_SMAPS_ROLLUP_PATH = '/proc/%s/smaps_rollup' PROC_LIMITS_PATH = '/proc/self/limits' PROC_FD_GLOB = '/proc/self/fd/*' @@ -34,14 +34,14 @@ module Gitlab } end - # Returns the current process' RSS (resident set size) in bytes. - def memory_usage_rss - sum_matches(PROC_STATUS_PATH, rss: RSS_PATTERN)[:rss].kilobytes + # Returns the given process' RSS (resident set size) in bytes. + def memory_usage_rss(pid: 'self') + sum_matches(PROC_STATUS_PATH % pid, rss: RSS_PATTERN)[:rss].kilobytes end - # Returns the current process' USS/PSS (unique/proportional set size) in bytes. - def memory_usage_uss_pss - sum_matches(PROC_SMAPS_ROLLUP_PATH, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN) + # Returns the given process' USS/PSS (unique/proportional set size) in bytes. + def memory_usage_uss_pss(pid: 'self') + sum_matches(PROC_SMAPS_ROLLUP_PATH % pid, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN) .transform_values(&:kilobytes) end diff --git a/rubocop/cop/gitlab/mark_used_feature_flags.rb b/rubocop/cop/gitlab/mark_used_feature_flags.rb index 4844dab0a30..9f637c88b01 100644 --- a/rubocop/cop/gitlab/mark_used_feature_flags.rb +++ b/rubocop/cop/gitlab/mark_used_feature_flags.rb @@ -166,7 +166,7 @@ module RuboCop end def caller_is_feature?(node) - class_caller(node) == "Feature" + %w[Feature YamlProcessor::FeatureFlags].include?(class_caller(node)) end def caller_is_feature_gitaly?(node) diff --git a/spec/factories/protected_branches.rb b/spec/factories/protected_branches.rb index bac1cf21596..cee7b577236 100644 --- a/spec/factories/protected_branches.rb +++ b/spec/factories/protected_branches.rb @@ -11,6 +11,12 @@ FactoryBot.define do default_access_level { true } end + after(:create) do |protected_branch, evaluator| + break unless protected_branch.project&.persisted? + + ProtectedBranches::CacheService.new(protected_branch.project).refresh + end + trait :create_branch_on_repository do association :project, factory: [:project, :repository] diff --git a/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb b/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb new file mode 100644 index 00000000000..f6c877c03ed --- /dev/null +++ b/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::RemoveBackfilledJobArtifactsExpireAtBatchingStrategy, '#next_batch' do # rubocop:disable Layout/LineLength + let_it_be(:namespace) { table(:namespaces).create!(id: 1, name: 'user', path: 'user') } + let_it_be(:project) do + table(:projects).create!( + id: 1, + name: 'gitlab1', + path: 'gitlab1', + project_namespace_id: 1, + namespace_id: namespace.id + ) + end + + let(:batching_strategy) { described_class.new(connection: Ci::ApplicationRecord.connection) } + let(:job_artifact) { table(:ci_job_artifacts, database: :ci) } + + # job artifacts expiring at midnight in various timezones + let!(:ci_job_artifact_1) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) } + let!(:ci_job_artifact_2) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 01:30:00.000')) } + let!(:ci_job_artifact_3) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:00:00.000')) } + let!(:ci_job_artifact_4) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:30:00.000')) } + let!(:ci_job_artifact_5) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:00:00.000')) } + let!(:ci_job_artifact_6) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:30:00.000')) } + let!(:ci_job_artifact_7) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 06:45:00.000')) } + # out ot scope job artifacts + let!(:ci_job_artifact_8) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.001')) } + let!(:ci_job_artifact_9) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-19 12:00:00.000')) } + # job artifacts of trace type (file_type: 3) + let!(:ci_job_artifact_10) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) } + let!(:ci_job_artifact_11) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) } + # out ot scope job artifacts + let!(:ci_job_artifact_12) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-24 23:30:00.000')) } + let!(:ci_job_artifact_13) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:30:00.000')) } + # job artifacts of trace type (file_type: 3) + let!(:ci_job_artifact_14) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) } + let!(:ci_job_artifact_15) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) } + + it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchingStrategy } + + context 'when starting on the first batch' do + it 'returns the bounds of the next batch' do + batch_bounds = batching_strategy.next_batch( + :ci_job_artifacts, + :id, + batch_min_value: ci_job_artifact_1.id, + batch_size: 5, + job_arguments: [] + ) + expect(batch_bounds).to eq([ci_job_artifact_1.id, ci_job_artifact_5.id]) + end + end + + context 'when the range includes out of scope records' do + it 'returns the bounds of the next batch, skipping records outside the scope' do + batch_bounds = batching_strategy.next_batch( + :ci_job_artifacts, + :id, + batch_min_value: ci_job_artifact_1.id, + batch_size: 10, + job_arguments: [] + ) + expect(batch_bounds).to eq([ci_job_artifact_1.id, ci_job_artifact_14.id]) + end + end + + context 'when the range begins on out of scope records' do + it 'returns the bounds of the next batch, skipping records outside the scope' do + batch_bounds = batching_strategy.next_batch( + :ci_job_artifacts, + :id, + batch_min_value: ci_job_artifact_8.id, + batch_size: 3, + job_arguments: [] + ) + expect(batch_bounds).to eq([ci_job_artifact_10.id, ci_job_artifact_14.id]) + end + end + + context 'when no additional batch remain' do + it 'returns nil' do + batch_bounds = batching_strategy.next_batch( + :ci_job_artifacts, + :id, + batch_min_value: ci_job_artifact_15.id + 1, + batch_size: 10, + job_arguments: [] + ) + expect(batch_bounds).to be_nil + end + end + + private + + def create_job_artifact(file_type:, expire_at:) + job = table(:ci_builds, database: :ci).create! + job_artifact.create!(job_id: job.id, expire_at: expire_at, project_id: project.id, file_type: file_type) + end +end diff --git a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb new file mode 100644 index 00000000000..41266cb24da --- /dev/null +++ b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::RemoveBackfilledJobArtifactsExpireAt do + it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchedMigrationJob } + + describe '#perform' do + let(:job_artifact) { table(:ci_job_artifacts, database: :ci) } + + let(:test_worker) do + described_class.new( + start_id: 1, + end_id: 100, + batch_table: :ci_job_artifacts, + batch_column: :id, + sub_batch_size: 10, + pause_ms: 0, + connection: Ci::ApplicationRecord.connection + ) + end + + let_it_be(:namespace) { table(:namespaces).create!(id: 1, name: 'user', path: 'user') } + let_it_be(:project) do + table(:projects).create!( + id: 1, + name: 'gitlab1', + path: 'gitlab1', + project_namespace_id: 1, + namespace_id: namespace.id + ) + end + + subject { test_worker.perform } + + context 'with artifacts that has backfilled expire_at' do + let!(:created_on_00_30_45_minutes_on_21_22_23) do + create_job_artifact(id: 1, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) + create_job_artifact(id: 2, file_type: 1, expire_at: Time.zone.parse('2022-01-21 01:30:00.000')) + create_job_artifact(id: 3, file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:00:00.000')) + create_job_artifact(id: 4, file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:30:00.000')) + create_job_artifact(id: 5, file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:00:00.000')) + create_job_artifact(id: 6, file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:30:00.000')) + create_job_artifact(id: 7, file_type: 1, expire_at: Time.zone.parse('2022-01-23 06:45:00.000')) + end + + let!(:created_close_to_00_or_30_minutes) do + create_job_artifact(id: 8, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.001')) + create_job_artifact(id: 9, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:30:00.999')) + end + + let!(:created_on_00_or_30_minutes_on_other_dates) do + create_job_artifact(id: 10, file_type: 1, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) + create_job_artifact(id: 11, file_type: 1, expire_at: Time.zone.parse('2022-01-19 12:00:00.000')) + create_job_artifact(id: 12, file_type: 1, expire_at: Time.zone.parse('2022-01-24 23:30:00.000')) + end + + let!(:created_at_other_times) do + create_job_artifact(id: 13, file_type: 1, expire_at: Time.zone.parse('2022-01-19 00:00:00.000')) + create_job_artifact(id: 14, file_type: 1, expire_at: Time.zone.parse('2022-01-19 00:30:00.000')) + create_job_artifact(id: 15, file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:00:00.000')) + create_job_artifact(id: 16, file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:30:00.000')) + end + + it 'removes expire_at on job artifacts that have expire_at on 00, 30 or 45 minute of 21, 22, 23 of the month' do + expect { subject }.to change { job_artifact.where(expire_at: nil).count }.from(0).to(7) + end + + it 'keeps expire_at on other job artifacts' do + expect { subject }.to change { job_artifact.where.not(expire_at: nil).count }.from(16).to(9) + end + end + + context 'with trace artifacts that has backfilled expire_at' do + let!(:trace_artifacts) do + create_job_artifact(id: 1, file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) + create_job_artifact(id: 2, file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) + end + + it 'removes expire_at on trace job artifacts' do + expect { subject }.to change { job_artifact.where(expire_at: nil).count }.from(0).to(2) + end + end + + private + + def create_job_artifact(id:, file_type:, expire_at:) + job = table(:ci_builds, database: :ci).create!(id: id) + job_artifact.create!(id: id, job_id: job.id, expire_at: expire_at, project_id: project.id, file_type: file_type) + end + end +end diff --git a/spec/lib/gitlab/metrics/system_spec.rb b/spec/lib/gitlab/metrics/system_spec.rb index 26026a9540b..7739501dd95 100644 --- a/spec/lib/gitlab/metrics/system_spec.rb +++ b/spec/lib/gitlab/metrics/system_spec.rb @@ -72,10 +72,20 @@ RSpec.describe Gitlab::Metrics::System do end describe '.memory_usage_rss' do - it "returns the process' resident set size (RSS) in bytes" do - mock_existing_proc_file('/proc/self/status', proc_status) + context 'without PID' do + it "returns the current process' resident set size (RSS) in bytes" do + mock_existing_proc_file('/proc/self/status', proc_status) - expect(described_class.memory_usage_rss).to eq(2527232) + expect(described_class.memory_usage_rss).to eq(2527232) + end + end + + context 'with PID' do + it "returns the given process' resident set size (RSS) in bytes" do + mock_existing_proc_file('/proc/7/status', proc_status) + + expect(described_class.memory_usage_rss(pid: 7)).to eq(2527232) + end end end @@ -96,11 +106,22 @@ RSpec.describe Gitlab::Metrics::System do end describe '.memory_usage_uss_pss' do - it "returns the process' unique and porportional set size (USS/PSS) in bytes" do - mock_existing_proc_file('/proc/self/smaps_rollup', proc_smaps_rollup) + context 'without PID' do + it "returns the current process' unique and porportional set size (USS/PSS) in bytes" do + mock_existing_proc_file('/proc/self/smaps_rollup', proc_smaps_rollup) - # (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024 - expect(described_class.memory_usage_uss_pss).to eq(uss: 475136, pss: 515072) + # (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024 + expect(described_class.memory_usage_uss_pss).to eq(uss: 475136, pss: 515072) + end + end + + context 'with PID' do + it "returns the given process' unique and porportional set size (USS/PSS) in bytes" do + mock_existing_proc_file('/proc/7/smaps_rollup', proc_smaps_rollup) + + # (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024 + expect(described_class.memory_usage_uss_pss(pid: 7)).to eq(uss: 475136, pss: 515072) + end end end diff --git a/spec/migrations/20220606054503_fix_incorrect_job_artifacts_expire_at_spec.rb b/spec/migrations/20220606054503_fix_incorrect_job_artifacts_expire_at_spec.rb new file mode 100644 index 00000000000..5921dd64c0e --- /dev/null +++ b/spec/migrations/20220606054503_fix_incorrect_job_artifacts_expire_at_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe FixIncorrectJobArtifactsExpireAt, migration: :gitlab_ci do + let_it_be(:batched_migration) { described_class::MIGRATION } + + it 'does not schedule background jobs when Gitlab.com is true' do + allow(Gitlab).to receive(:com?).and_return(true) + + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + end + end + + it 'schedules background job on non Gitlab.com' do + allow(Gitlab).to receive(:com?).and_return(false) + + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + gitlab_schema: :gitlab_ci, + table_name: :ci_job_artifacts, + column_name: :id, + interval: described_class::INTERVAL, + batch_size: described_class::BATCH_SIZE + ) + } + end + end +end diff --git a/spec/rubocop/cop/gitlab/mark_used_feature_flags_spec.rb b/spec/rubocop/cop/gitlab/mark_used_feature_flags_spec.rb index c61376d7760..515620bd820 100644 --- a/spec/rubocop/cop/gitlab/mark_used_feature_flags_spec.rb +++ b/spec/rubocop/cop/gitlab/mark_used_feature_flags_spec.rb @@ -48,6 +48,7 @@ RSpec.describe RuboCop::Cop::Gitlab::MarkUsedFeatureFlags do Feature.enabled? Feature.disabled? push_frontend_feature_flag + YamlProcessor::FeatureFlags.enabled? ].each do |feature_flag_method| context "#{feature_flag_method} method" do context 'a string feature flag' do diff --git a/spec/services/protected_branches/destroy_service_spec.rb b/spec/services/protected_branches/destroy_service_spec.rb index 9fa07820148..123deeea005 100644 --- a/spec/services/protected_branches/destroy_service_spec.rb +++ b/spec/services/protected_branches/destroy_service_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' RSpec.describe ProtectedBranches::DestroyService do let_it_be_with_reload(:project) { create(:project) } - let(:protected_branch) { create(:protected_branch, project: project) } + let!(:protected_branch) { create(:protected_branch, project: project) } let(:user) { project.first_owner } subject(:service) { described_class.new(project, user) } diff --git a/spec/services/protected_branches/update_service_spec.rb b/spec/services/protected_branches/update_service_spec.rb index c4fe4d78070..2ff6c3c489a 100644 --- a/spec/services/protected_branches/update_service_spec.rb +++ b/spec/services/protected_branches/update_service_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' RSpec.describe ProtectedBranches::UpdateService do let_it_be_with_reload(:project) { create(:project) } - let(:protected_branch) { create(:protected_branch, project: project) } + let!(:protected_branch) { create(:protected_branch, project: project) } let(:user) { project.first_owner } let(:params) { { name: new_name } }