Add latest changes from gitlab-org/gitlab@master
This commit is contained in:
GitLab Bot
parent
55cd0a88bb
commit
0c918eb567
|
|
@ -9,7 +9,6 @@ import {
|
|||
GlTooltipDirective,
|
||||
GlSafeHtmlDirective,
|
||||
} from '@gitlab/ui';
|
||||
import mrWidgetPipelineMixin from '~/vue_merge_request_widget/mixins/mr_widget_pipeline';
|
||||
import { s__, n__ } from '~/locale';
|
||||
import CiIcon from '~/vue_shared/components/ci_icon.vue';
|
||||
import PipelineArtifacts from '~/pipelines/components/pipelines_list/pipelines_artifacts.vue';
|
||||
|
|
@ -36,7 +35,6 @@ export default {
|
|||
GlTooltip: GlTooltipDirective,
|
||||
SafeHtml: GlSafeHtmlDirective,
|
||||
},
|
||||
mixins: [mrWidgetPipelineMixin],
|
||||
props: {
|
||||
pipeline: {
|
||||
type: Object,
|
||||
|
|
@ -277,10 +275,10 @@ export default {
|
|||
<span class="gl-align-items-center gl-display-inline-flex">
|
||||
<pipeline-mini-graph
|
||||
v-if="pipeline.details.stages"
|
||||
:downstream-pipelines="triggered"
|
||||
:downstream-pipelines="pipeline.triggered"
|
||||
:is-merge-train="isMergeTrain"
|
||||
:stages="pipeline.details.stages"
|
||||
:upstream-pipeline="triggeredBy[0]"
|
||||
:upstream-pipeline="pipeline.triggered_by"
|
||||
stages-class="mr-widget-pipeline-stages"
|
||||
/>
|
||||
<pipeline-artifacts :pipeline-id="pipeline.id" :artifacts="artifacts" class="gl-ml-3" />
|
||||
|
|
|
|||
|
|
@ -1,10 +0,0 @@
|
|||
export default {
|
||||
computed: {
|
||||
triggered() {
|
||||
return [];
|
||||
},
|
||||
triggeredBy() {
|
||||
return [];
|
||||
},
|
||||
},
|
||||
};
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class FixIncorrectJobArtifactsExpireAt < Gitlab::Database::Migration[2.0]
|
||||
disable_ddl_transaction!
|
||||
|
||||
restrict_gitlab_migration gitlab_schema: :gitlab_ci
|
||||
|
||||
MIGRATION = 'RemoveBackfilledJobArtifactsExpireAt'
|
||||
BATCH_CLASS = 'RemoveBackfilledJobArtifactsExpireAtBatchingStrategy'
|
||||
BATCH_SIZE = 500
|
||||
INTERVAL = 2.minutes.freeze
|
||||
|
||||
def up
|
||||
return if Gitlab.com?
|
||||
|
||||
queue_batched_background_migration(
|
||||
MIGRATION,
|
||||
:ci_job_artifacts,
|
||||
:id,
|
||||
job_interval: INTERVAL,
|
||||
batch_class_name: BATCH_CLASS,
|
||||
batch_size: BATCH_SIZE
|
||||
)
|
||||
end
|
||||
|
||||
def down
|
||||
return if Gitlab.com?
|
||||
|
||||
delete_batched_background_migration(MIGRATION, :ci_job_artifacts, :id, [])
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class AddTmpIndexJobArtifactsIdAndExpireAt < Gitlab::Database::Migration[2.0]
|
||||
disable_ddl_transaction!
|
||||
|
||||
INDEX_NAME = 'tmp_index_ci_job_artifacts_on_id_expire_at_file_type_trace'
|
||||
|
||||
EXPIRE_AT_ON_22_MIDNIGHT_IN_TIMEZONE_OR_TRACE = <<~SQL
|
||||
(EXTRACT(day FROM timezone('UTC', expire_at)) IN (21, 22, 23)
|
||||
AND EXTRACT(minute FROM timezone('UTC', expire_at)) IN (0, 30, 45)
|
||||
AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0)
|
||||
OR file_type = 3
|
||||
SQL
|
||||
|
||||
def up
|
||||
return if Gitlab.com?
|
||||
return if index_exists_by_name?(:ci_job_artifacts, INDEX_NAME)
|
||||
|
||||
add_concurrent_index :ci_job_artifacts, :id,
|
||||
where: EXPIRE_AT_ON_22_MIDNIGHT_IN_TIMEZONE_OR_TRACE, name: INDEX_NAME
|
||||
end
|
||||
|
||||
def down
|
||||
return if Gitlab.com?
|
||||
return unless index_exists_by_name?(:ci_job_artifacts, INDEX_NAME)
|
||||
|
||||
remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1 @@
|
|||
ed815f5e2766913ded3479c2cdc8a885ab7164ae280e309cba34394668392a2b
|
||||
|
|
@ -0,0 +1 @@
|
|||
3afc50d92878da71453cfb23ad29d16123e4986e3304aff62013f4655b065d38
|
||||
|
|
@ -30638,6 +30638,8 @@ CREATE UNIQUE INDEX term_agreements_unique_index ON term_agreements USING btree
|
|||
|
||||
CREATE INDEX tmp_index_ci_job_artifacts_on_expire_at_where_locked_unknown ON ci_job_artifacts USING btree (expire_at, job_id) WHERE ((locked = 2) AND (expire_at IS NOT NULL));
|
||||
|
||||
CREATE INDEX tmp_index_ci_job_artifacts_on_id_expire_at_file_type_trace ON ci_job_artifacts USING btree (id) WHERE (((date_part('day'::text, timezone('UTC'::text, expire_at)) = ANY (ARRAY[(21)::double precision, (22)::double precision, (23)::double precision])) AND (date_part('minute'::text, timezone('UTC'::text, expire_at)) = ANY (ARRAY[(0)::double precision, (30)::double precision, (45)::double precision])) AND (date_part('second'::text, timezone('UTC'::text, expire_at)) = (0)::double precision)) OR (file_type = 3));
|
||||
|
||||
CREATE INDEX tmp_index_ci_job_artifacts_on_id_where_trace_and_expire_at ON ci_job_artifacts USING btree (id) WHERE ((file_type = 3) AND (expire_at = ANY (ARRAY['2021-04-22 00:00:00+00'::timestamp with time zone, '2021-05-22 00:00:00+00'::timestamp with time zone, '2021-06-22 00:00:00+00'::timestamp with time zone, '2022-01-22 00:00:00+00'::timestamp with time zone, '2022-02-22 00:00:00+00'::timestamp with time zone, '2022-03-22 00:00:00+00'::timestamp with time zone, '2022-04-22 00:00:00+00'::timestamp with time zone])));
|
||||
|
||||
CREATE INDEX tmp_index_cis_vulnerability_reads_on_id ON vulnerability_reads USING btree (id) WHERE (report_type = 7);
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 6.2 KiB |
|
|
@ -7,22 +7,14 @@ type: concepts, reference, howto
|
|||
|
||||
# Webhooks and insecure internal web services **(FREE SELF)**
|
||||
|
||||
NOTE:
|
||||
On GitLab.com, the [maximum number of webhooks and their size](../user/gitlab_com/index.md#webhooks) per project, and per group, is limited.
|
||||
Users with at least the Maintainer role can set up [Webhooks](../user/project/integrations/webhooks.md) that are
|
||||
triggered when specific changes occur in a project. When triggered, a `POST` HTTP request is sent to a URL. A webhook is
|
||||
usually configured to send data to a specific external web service, which
|
||||
processes the data in an appropriate way.
|
||||
|
||||
If you have non-GitLab web services running on your GitLab server or within its
|
||||
local network, these may be vulnerable to exploitation via Webhooks.
|
||||
|
||||
With [Webhooks](../user/project/integrations/webhooks.md), you and your project
|
||||
maintainers and owners can set up URLs to be triggered when specific changes
|
||||
occur in your projects. Normally, these requests are sent to external web
|
||||
services specifically set up for this purpose, that process the request and its
|
||||
attached data in some appropriate way.
|
||||
|
||||
Things get hairy, however, when a Webhook is set up with a URL that doesn't
|
||||
point to an external, but to an internal service, that may do something
|
||||
completely unintended when the webhook is triggered and the POST request is
|
||||
sent.
|
||||
However, a Webhook can be configured with a URL for an internal web service instead of an external web service.
|
||||
When the Webhook is triggered,
|
||||
non-GitLab web services running on your GitLab server or in its local network could be exploited.
|
||||
|
||||
Webhook requests are made by the GitLab server itself and use a single
|
||||
(optional) secret token per hook for authorization (instead of a user or
|
||||
|
|
@ -38,6 +30,8 @@ If a web service does not require authentication, Webhooks can be used to
|
|||
trigger destructive commands by getting the GitLab server to make POST requests
|
||||
to endpoints like `http://localhost:123/some-resource/delete`.
|
||||
|
||||
## Allow requests to local network
|
||||
|
||||
To prevent this type of exploitation from happening, starting with GitLab 10.6,
|
||||
all Webhook requests to the current GitLab instance server address and/or in a
|
||||
private network are forbidden by default. That means that all requests made
|
||||
|
|
@ -48,8 +42,7 @@ This behavior can be overridden:
|
|||
|
||||
1. On the top bar, select **Menu > Admin**.
|
||||
1. On the left sidebar, select **Settings > Network**.
|
||||
1. Expand the **Outbound requests** section:
|
||||
|
||||
1. Expand the **Outbound requests** section.
|
||||
1. Select **Allow requests to the local network from web hooks and services**.
|
||||
|
||||
NOTE:
|
||||
|
|
|
|||
|
|
@ -380,7 +380,7 @@ Find where your version sits in the upgrade path below, and upgrade GitLab
|
|||
accordingly, while also consulting the
|
||||
[version-specific upgrade instructions](#version-specific-upgrading-instructions):
|
||||
|
||||
`8.11.Z` -> `8.12.0` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> [`11.11.8`](#1200) -> `12.0.12` -> [`12.1.17`](#1210) -> [`12.10.14`](#12100) -> `13.0.14` -> [`13.1.11`](#1310) -> [`13.8.8`](#1388) -> [`13.12.15`](#13120) -> [`14.0.12`](#1400) -> [`14.3.6`](#1430) -> [`14.9.5`](#1490) -> [`14.10.Z`](#14100) -> [`15.0.Z`](#1500) -> [latest `15.Y.Z`](https://gitlab.com/gitlab-org/gitlab/-/releases)
|
||||
`8.11.Z` -> `8.12.0` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> [`11.11.8`](#1200) -> `12.0.12` -> [`12.1.17`](#1210) -> [`12.10.14`](#12100) -> `13.0.14` -> [`13.1.11`](#1310) -> [`13.8.8`](#1388) -> [`13.12.15`](#13120) -> [`14.0.12`](#1400) -> [`14.3.6`](#1430) -> [`14.9.5`](#1490) -> [`14.10.Z`](#14100) -> [`15.0.Z`](#1500) -> [`15.4.0`](#1540) -> [latest `15.Y.Z`](https://gitlab.com/gitlab-org/gitlab/-/releases)
|
||||
|
||||
NOTE:
|
||||
When not explicitly specified, upgrade GitLab to the latest available patch
|
||||
|
|
@ -465,6 +465,11 @@ NOTE:
|
|||
Specific information that follow related to Ruby and Git versions do not apply to [Omnibus installations](https://docs.gitlab.com/omnibus/)
|
||||
and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with appropriate Ruby and Git versions and are not using system binaries for Ruby and Git. There is no need to install Ruby or Git when utilizing these two approaches.
|
||||
|
||||
### 15.4.0
|
||||
|
||||
- GitLab 15.4.0 includes a [batched background migration](#batched-background-migrations) to [remove incorrect values from `expire_at` in `ci_job_artifacts` table](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89318).
|
||||
This migration might take hours or days to complete on larger GitLab instances.
|
||||
|
||||
### 15.3.2
|
||||
|
||||
In GitLab 15.3.2, [SAML Group Links](../api/groups.md#saml-group-links) API `access_level` attribute type changed to `integer`. See
|
||||
|
|
|
|||
|
|
@ -8,98 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
|
|||
# Migrate from Subversion to GitLab **(FREE)**
|
||||
|
||||
GitLab uses Git as its version control system. If you're using Subversion (SVN) as your version control system,
|
||||
you can migrate to using a Git repository in GitLab in two ways:
|
||||
|
||||
- Using SubGit to set up a temporary mirror of the SVN repository for GitLab. You can use the SVN repository and the Git
|
||||
repository at the same time, and check everything functions properly before removing access to the SVN repository and
|
||||
shutting down the mirror.
|
||||
- Using `svn2git` to migrate immediately from SVN to Git. You stop using SVN, migrate the SVN repository, then
|
||||
start using the Git repository in GitLab.
|
||||
|
||||
## Migrate using SubGit
|
||||
|
||||
[SubGit](https://subgit.com) creates a writable Git mirror of a local or remote SVN repository. SubGit requires access
|
||||
to your GitLab server because it accesses the Git repositories directly at the file-system level.
|
||||
|
||||
### SubGit prerequisites
|
||||
|
||||
1. Install Oracle JRE 1.8 or newer. On Debian-based Linux distributions you can
|
||||
follow [this article](http://www.webupd8.org/2012/09/install-oracle-java-8-in-ubuntu-via-ppa.html).
|
||||
1. Download SubGit from <https://subgit.com/download>.
|
||||
1. Unpack the downloaded SubGit zip archive to the `/opt` directory. The `subgit`
|
||||
command is available at `/opt/subgit-VERSION/bin/subgit`.
|
||||
|
||||
### SubGit configuration
|
||||
|
||||
The first step to mirror you SVN repository in GitLab is to create a new empty
|
||||
project that is used as a mirror. For Omnibus installations the path to
|
||||
the repository is
|
||||
`/var/opt/gitlab/git-data/repositories/USER/REPO.git` by default. For
|
||||
installations from source, the default repository directory is
|
||||
`/home/git/repositories/USER/REPO.git`. For convenience, assign this path to a
|
||||
variable:
|
||||
|
||||
```shell
|
||||
GIT_REPO_PATH=/var/opt/gitlab/git-data/repositories/USER/REPOS.git
|
||||
```
|
||||
|
||||
SubGit keeps this repository in sync with a remote SVN project. For
|
||||
convenience, assign your remote SVN project URL to a variable:
|
||||
|
||||
```shell
|
||||
SVN_PROJECT_URL=http://svn.company.com/repos/project
|
||||
```
|
||||
|
||||
Next you need to run SubGit to set up a Git/SVN mirror. Make sure the following
|
||||
`subgit` command is ran on behalf of the same user that keeps ownership of
|
||||
GitLab Git repositories (by default `git`):
|
||||
|
||||
```shell
|
||||
subgit configure --layout auto $SVN_PROJECT_URL $GIT_REPO_PATH
|
||||
```
|
||||
|
||||
Adjust authors and branches mappings, if necessary. Open with your favorite
|
||||
text editor:
|
||||
|
||||
```shell
|
||||
edit $GIT_REPO_PATH/subgit/authors.txt
|
||||
edit $GIT_REPO_PATH/subgit/config
|
||||
```
|
||||
|
||||
For more information regarding the SubGit configuration options, refer to
|
||||
[SubGit's documentation](https://subgit.com/documentation/) website.
|
||||
|
||||
### Initial translation
|
||||
|
||||
Now that SubGit has configured the Git/SVN repositories, run `subgit` to perform the
|
||||
initial translation of existing SVN revisions into the Git repository:
|
||||
|
||||
```shell
|
||||
subgit install $GIT_REPO_PATH
|
||||
```
|
||||
|
||||
After the initial translation is completed, `subgit` keeps the Git repository and the SVN
|
||||
project sync - new Git commits are translated to
|
||||
SVN revisions and new SVN revisions are translated to Git commits. Mirror
|
||||
works transparently and does not require any special commands.
|
||||
|
||||
If you would prefer to perform one-time cut over migration with `subgit`, use
|
||||
the `import` command instead of `install`:
|
||||
|
||||
```shell
|
||||
subgit import $GIT_REPO_PATH
|
||||
```
|
||||
|
||||
### SubGit licensing
|
||||
|
||||
Running SubGit in a mirror mode requires a
|
||||
[registration](https://subgit.com/pricing). Registration is free for open
|
||||
source, academic and startup projects.
|
||||
|
||||
### SubGit support
|
||||
|
||||
For any questions related to SVN to GitLab migration with SubGit, you can
|
||||
contact the SubGit team directly at [support@subgit.com](mailto:support@subgit.com).
|
||||
you can migrate to using a Git repository in GitLab using `svn2git`.
|
||||
|
||||
## Migrate using `svn2git`
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module Gitlab
|
||||
module BackgroundMigration
|
||||
module BatchingStrategies
|
||||
# Batching class to use for removing backfilled job artifact expire_at.
|
||||
# Batches will be scoped to records where either:
|
||||
# - expire_at is set to midnight on the 22nd of the month of the local timezone,
|
||||
# - record that has file_type = 3 (trace)
|
||||
#
|
||||
# If no more batches exist in the table, returns nil.
|
||||
class RemoveBackfilledJobArtifactsExpireAtBatchingStrategy < PrimaryKeyBatchingStrategy
|
||||
EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE = <<~SQL
|
||||
EXTRACT(day FROM timezone('UTC', expire_at)) IN (21, 22, 23)
|
||||
AND EXTRACT(minute FROM timezone('UTC', expire_at)) IN (0, 30, 45)
|
||||
AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0
|
||||
SQL
|
||||
|
||||
def apply_additional_filters(relation, job_arguments: [], job_class: nil)
|
||||
relation.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE)
|
||||
.or(relation.where(file_type: 3))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module Gitlab
|
||||
module BackgroundMigration
|
||||
# This detects and fixes job artifacts that have `expire_at` wrongly backfilled by the migration
|
||||
# https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723.
|
||||
# These job artifacts will not be deleted and will have their `expire_at` removed.
|
||||
class RemoveBackfilledJobArtifactsExpireAt < BatchedMigrationJob
|
||||
# The migration would have backfilled `expire_at`
|
||||
# to midnight on the 22nd of the month of the local timezone,
|
||||
# storing it as UTC time in the database.
|
||||
#
|
||||
# If the timezone setting has changed since the migration,
|
||||
# the `expire_at` stored in the database could have changed to a different local time other than midnight.
|
||||
# For example:
|
||||
# - changing timezone from UTC+02:00 to UTC+02:30 would change the `expire_at` in local time 00:00:00 to 00:30:00.
|
||||
# - changing timezone from UTC+00:00 to UTC-01:00 would change the `expire_at` in local time 00:00:00 to 23:00:00
|
||||
# on the previous day (21st).
|
||||
#
|
||||
# Therefore job artifacts that have `expire_at` exactly on the 00, 30 or 45 minute mark
|
||||
# on the dates 21, 22, 23 of the month will not be deleted.
|
||||
# https://en.wikipedia.org/wiki/List_of_UTC_time_offsets
|
||||
EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE = <<~SQL
|
||||
EXTRACT(day FROM timezone('UTC', expire_at)) IN (21, 22, 23)
|
||||
AND EXTRACT(minute FROM timezone('UTC', expire_at)) IN (0, 30, 45)
|
||||
AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0
|
||||
SQL
|
||||
|
||||
def perform
|
||||
each_sub_batch(
|
||||
operation_name: :update_all
|
||||
) do |sub_batch|
|
||||
sub_batch.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE)
|
||||
.or(sub_batch.where(file_type: 3))
|
||||
.update_all(expire_at: nil)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -10,8 +10,8 @@ module Gitlab
|
|||
extend self
|
||||
|
||||
PROC_STAT_PATH = '/proc/self/stat'
|
||||
PROC_STATUS_PATH = '/proc/self/status'
|
||||
PROC_SMAPS_ROLLUP_PATH = '/proc/self/smaps_rollup'
|
||||
PROC_STATUS_PATH = '/proc/%s/status'
|
||||
PROC_SMAPS_ROLLUP_PATH = '/proc/%s/smaps_rollup'
|
||||
PROC_LIMITS_PATH = '/proc/self/limits'
|
||||
PROC_FD_GLOB = '/proc/self/fd/*'
|
||||
|
||||
|
|
@ -34,14 +34,14 @@ module Gitlab
|
|||
}
|
||||
end
|
||||
|
||||
# Returns the current process' RSS (resident set size) in bytes.
|
||||
def memory_usage_rss
|
||||
sum_matches(PROC_STATUS_PATH, rss: RSS_PATTERN)[:rss].kilobytes
|
||||
# Returns the given process' RSS (resident set size) in bytes.
|
||||
def memory_usage_rss(pid: 'self')
|
||||
sum_matches(PROC_STATUS_PATH % pid, rss: RSS_PATTERN)[:rss].kilobytes
|
||||
end
|
||||
|
||||
# Returns the current process' USS/PSS (unique/proportional set size) in bytes.
|
||||
def memory_usage_uss_pss
|
||||
sum_matches(PROC_SMAPS_ROLLUP_PATH, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN)
|
||||
# Returns the given process' USS/PSS (unique/proportional set size) in bytes.
|
||||
def memory_usage_uss_pss(pid: 'self')
|
||||
sum_matches(PROC_SMAPS_ROLLUP_PATH % pid, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN)
|
||||
.transform_values(&:kilobytes)
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -166,7 +166,7 @@ module RuboCop
|
|||
end
|
||||
|
||||
def caller_is_feature?(node)
|
||||
class_caller(node) == "Feature"
|
||||
%w[Feature YamlProcessor::FeatureFlags].include?(class_caller(node))
|
||||
end
|
||||
|
||||
def caller_is_feature_gitaly?(node)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,12 @@ FactoryBot.define do
|
|||
default_access_level { true }
|
||||
end
|
||||
|
||||
after(:create) do |protected_branch, evaluator|
|
||||
break unless protected_branch.project&.persisted?
|
||||
|
||||
ProtectedBranches::CacheService.new(protected_branch.project).refresh
|
||||
end
|
||||
|
||||
trait :create_branch_on_repository do
|
||||
association :project, factory: [:project, :repository]
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,101 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'spec_helper'
|
||||
|
||||
RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::RemoveBackfilledJobArtifactsExpireAtBatchingStrategy, '#next_batch' do # rubocop:disable Layout/LineLength
|
||||
let_it_be(:namespace) { table(:namespaces).create!(id: 1, name: 'user', path: 'user') }
|
||||
let_it_be(:project) do
|
||||
table(:projects).create!(
|
||||
id: 1,
|
||||
name: 'gitlab1',
|
||||
path: 'gitlab1',
|
||||
project_namespace_id: 1,
|
||||
namespace_id: namespace.id
|
||||
)
|
||||
end
|
||||
|
||||
let(:batching_strategy) { described_class.new(connection: Ci::ApplicationRecord.connection) }
|
||||
let(:job_artifact) { table(:ci_job_artifacts, database: :ci) }
|
||||
|
||||
# job artifacts expiring at midnight in various timezones
|
||||
let!(:ci_job_artifact_1) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) }
|
||||
let!(:ci_job_artifact_2) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 01:30:00.000')) }
|
||||
let!(:ci_job_artifact_3) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:00:00.000')) }
|
||||
let!(:ci_job_artifact_4) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:30:00.000')) }
|
||||
let!(:ci_job_artifact_5) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:00:00.000')) }
|
||||
let!(:ci_job_artifact_6) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:30:00.000')) }
|
||||
let!(:ci_job_artifact_7) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 06:45:00.000')) }
|
||||
# out ot scope job artifacts
|
||||
let!(:ci_job_artifact_8) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.001')) }
|
||||
let!(:ci_job_artifact_9) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-19 12:00:00.000')) }
|
||||
# job artifacts of trace type (file_type: 3)
|
||||
let!(:ci_job_artifact_10) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) }
|
||||
let!(:ci_job_artifact_11) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) }
|
||||
# out ot scope job artifacts
|
||||
let!(:ci_job_artifact_12) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-24 23:30:00.000')) }
|
||||
let!(:ci_job_artifact_13) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:30:00.000')) }
|
||||
# job artifacts of trace type (file_type: 3)
|
||||
let!(:ci_job_artifact_14) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) }
|
||||
let!(:ci_job_artifact_15) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) }
|
||||
|
||||
it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchingStrategy }
|
||||
|
||||
context 'when starting on the first batch' do
|
||||
it 'returns the bounds of the next batch' do
|
||||
batch_bounds = batching_strategy.next_batch(
|
||||
:ci_job_artifacts,
|
||||
:id,
|
||||
batch_min_value: ci_job_artifact_1.id,
|
||||
batch_size: 5,
|
||||
job_arguments: []
|
||||
)
|
||||
expect(batch_bounds).to eq([ci_job_artifact_1.id, ci_job_artifact_5.id])
|
||||
end
|
||||
end
|
||||
|
||||
context 'when the range includes out of scope records' do
|
||||
it 'returns the bounds of the next batch, skipping records outside the scope' do
|
||||
batch_bounds = batching_strategy.next_batch(
|
||||
:ci_job_artifacts,
|
||||
:id,
|
||||
batch_min_value: ci_job_artifact_1.id,
|
||||
batch_size: 10,
|
||||
job_arguments: []
|
||||
)
|
||||
expect(batch_bounds).to eq([ci_job_artifact_1.id, ci_job_artifact_14.id])
|
||||
end
|
||||
end
|
||||
|
||||
context 'when the range begins on out of scope records' do
|
||||
it 'returns the bounds of the next batch, skipping records outside the scope' do
|
||||
batch_bounds = batching_strategy.next_batch(
|
||||
:ci_job_artifacts,
|
||||
:id,
|
||||
batch_min_value: ci_job_artifact_8.id,
|
||||
batch_size: 3,
|
||||
job_arguments: []
|
||||
)
|
||||
expect(batch_bounds).to eq([ci_job_artifact_10.id, ci_job_artifact_14.id])
|
||||
end
|
||||
end
|
||||
|
||||
context 'when no additional batch remain' do
|
||||
it 'returns nil' do
|
||||
batch_bounds = batching_strategy.next_batch(
|
||||
:ci_job_artifacts,
|
||||
:id,
|
||||
batch_min_value: ci_job_artifact_15.id + 1,
|
||||
batch_size: 10,
|
||||
job_arguments: []
|
||||
)
|
||||
expect(batch_bounds).to be_nil
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def create_job_artifact(file_type:, expire_at:)
|
||||
job = table(:ci_builds, database: :ci).create!
|
||||
job_artifact.create!(job_id: job.id, expire_at: expire_at, project_id: project.id, file_type: file_type)
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'spec_helper'
|
||||
|
||||
RSpec.describe Gitlab::BackgroundMigration::RemoveBackfilledJobArtifactsExpireAt do
|
||||
it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchedMigrationJob }
|
||||
|
||||
describe '#perform' do
|
||||
let(:job_artifact) { table(:ci_job_artifacts, database: :ci) }
|
||||
|
||||
let(:test_worker) do
|
||||
described_class.new(
|
||||
start_id: 1,
|
||||
end_id: 100,
|
||||
batch_table: :ci_job_artifacts,
|
||||
batch_column: :id,
|
||||
sub_batch_size: 10,
|
||||
pause_ms: 0,
|
||||
connection: Ci::ApplicationRecord.connection
|
||||
)
|
||||
end
|
||||
|
||||
let_it_be(:namespace) { table(:namespaces).create!(id: 1, name: 'user', path: 'user') }
|
||||
let_it_be(:project) do
|
||||
table(:projects).create!(
|
||||
id: 1,
|
||||
name: 'gitlab1',
|
||||
path: 'gitlab1',
|
||||
project_namespace_id: 1,
|
||||
namespace_id: namespace.id
|
||||
)
|
||||
end
|
||||
|
||||
subject { test_worker.perform }
|
||||
|
||||
context 'with artifacts that has backfilled expire_at' do
|
||||
let!(:created_on_00_30_45_minutes_on_21_22_23) do
|
||||
create_job_artifact(id: 1, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.000'))
|
||||
create_job_artifact(id: 2, file_type: 1, expire_at: Time.zone.parse('2022-01-21 01:30:00.000'))
|
||||
create_job_artifact(id: 3, file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:00:00.000'))
|
||||
create_job_artifact(id: 4, file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:30:00.000'))
|
||||
create_job_artifact(id: 5, file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:00:00.000'))
|
||||
create_job_artifact(id: 6, file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:30:00.000'))
|
||||
create_job_artifact(id: 7, file_type: 1, expire_at: Time.zone.parse('2022-01-23 06:45:00.000'))
|
||||
end
|
||||
|
||||
let!(:created_close_to_00_or_30_minutes) do
|
||||
create_job_artifact(id: 8, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.001'))
|
||||
create_job_artifact(id: 9, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:30:00.999'))
|
||||
end
|
||||
|
||||
let!(:created_on_00_or_30_minutes_on_other_dates) do
|
||||
create_job_artifact(id: 10, file_type: 1, expire_at: Time.zone.parse('2022-01-01 00:00:00.000'))
|
||||
create_job_artifact(id: 11, file_type: 1, expire_at: Time.zone.parse('2022-01-19 12:00:00.000'))
|
||||
create_job_artifact(id: 12, file_type: 1, expire_at: Time.zone.parse('2022-01-24 23:30:00.000'))
|
||||
end
|
||||
|
||||
let!(:created_at_other_times) do
|
||||
create_job_artifact(id: 13, file_type: 1, expire_at: Time.zone.parse('2022-01-19 00:00:00.000'))
|
||||
create_job_artifact(id: 14, file_type: 1, expire_at: Time.zone.parse('2022-01-19 00:30:00.000'))
|
||||
create_job_artifact(id: 15, file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:00:00.000'))
|
||||
create_job_artifact(id: 16, file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:30:00.000'))
|
||||
end
|
||||
|
||||
it 'removes expire_at on job artifacts that have expire_at on 00, 30 or 45 minute of 21, 22, 23 of the month' do
|
||||
expect { subject }.to change { job_artifact.where(expire_at: nil).count }.from(0).to(7)
|
||||
end
|
||||
|
||||
it 'keeps expire_at on other job artifacts' do
|
||||
expect { subject }.to change { job_artifact.where.not(expire_at: nil).count }.from(16).to(9)
|
||||
end
|
||||
end
|
||||
|
||||
context 'with trace artifacts that has backfilled expire_at' do
|
||||
let!(:trace_artifacts) do
|
||||
create_job_artifact(id: 1, file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000'))
|
||||
create_job_artifact(id: 2, file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000'))
|
||||
end
|
||||
|
||||
it 'removes expire_at on trace job artifacts' do
|
||||
expect { subject }.to change { job_artifact.where(expire_at: nil).count }.from(0).to(2)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def create_job_artifact(id:, file_type:, expire_at:)
|
||||
job = table(:ci_builds, database: :ci).create!(id: id)
|
||||
job_artifact.create!(id: id, job_id: job.id, expire_at: expire_at, project_id: project.id, file_type: file_type)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -72,10 +72,20 @@ RSpec.describe Gitlab::Metrics::System do
|
|||
end
|
||||
|
||||
describe '.memory_usage_rss' do
|
||||
it "returns the process' resident set size (RSS) in bytes" do
|
||||
mock_existing_proc_file('/proc/self/status', proc_status)
|
||||
context 'without PID' do
|
||||
it "returns the current process' resident set size (RSS) in bytes" do
|
||||
mock_existing_proc_file('/proc/self/status', proc_status)
|
||||
|
||||
expect(described_class.memory_usage_rss).to eq(2527232)
|
||||
expect(described_class.memory_usage_rss).to eq(2527232)
|
||||
end
|
||||
end
|
||||
|
||||
context 'with PID' do
|
||||
it "returns the given process' resident set size (RSS) in bytes" do
|
||||
mock_existing_proc_file('/proc/7/status', proc_status)
|
||||
|
||||
expect(described_class.memory_usage_rss(pid: 7)).to eq(2527232)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -96,11 +106,22 @@ RSpec.describe Gitlab::Metrics::System do
|
|||
end
|
||||
|
||||
describe '.memory_usage_uss_pss' do
|
||||
it "returns the process' unique and porportional set size (USS/PSS) in bytes" do
|
||||
mock_existing_proc_file('/proc/self/smaps_rollup', proc_smaps_rollup)
|
||||
context 'without PID' do
|
||||
it "returns the current process' unique and porportional set size (USS/PSS) in bytes" do
|
||||
mock_existing_proc_file('/proc/self/smaps_rollup', proc_smaps_rollup)
|
||||
|
||||
# (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024
|
||||
expect(described_class.memory_usage_uss_pss).to eq(uss: 475136, pss: 515072)
|
||||
# (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024
|
||||
expect(described_class.memory_usage_uss_pss).to eq(uss: 475136, pss: 515072)
|
||||
end
|
||||
end
|
||||
|
||||
context 'with PID' do
|
||||
it "returns the given process' unique and porportional set size (USS/PSS) in bytes" do
|
||||
mock_existing_proc_file('/proc/7/smaps_rollup', proc_smaps_rollup)
|
||||
|
||||
# (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024
|
||||
expect(described_class.memory_usage_uss_pss(pid: 7)).to eq(uss: 475136, pss: 515072)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'spec_helper'
|
||||
require_migration!
|
||||
|
||||
RSpec.describe FixIncorrectJobArtifactsExpireAt, migration: :gitlab_ci do
|
||||
let_it_be(:batched_migration) { described_class::MIGRATION }
|
||||
|
||||
it 'does not schedule background jobs when Gitlab.com is true' do
|
||||
allow(Gitlab).to receive(:com?).and_return(true)
|
||||
|
||||
reversible_migration do |migration|
|
||||
migration.before -> {
|
||||
expect(batched_migration).not_to have_scheduled_batched_migration
|
||||
}
|
||||
|
||||
migration.after -> {
|
||||
expect(batched_migration).not_to have_scheduled_batched_migration
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
it 'schedules background job on non Gitlab.com' do
|
||||
allow(Gitlab).to receive(:com?).and_return(false)
|
||||
|
||||
reversible_migration do |migration|
|
||||
migration.before -> {
|
||||
expect(batched_migration).not_to have_scheduled_batched_migration
|
||||
}
|
||||
|
||||
migration.after -> {
|
||||
expect(batched_migration).to have_scheduled_batched_migration(
|
||||
gitlab_schema: :gitlab_ci,
|
||||
table_name: :ci_job_artifacts,
|
||||
column_name: :id,
|
||||
interval: described_class::INTERVAL,
|
||||
batch_size: described_class::BATCH_SIZE
|
||||
)
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -48,6 +48,7 @@ RSpec.describe RuboCop::Cop::Gitlab::MarkUsedFeatureFlags do
|
|||
Feature.enabled?
|
||||
Feature.disabled?
|
||||
push_frontend_feature_flag
|
||||
YamlProcessor::FeatureFlags.enabled?
|
||||
].each do |feature_flag_method|
|
||||
context "#{feature_flag_method} method" do
|
||||
context 'a string feature flag' do
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ require 'spec_helper'
|
|||
RSpec.describe ProtectedBranches::DestroyService do
|
||||
let_it_be_with_reload(:project) { create(:project) }
|
||||
|
||||
let(:protected_branch) { create(:protected_branch, project: project) }
|
||||
let!(:protected_branch) { create(:protected_branch, project: project) }
|
||||
let(:user) { project.first_owner }
|
||||
|
||||
subject(:service) { described_class.new(project, user) }
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ require 'spec_helper'
|
|||
RSpec.describe ProtectedBranches::UpdateService do
|
||||
let_it_be_with_reload(:project) { create(:project) }
|
||||
|
||||
let(:protected_branch) { create(:protected_branch, project: project) }
|
||||
let!(:protected_branch) { create(:protected_branch, project: project) }
|
||||
let(:user) { project.first_owner }
|
||||
let(:params) { { name: new_name } }
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue