Add latest changes from gitlab-org/gitlab@master
This commit is contained in:
parent
17569e185c
commit
fd767b7d65
|
@ -31,11 +31,14 @@
|
|||
.if-merge-request: &if-merge-request
|
||||
if: '$CI_MERGE_REQUEST_IID'
|
||||
|
||||
# Once https://gitlab.com/gitlab-org/gitlab/-/issues/373904 is implemented, we should be able to change this back to
|
||||
# if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_APPROVALS_COUNT > 0'
|
||||
# or any similar condition to check that the MR has *any* approval (not just required approval).
|
||||
.if-merge-request-approved: &if-merge-request-approved
|
||||
if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_APPROVED'
|
||||
if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_LABELS =~ /pipeline:run-full-rspec/'
|
||||
|
||||
.if-merge-request-not-approved: &if-merge-request-not-approved
|
||||
if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_APPROVED != "true"'
|
||||
if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_LABELS !~ /pipeline:run-full-rspec/'
|
||||
|
||||
.if-automated-merge-request: &if-automated-merge-request
|
||||
if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "release-tools/update-gitaly" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /stable-ee$/'
|
||||
|
@ -532,9 +535,6 @@
|
|||
rules:
|
||||
- <<: *if-merge-request-approved
|
||||
when: never
|
||||
# Temporarily disabled minimal rspec jobs before and after approval because of https://gitlab.com/gitlab-org/gitlab/-/issues/373064.
|
||||
- <<: *if-merge-request-not-approved
|
||||
when: never
|
||||
- <<: *if-automated-merge-request
|
||||
when: never
|
||||
- <<: *if-security-merge-request
|
||||
|
@ -554,6 +554,8 @@
|
|||
changes: *backend-patterns
|
||||
- <<: *if-security-merge-request
|
||||
changes: *backend-patterns
|
||||
- <<: *if-merge-request-not-approved
|
||||
when: never
|
||||
|
||||
.rails:rules:as-if-foss-migration-unit-integration:minimal-default-rules:
|
||||
rules:
|
||||
|
@ -581,6 +583,8 @@
|
|||
changes: *code-backstage-patterns
|
||||
- <<: *if-security-merge-request
|
||||
changes: *code-backstage-patterns
|
||||
- <<: *if-merge-request-not-approved
|
||||
when: never
|
||||
|
||||
.rails:rules:system:minimal-default-rules:
|
||||
rules:
|
||||
|
@ -1007,6 +1011,8 @@
|
|||
changes: *db-patterns
|
||||
- <<: *if-security-merge-request
|
||||
changes: *db-patterns
|
||||
- <<: *if-merge-request-not-approved
|
||||
when: never
|
||||
- changes: *db-patterns
|
||||
|
||||
.rails:rules:ee-and-foss-migration:minimal:
|
||||
|
@ -1108,6 +1114,8 @@
|
|||
changes: *db-patterns
|
||||
- <<: *if-security-merge-request
|
||||
changes: *db-patterns
|
||||
- <<: *if-merge-request-not-approved
|
||||
when: never
|
||||
- changes: *db-patterns
|
||||
|
||||
.rails:rules:ee-only-migration:minimal:
|
||||
|
@ -1195,6 +1203,8 @@
|
|||
changes: *db-patterns
|
||||
- <<: *if-security-merge-request
|
||||
changes: *db-patterns
|
||||
- <<: *if-merge-request-not-approved
|
||||
when: never
|
||||
|
||||
.rails:rules:as-if-foss-migration:minimal:
|
||||
rules:
|
||||
|
|
|
@ -13,6 +13,9 @@ inherit_from:
|
|||
<% end %>
|
||||
- '.rubocop_todo.yml'
|
||||
<% end %>
|
||||
<% if RUBY_VERSION[/^\d+\.\d+/, 0] == '3.0' %>
|
||||
- ./rubocop/rubocop-ruby30.yml
|
||||
<% end %>
|
||||
- ./rubocop/rubocop-migrations.yml
|
||||
- ./rubocop/rubocop-usage-data.yml
|
||||
- ./rubocop/rubocop-code_reuse.yml
|
||||
|
@ -84,7 +87,6 @@ Lint/EmptyFile:
|
|||
# This cop checks whether some constant value isn't a
|
||||
# mutable literal (e.g. array or hash).
|
||||
Style/MutableConstant:
|
||||
Enabled: true
|
||||
Exclude:
|
||||
- 'db/migrate/**/*'
|
||||
- 'db/post_migrate/**/*'
|
||||
|
|
|
@ -1 +1 @@
|
|||
9498ab9459048cc595d8e2e411b027d080c0ab0f
|
||||
e4d8f69ffa2efd3f2cb0adff5fa66f367f66f6fb
|
||||
|
|
|
@ -110,7 +110,7 @@ export default {
|
|||
</script>
|
||||
|
||||
<template>
|
||||
<div class="row gl-mt-3 js-preferences-form">
|
||||
<div class="row gl-mt-3 js-preferences-form js-search-settings-section">
|
||||
<div v-if="integrationViews.length" class="col-sm-12">
|
||||
<hr data-testid="profile-preferences-integrations-rule" />
|
||||
</div>
|
||||
|
|
|
@ -0,0 +1,29 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class BackfillEpicCacheCounts < Gitlab::Database::Migration[2.0]
|
||||
MIGRATION = 'BackfillEpicCacheCounts'
|
||||
DELAY_INTERVAL = 2.minutes.to_i
|
||||
BATCH_SIZE = 200
|
||||
MAX_BATCH_SIZE = 1000
|
||||
SUB_BATCH_SIZE = 20
|
||||
|
||||
disable_ddl_transaction!
|
||||
restrict_gitlab_migration gitlab_schema: :gitlab_main
|
||||
|
||||
def up
|
||||
queue_batched_background_migration(
|
||||
MIGRATION,
|
||||
:epics,
|
||||
:id,
|
||||
job_interval: DELAY_INTERVAL,
|
||||
batch_size: BATCH_SIZE,
|
||||
max_batch_size: MAX_BATCH_SIZE,
|
||||
sub_batch_size: SUB_BATCH_SIZE,
|
||||
gitlab_schema: :gitlab_main
|
||||
)
|
||||
end
|
||||
|
||||
def down
|
||||
delete_batched_background_migration(MIGRATION, :epics, :id, [])
|
||||
end
|
||||
end
|
|
@ -0,0 +1,10 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class RemoveAndAddCiPipelineVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0]
|
||||
enable_lock_retries!
|
||||
|
||||
def change
|
||||
remove_column :ci_pipeline_variables, :raw, :boolean, null: false, default: true
|
||||
add_column :ci_pipeline_variables, :raw, :boolean, null: false, default: false
|
||||
end
|
||||
end
|
|
@ -0,0 +1,10 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class RemoveAndAddCiGroupVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0]
|
||||
enable_lock_retries!
|
||||
|
||||
def change
|
||||
remove_column :ci_group_variables, :raw, :boolean, null: false, default: true
|
||||
add_column :ci_group_variables, :raw, :boolean, null: false, default: false
|
||||
end
|
||||
end
|
|
@ -0,0 +1,10 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class RemoveAndAddCiInstanceVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0]
|
||||
enable_lock_retries!
|
||||
|
||||
def change
|
||||
remove_column :ci_instance_variables, :raw, :boolean, null: false, default: true
|
||||
add_column :ci_instance_variables, :raw, :boolean, null: false, default: false
|
||||
end
|
||||
end
|
|
@ -0,0 +1,10 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class RemoveAndAddCiJobVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0]
|
||||
enable_lock_retries!
|
||||
|
||||
def change
|
||||
remove_column :ci_job_variables, :raw, :boolean, null: false, default: true
|
||||
add_column :ci_job_variables, :raw, :boolean, null: false, default: false
|
||||
end
|
||||
end
|
|
@ -0,0 +1,10 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class RemoveAndAddCiPipelineScheduleVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0]
|
||||
enable_lock_retries!
|
||||
|
||||
def change
|
||||
remove_column :ci_pipeline_schedule_variables, :raw, :boolean, null: false, default: true
|
||||
add_column :ci_pipeline_schedule_variables, :raw, :boolean, null: false, default: false
|
||||
end
|
||||
end
|
|
@ -0,0 +1,10 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class RemoveAndAddCiVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0]
|
||||
enable_lock_retries!
|
||||
|
||||
def change
|
||||
remove_column :ci_variables, :raw, :boolean, null: false, default: true
|
||||
add_column :ci_variables, :raw, :boolean, null: false, default: false
|
||||
end
|
||||
end
|
|
@ -0,0 +1 @@
|
|||
f8196de8a4c8f6e8c6790c0d741b0deb455c533a35f665fffeb70c833d0ecd29
|
|
@ -0,0 +1 @@
|
|||
f06d7555d3541abbb9fd671df3718645203aef111674b91ad386dac434c4ede2
|
|
@ -0,0 +1 @@
|
|||
a515d3e34f4b467c5e6fbd9b55135a676277ff6388eb1e3fc14df4b11d8eb3c0
|
|
@ -0,0 +1 @@
|
|||
0fdbb888bdbd4f68619466a8f4384e44062b0cf854790c648a6a060ab1e71806
|
|
@ -0,0 +1 @@
|
|||
2041cb81c534be0cc45b1cc77fa7fd2e31615129e1ba60a146cca88d58b77605
|
|
@ -0,0 +1 @@
|
|||
cbef1d036abb0035d710cf912e554e32fa88df3abaed17cb938e0b18032c7448
|
|
@ -0,0 +1 @@
|
|||
fbc4c12aedd9d0f183e9444f2cb42c11a2b894c11684e80a5dbe847c7bccb21f
|
|
@ -12772,7 +12772,7 @@ CREATE TABLE ci_group_variables (
|
|||
masked boolean DEFAULT false NOT NULL,
|
||||
variable_type smallint DEFAULT 1 NOT NULL,
|
||||
environment_scope text DEFAULT '*'::text NOT NULL,
|
||||
raw boolean DEFAULT true NOT NULL,
|
||||
raw boolean DEFAULT false NOT NULL,
|
||||
CONSTRAINT check_dfe009485a CHECK ((char_length(environment_scope) <= 255))
|
||||
);
|
||||
|
||||
|
@ -12793,7 +12793,7 @@ CREATE TABLE ci_instance_variables (
|
|||
key text NOT NULL,
|
||||
encrypted_value text,
|
||||
encrypted_value_iv text,
|
||||
raw boolean DEFAULT true NOT NULL,
|
||||
raw boolean DEFAULT false NOT NULL,
|
||||
CONSTRAINT check_07a45a5bcb CHECK ((char_length(encrypted_value_iv) <= 255)),
|
||||
CONSTRAINT check_5aede12208 CHECK ((char_length(key) <= 255)),
|
||||
CONSTRAINT check_956afd70f1 CHECK ((char_length(encrypted_value) <= 13579))
|
||||
|
@ -12875,7 +12875,7 @@ CREATE TABLE ci_job_variables (
|
|||
job_id bigint NOT NULL,
|
||||
variable_type smallint DEFAULT 1 NOT NULL,
|
||||
source smallint DEFAULT 0 NOT NULL,
|
||||
raw boolean DEFAULT true NOT NULL
|
||||
raw boolean DEFAULT false NOT NULL
|
||||
);
|
||||
|
||||
CREATE SEQUENCE ci_job_variables_id_seq
|
||||
|
@ -13057,7 +13057,7 @@ CREATE TABLE ci_pipeline_schedule_variables (
|
|||
created_at timestamp with time zone,
|
||||
updated_at timestamp with time zone,
|
||||
variable_type smallint DEFAULT 1 NOT NULL,
|
||||
raw boolean DEFAULT true NOT NULL
|
||||
raw boolean DEFAULT false NOT NULL
|
||||
);
|
||||
|
||||
CREATE SEQUENCE ci_pipeline_schedule_variables_id_seq
|
||||
|
@ -13101,8 +13101,8 @@ CREATE TABLE ci_pipeline_variables (
|
|||
encrypted_value_iv character varying,
|
||||
pipeline_id integer NOT NULL,
|
||||
variable_type smallint DEFAULT 1 NOT NULL,
|
||||
raw boolean DEFAULT true NOT NULL,
|
||||
partition_id bigint DEFAULT 100 NOT NULL
|
||||
partition_id bigint DEFAULT 100 NOT NULL,
|
||||
raw boolean DEFAULT false NOT NULL
|
||||
);
|
||||
|
||||
CREATE SEQUENCE ci_pipeline_variables_id_seq
|
||||
|
@ -13564,7 +13564,7 @@ CREATE TABLE ci_variables (
|
|||
environment_scope character varying DEFAULT '*'::character varying NOT NULL,
|
||||
masked boolean DEFAULT false NOT NULL,
|
||||
variable_type smallint DEFAULT 1 NOT NULL,
|
||||
raw boolean DEFAULT true NOT NULL
|
||||
raw boolean DEFAULT false NOT NULL
|
||||
);
|
||||
|
||||
CREATE SEQUENCE ci_variables_id_seq
|
||||
|
|
|
@ -94,7 +94,7 @@ data that can be shared between job runs.
|
|||
Because there is no viable replacement and we might be unable to support all
|
||||
cloud providers that Docker Machine used to support, the key design requirement
|
||||
is to make it really simple and easy for the wider community to write a custom
|
||||
GitLab auto-scaling plugin, whatever cloud provider they might be using. We
|
||||
GitLab plugin for whatever cloud provider they might be using. We
|
||||
want to design a simple abstraction that users will be able to build on top, as
|
||||
will we to support existing workflows on GitLab.com.
|
||||
|
||||
|
@ -129,12 +129,11 @@ the need of rebuilding GitLab Runner whenever it happens.
|
|||
|
||||
### 💡 Write a solid documentation about how to build your own plugin
|
||||
|
||||
It is important to show users how to build an auto-scaling plugin, so that they
|
||||
It is important to show users how to build a plugin, so that they
|
||||
can implement support for their own cloud infrastructure.
|
||||
|
||||
Building new plugins should be simple, and with the support of great
|
||||
documentation it should not require advanced skills, like understanding how
|
||||
gRPC works. We want to design the plugin system in a way that the entry barrier
|
||||
Building new plugins should be simple and supported with great
|
||||
documentation. We want to design the plugin system in a way that the entry barrier
|
||||
for contributing new plugins is very low.
|
||||
|
||||
### 💡 Build a PoC to run multiple builds on a single machine
|
||||
|
@ -171,38 +170,65 @@ configures the Docker daemon there to allow external authenticated requests. It
|
|||
stores credentials to such ephemeral Docker environments on disk. Once a
|
||||
machine has been provisioned and made available for GitLab Runner Manager to
|
||||
run builds, it is using one of the existing executors to run a user-provided
|
||||
script. In auto-scaling, this is typically done using Docker executor.
|
||||
script. In auto-scaling, this is typically done using the Docker executor.
|
||||
|
||||
### Custom provider
|
||||
### Separation of concerns
|
||||
|
||||
In order to reduce the scope of work, we only want to introduce the new
|
||||
abstraction layer in one place.
|
||||
There are several concerns represented in the current architecture. They are
|
||||
coupled in the current implementation so we will break them out here to consider
|
||||
them each separately.
|
||||
|
||||
A few years ago we introduced the [Custom Executor](https://docs.gitlab.com/runner/executors/custom.html)
|
||||
feature in GitLab Runner. It allows users to design custom build execution
|
||||
methods. The custom executor driver can be implemented in any way - from a
|
||||
simple shell script to a dedicated binary - that is then used by a Runner
|
||||
through os/exec system calls.
|
||||
1. **Virtual Machine (VM) shape**. The underlying provider of a VM requires configuration to
|
||||
know what kind of machine to create. E.g. Cores, memory, failure domain,
|
||||
etc... This information is very provider specific.
|
||||
1. **VM lifecycle management**. Multiple machines will be created and a
|
||||
system must keep track of which machines belong to this executor. Typically
|
||||
a cloud provider will have a way to manage a set of homogenous machines.
|
||||
E.g. GCE Instance Group. The basic operations are increase, decrease and
|
||||
usually delete a specific machine.
|
||||
1. **VM autoscaling**. In addition to low-level lifecycle management,
|
||||
job-aware capacity decisions must be made to the set of machines to provide
|
||||
capacity when it is needed but not maintain excess capacity for cost reasons.
|
||||
1. **Job to VM mapping (routing)**. Currently the system assigns only one job to a
|
||||
given a machine. A machine may be reused based on the specific executor
|
||||
configuration.
|
||||
1. **In-VM job execution**. Within each VM a job must be driven through
|
||||
various pre-defined stages and results and trace information returned
|
||||
to the Runner system. These details are highly dependent on the VM
|
||||
architecture and operating system as well as Executor type.
|
||||
|
||||
Thanks to the custom executor abstraction there is no more need to implement
|
||||
new executors internally in Runner. Users who have specific needs can implement
|
||||
their own drivers and don't need to wait for us to make their work part of the
|
||||
"official" GitLab Runner. As each driver is a separate project, it also makes
|
||||
it easier to create communities around them, where interested people can
|
||||
collaborate together on improvements and bug fixes.
|
||||
The current architecture has several points of coupling between concerns.
|
||||
Coupling reduces opportunities for abstraction (e.g. community supported
|
||||
plugins) and increases complexity, making the code harder to understand,
|
||||
test, maintain and extend.
|
||||
|
||||
We want to design the new Custom Provider to replicate the success of the
|
||||
Custom Executor. It will make it easier for users to build their own ways to
|
||||
provide a context and an environment in which a build will be executed by one
|
||||
of the Custom Executors.
|
||||
A primary design decision will be which concerns to externalize to the plugin
|
||||
and which should remain with the runner system. The current implementation
|
||||
has several abstractions internally which could be used as cut points for a
|
||||
new abstraction.
|
||||
|
||||
There are multiple solutions to implementing a custom provider abstraction. We
|
||||
can use raw Go plugins, Hashcorp's Go Plugin, HTTP interface or gRPC based
|
||||
facade service. There are many solutions, and we want to choose the most
|
||||
optimal one. In order to do that, we will describe the solutions in a separate
|
||||
document, define requirements and score the solution accordingly. This will
|
||||
allow us to choose a solution that will work best for us and the wider
|
||||
community.
|
||||
For example the [`Build`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/build.go#L125)
|
||||
type uses the [`GetExecutorProvider`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/executor.go#L171)
|
||||
function to get an executor provider based on a dispatching executor string.
|
||||
Various executor types register with the system by being imported and calling
|
||||
[`RegisterExecutorProvider`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/executor.go#L154)
|
||||
during initialization. Here the abstractions are the [`ExecutorProvider`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/executor.go#L80)
|
||||
and [`Executor`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/executor.go#L59)
|
||||
interfaces.
|
||||
|
||||
Within the `docker+autoscaling` executor the [`machineExecutor`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/executors/docker/machine/machine.go#L19)
|
||||
type has a [`Machine`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/helpers/docker/machine.go#L7)
|
||||
interface which it uses to aquire a VM during the common [`Prepare`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/executors/docker/machine/machine.go#L71)
|
||||
phase. This abstraction primarily creates, accesses and deletes VMs.
|
||||
|
||||
There is no current abstraction for the VM autoscaling logic. It is tightly
|
||||
coupled with the VM lifecycle and job routing logic. Creating idle capacity
|
||||
happens as a side-effect of calling [`Acquire`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/executors/docker/machine/provider.go#L449) on the `machineProvider` while binding a job to a VM.
|
||||
|
||||
There is also no current abstraction for in-VM job execution. VM-specific
|
||||
commands are generated by the Runner Manager using the [`GenerateShellScript`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/build.go#L336)
|
||||
function and [injected](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/build.go#L373)
|
||||
into the VM as the manager drives the job execution stages.
|
||||
|
||||
### Design principles
|
||||
|
||||
|
@ -235,12 +261,84 @@ abstraction.
|
|||
1. Build an abstraction that serves our community well but allows us to ship it quickly.
|
||||
1. Invest in a flexible solution, avoid one-way-door decisions, foster iteration.
|
||||
1. When in doubts err on the side of making things more simple for the wider community.
|
||||
1. Limit coupling between concerns in order to make the system more simple and extensible.
|
||||
1. Concerns should live on one side of the plug or the other--not both, which
|
||||
duplicates effort and increases coupling.
|
||||
|
||||
#### The most important technical details
|
||||
|
||||
1. Favor gRPC communication between a plugin and GitLab Runner.
|
||||
1. Make it possible to version communication interface and support many versions.
|
||||
1. Make Go a primary language for writing plugins but accept other languages too.
|
||||
1. Prefer a GitLab job-aware autoscaler to provider specific autoscalers. Cloud provider
|
||||
autoscalers don't know which VM to delete when scaling down so they make sub-optimal
|
||||
decisions. Rather than teaching all autoscalers about GitLab jobs, we prefer to
|
||||
have one, GitLab-owned autoscaler (not in the plugin).
|
||||
|
||||
## Plugin boundary proposals
|
||||
|
||||
The following are proposals for where to draw the plugin boundary. We will evaluate
|
||||
these proposals and others by the design principles and technical constraints
|
||||
listed above.
|
||||
|
||||
### Custom provider
|
||||
|
||||
In order to reduce the scope of work, we only want to introduce the new
|
||||
abstraction layer in one place.
|
||||
|
||||
A few years ago we introduced the [Custom Executor](https://docs.gitlab.com/runner/executors/custom.html)
|
||||
feature in GitLab Runner. It allows users to design custom build execution
|
||||
methods. The custom executor driver can be implemented in any way - from a
|
||||
simple shell script to a dedicated binary - that is then used by a Runner
|
||||
through os/exec system calls.
|
||||
|
||||
Thanks to the custom executor abstraction there is no more need to implement
|
||||
new executors internally in Runner. Users who have specific needs can implement
|
||||
their own drivers and don't need to wait for us to make their work part of the
|
||||
"official" GitLab Runner. As each driver is a separate project, it also makes
|
||||
it easier to create communities around them, where interested people can
|
||||
collaborate together on improvements and bug fixes.
|
||||
|
||||
We want to design the new Custom Provider to replicate the success of the
|
||||
Custom Executor. It will make it easier for users to build their own ways to
|
||||
provide a context and an environment in which a build will be executed by one
|
||||
of the Custom Executors.
|
||||
|
||||
There are multiple solutions to implementing a custom provider abstraction. We
|
||||
can use raw Go plugins, Hashcorp's Go Plugin, HTTP interface or gRPC based
|
||||
facade service. There are many solutions, and we want to choose the most
|
||||
optimal one. In order to do that, we will describe the solutions in a separate
|
||||
document, define requirements and score the solution accordingly. This will
|
||||
allow us to choose a solution that will work best for us and the wider
|
||||
community.
|
||||
|
||||
This proposal places VM lifecycle and autoscaling concerns as well as job to
|
||||
VM mapping (routing) into the plugin. The build need only ask for a VM and
|
||||
it will get one with all aspects of lifecycle and routing already accounted
|
||||
for by the plugin.
|
||||
|
||||
Rationale: [Description of the Custom Executor Provider proposal](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/28848#note_823321515)
|
||||
|
||||
### Fleeting VM provider
|
||||
|
||||
We can introduce a more simple version of the `Machine` abstraction in the
|
||||
form of a "Fleeting" interface. Fleeting provides a low-level interface to
|
||||
a homogenous VM group which allows increasing and decreasing the set size
|
||||
as well as consuming a VM from within the set.
|
||||
|
||||
Plugins for cloud providers and other VM sources are implemented via the
|
||||
Hashicorp go-plugin library. This is in practice gRPC over STDIN/STDOUT
|
||||
but other wire protocols can be used also.
|
||||
|
||||
In order to make use of the new interface, the autoscaling logic is pulled
|
||||
out of the Docker Executor and placed into a new Taskscaler library.
|
||||
|
||||
This places the concerns of VM lifecycle, VM shape and job routing within
|
||||
the plugin. It also places the conern of VM autoscaling into a separate
|
||||
component so it can be used by multiple Runner Executors (not just `docker+autoscaling`).
|
||||
|
||||
Rationale: [Description of the InstanceGroup / Fleeting proposal](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/28848#note_823430883)
|
||||
POC: [Merge request](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3315)
|
||||
|
||||
## Status
|
||||
|
||||
|
@ -253,12 +351,12 @@ Proposal:
|
|||
<!-- vale gitlab.Spelling = NO -->
|
||||
|
||||
| Role | Who
|
||||
|------------------------------|------------------------------------------|
|
||||
| Authors | Grzegorz Bizon, Tomasz Maczukin |
|
||||
| Architecture Evolution Coach | Kamil Trzciński |
|
||||
| Engineering Leader | Elliot Rushton, Cheryl Li |
|
||||
| Product Manager | Darren Eastman, Jackie Porter |
|
||||
| Domain Expert / Runner | Arran Walker |
|
||||
|------------------------------|-------------------------------------------------|
|
||||
| Authors | Grzegorz Bizon, Tomasz Maczukin, Joseph Burnett |
|
||||
| Architecture Evolution Coach | Kamil Trzciński |
|
||||
| Engineering Leader | Elliot Rushton, Cheryl Li |
|
||||
| Product Manager | Darren Eastman, Jackie Porter |
|
||||
| Domain Expert / Runner | Arran Walker |
|
||||
|
||||
DRIs:
|
||||
|
||||
|
|
|
@ -70,7 +70,7 @@ To simplify administration, we recommend that a GitLab group maintainer or group
|
|||
| Jira usage | GitLab.com customers need | GitLab self-managed customers need |
|
||||
|------------|---------------------------|------------------------------------|
|
||||
| [Atlassian cloud](https://www.atlassian.com/migration/assess/why-cloud) | The [GitLab.com for Jira Cloud app](https://marketplace.atlassian.com/apps/1221011/gitlab-com-for-jira-cloud?hosting=cloud&tab=overview) installed from the [Atlassian Marketplace](https://marketplace.atlassian.com). This offers real-time sync between GitLab.com and Jira. For more information, see the documentation for the [GitLab.com for Jira Cloud app](connect-app.md). | The [GitLab.com for Jira Cloud app](https://marketplace.atlassian.com/apps/1221011/gitlab-com-for-jira-cloud?hosting=cloud&tab=overview), using a workaround process. See the documentation for [installing the GitLab.com for Jira Cloud app for self-managed instances](connect-app.md#install-the-gitlabcom-for-jira-cloud-app-for-self-managed-instances) for more information. |
|
||||
| Your own server | The [Jira DVCS (distributed version control system) connector](dvcs.md). This syncs data hourly. | The [Jira DVCS Connector](dvcs.md). |
|
||||
| Your own server | The [Jira DVCS (distributed version control system) connector](dvcs.md). This syncs data hourly. | The [Jira DVCS (distributed version control system) connector](dvcs.md). This syncs data hourly. |
|
||||
|
||||
Each GitLab project can be configured to connect to an entire Jira instance. That means after
|
||||
configuration, one GitLab project can interact with all Jira projects in that instance. For:
|
||||
|
|
|
@ -26,36 +26,24 @@ For an introduction to Auto DevOps, watch [Auto DevOps in GitLab 11.0](https://y
|
|||
|
||||
## Auto DevOps features
|
||||
|
||||
Based on the DevOps [stages](stages.md), use Auto DevOps to:
|
||||
Auto DevOps supports development during each of the [DevOps stages](stages.md).
|
||||
|
||||
**Build your app:**
|
||||
|
||||
- [Auto Build](stages.md#auto-build)
|
||||
- [Auto Dependency Scanning](stages.md#auto-dependency-scanning)
|
||||
|
||||
**Test your app:**
|
||||
|
||||
- [Auto Test](stages.md#auto-test)
|
||||
- [Auto Browser Performance Testing](stages.md#auto-browser-performance-testing)
|
||||
- [Auto Code Intelligence](stages.md#auto-code-intelligence)
|
||||
- [Auto Code Quality](stages.md#auto-code-quality)
|
||||
- [Auto Container Scanning](stages.md#auto-container-scanning)
|
||||
- [Auto License Compliance](stages.md#auto-license-compliance)
|
||||
|
||||
**Deploy your app:**
|
||||
|
||||
- [Auto Review Apps](stages.md#auto-review-apps)
|
||||
- [Auto Deploy](stages.md#auto-deploy)
|
||||
|
||||
**Monitor your app:**
|
||||
|
||||
- [Auto Monitoring](stages.md#auto-monitoring)
|
||||
|
||||
**Secure your app:**
|
||||
|
||||
- [Auto Dynamic Application Security Testing (DAST)](stages.md#auto-dast)
|
||||
- [Auto Static Application Security Testing (SAST)](stages.md#auto-sast)
|
||||
- [Auto Secret Detection](stages.md#auto-secret-detection)
|
||||
| Stage | Auto DevOps feature |
|
||||
|---------|-------------|
|
||||
| Build | [Auto Build](stages.md#auto-build) |
|
||||
| Build | [Auto Dependency Scanning](stages.md#auto-dependency-scanning) |
|
||||
| Test | [Auto Test](stages.md#auto-test) |
|
||||
| Test | [Auto Browser Performance Testing](stages.md#auto-browser-performance-testing) |
|
||||
| Test | [Auto Code Intelligence](stages.md#auto-code-intelligence) |
|
||||
| Test | [Auto Code Quality](stages.md#auto-code-quality) |
|
||||
| Test | [Auto Container Scanning](stages.md#auto-container-scanning) |
|
||||
| Test | [Auto License Compliance](stages.md#auto-license-compliance) |
|
||||
| Deploy | [Auto Review Apps](stages.md#auto-review-apps) |
|
||||
| Deploy | [Auto Deploy](stages.md#auto-deploy) |
|
||||
| Monitor | [Auto Monitoring](stages.md#auto-monitoring) |
|
||||
| Secure | [Auto Dynamic Application Security Testing (DAST)](stages.md#auto-dast) |
|
||||
| Secure | [Auto Static Application Security Testing (SAST)](stages.md#auto-sast) |
|
||||
| Secure | [Auto Secret Detection](stages.md#auto-secret-detection) |
|
||||
|
||||
### Comparison to application platforms and PaaS
|
||||
|
||||
|
@ -80,24 +68,12 @@ If you want to build, test, and deploy your app:
|
|||
|
||||
1. View the [requirements for deployment](requirements.md).
|
||||
1. [Enable Auto DevOps](#enable-or-disable-auto-devops).
|
||||
1. Follow the [quick start guide](#quick-start).
|
||||
|
||||
As Auto DevOps relies on many components, be familiar with:
|
||||
|
||||
- [Continuous methodologies](../../ci/introduction/index.md)
|
||||
- [Docker](https://docs.docker.com)
|
||||
- [GitLab Runner](https://docs.gitlab.com/runner/)
|
||||
|
||||
When deploying to a Kubernetes cluster make sure you're also familiar with:
|
||||
|
||||
- [Kubernetes](https://kubernetes.io/docs/home/)
|
||||
- [Helm](https://helm.sh/docs/)
|
||||
- [Prometheus](https://prometheus.io/docs/introduction/overview/)
|
||||
1. [Deploy your app to a cloud provider](#deploy-your-app-to-a-cloud-provider).
|
||||
|
||||
### Enable or disable Auto DevOps
|
||||
|
||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41729) in GitLab 11.3, Auto DevOps is enabled by default.
|
||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26655) GitLab 12.7, Auto DevOps runs pipelines automatically only if a [`Dockerfile` or matching buildpack](stages.md#auto-build) exists.
|
||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26655) in GitLab 12.7, Auto DevOps runs pipelines automatically only if a [`Dockerfile` or matching buildpack](stages.md#auto-build) exists.
|
||||
|
||||
Depending on your instance type, you can enable or disable Auto DevOps at the
|
||||
following levels:
|
||||
|
@ -116,7 +92,7 @@ To use Auto DevOps for individual projects, you can enable it in a
|
|||
project-by-project basis. If you intend to use it for more projects,
|
||||
you can enable it for a [group](#at-the-group-level) or an
|
||||
[instance](#at-the-instance-level). This can save you the time of
|
||||
enabling it one by one.
|
||||
enabling it in each project.
|
||||
|
||||
Prerequisites:
|
||||
|
||||
|
@ -143,9 +119,10 @@ To disable it, follow the same process and clear the
|
|||
|
||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/52447) in GitLab 11.10.
|
||||
|
||||
When you enable Auto DevOps at group level, the subgroups and projects in that
|
||||
group inherit the configuration. This saves you some time by batch-enabling it
|
||||
rather than enabling individually for each subgroup or project.
|
||||
When you enable Auto DevOps at the group level, the subgroups and
|
||||
projects in that group inherit the configuration. You can save time by
|
||||
enabling Auto DevOps for a group instead of enabling it for each
|
||||
subgroup or project.
|
||||
|
||||
When enabled for a group, you can still disable Auto DevOps
|
||||
for the subgroups and projects where you don't want to use it.
|
||||
|
@ -162,7 +139,7 @@ To enable Auto DevOps for a group:
|
|||
1. Select the **Default to Auto DevOps pipeline** checkbox.
|
||||
1. Select **Save changes**.
|
||||
|
||||
To disable Auto DevOps on the group level, follow the same process and
|
||||
To disable Auto DevOps at the group level, follow the same process and
|
||||
clear the **Default to Auto DevOps pipeline** checkbox.
|
||||
|
||||
After enabling Auto DevOps at the group level, you can trigger the
|
||||
|
@ -175,10 +152,9 @@ Auto DevOps pipeline for any project that belongs to that group:
|
|||
|
||||
#### At the instance level **(FREE SELF)**
|
||||
|
||||
By enabling Auto DevOps in the instance level, all projects created in that
|
||||
instance become enabled. This is convenient when you want to run Auto DevOps by
|
||||
default for all projects. You can still disable Auto DevOps individually for
|
||||
the groups and projects where you don't want to run it.
|
||||
To enable Auto DevOps by default for all projects, you can enable it at the instance level.
|
||||
You can still disable Auto DevOps for each group and project
|
||||
where you don't want to run it.
|
||||
|
||||
Even when disabled for an instance, group Owners and project Maintainers
|
||||
can still enable Auto DevOps at the group and project levels.
|
||||
|
@ -196,24 +172,17 @@ To enable Auto DevOps for your instance:
|
|||
1. Optional. Add the Auto DevOps [base domain](requirements.md#auto-devops-base-domain).
|
||||
1. Select **Save changes**.
|
||||
|
||||
When enabled, it attempts to run Auto DevOps pipelines in every project. If the
|
||||
When enabled, Auto DevOps attempts to run pipelines in every project. If the
|
||||
pipeline fails in a particular project, it disables itself.
|
||||
GitLab administrators can change this in the [Auto DevOps settings](../../user/admin_area/settings/continuous_integration.md#auto-devops).
|
||||
|
||||
If a [CI/CD configuration file](../../ci/yaml/index.md) is present,
|
||||
it remains unchanged and Auto DevOps doesn't affect it.
|
||||
it remains unchanged and Auto DevOps does not affect it.
|
||||
|
||||
To disable Auto DevOps in the instance level, follow the same process
|
||||
and clear the **Default to Auto DevOps pipeline** checkbox.
|
||||
|
||||
### Private registry support
|
||||
|
||||
There is no guarantee that you can use a private container registry with Auto DevOps.
|
||||
|
||||
Instead, use the [GitLab Container Registry](../../user/packages/container_registry/index.md) with Auto DevOps to
|
||||
simplify configuration and prevent any unforeseen issues.
|
||||
|
||||
### Quick start
|
||||
### Deploy your app to a cloud provider
|
||||
|
||||
- [Use Auto DevOps to deploy to a Kubernetes cluster on Google Kubernetes Engine (GKE)](cloud_deployments/auto_devops_with_gke.md)
|
||||
- [Use Auto DevOps to deploy to EC2](cloud_deployments/auto_devops_with_ec2.md)
|
||||
|
@ -221,7 +190,7 @@ simplify configuration and prevent any unforeseen issues.
|
|||
|
||||
## Upgrade Auto DevOps dependencies when updating GitLab
|
||||
|
||||
When updating GitLab, you may need to upgrade Auto DevOps dependencies to
|
||||
When updating GitLab, you might need to upgrade Auto DevOps dependencies to
|
||||
match your new GitLab version:
|
||||
|
||||
- [Upgrading Auto DevOps resources](upgrading_auto_deploy_dependencies.md):
|
||||
|
@ -233,30 +202,29 @@ match your new GitLab version:
|
|||
- Environment variables.
|
||||
- [Upgrading PostgreSQL](upgrading_postgresql.md).
|
||||
|
||||
## Private registry support
|
||||
|
||||
There is no guarantee that you can use a private container registry with Auto DevOps.
|
||||
|
||||
Instead, use the [GitLab Container Registry](../../user/packages/container_registry/index.md) with Auto DevOps to
|
||||
simplify configuration and prevent any unforeseen issues.
|
||||
|
||||
## Install applications behind a proxy
|
||||
|
||||
The GitLab integration with Helm does not support installing applications when
|
||||
behind a proxy.
|
||||
|
||||
To do so, inject proxy settings into the installation pods at runtime.
|
||||
For example, you can use a `PodPreset`:
|
||||
If you want to do so, you must inject proxy settings into the
|
||||
installation pods at runtime.
|
||||
|
||||
NOTE:
|
||||
[PodPreset was removed in Kubernetes v1.20](https://github.com/kubernetes/kubernetes/pull/94090).
|
||||
## Related topics
|
||||
|
||||
```yaml
|
||||
apiVersion: settings.k8s.io/v1alpha1
|
||||
kind: PodPreset
|
||||
metadata:
|
||||
name: gitlab-managed-apps-default-proxy
|
||||
namespace: gitlab-managed-apps
|
||||
spec:
|
||||
env:
|
||||
- name: http_proxy
|
||||
value: "PUT_YOUR_HTTP_PROXY_HERE"
|
||||
- name: https_proxy
|
||||
value: "PUT_YOUR_HTTPS_PROXY_HERE"
|
||||
```
|
||||
- [Continuous methodologies](../../ci/introduction/index.md)
|
||||
- [Docker](https://docs.docker.com)
|
||||
- [GitLab Runner](https://docs.gitlab.com/runner/)
|
||||
- [Helm](https://helm.sh/docs/)
|
||||
- [Kubernetes](https://kubernetes.io/docs/home/)
|
||||
- [Prometheus](https://prometheus.io/docs/introduction/overview/)
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
|
|
|
@ -0,0 +1,15 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module Gitlab
|
||||
module BackgroundMigration
|
||||
# rubocop: disable Style/Documentation
|
||||
class BackfillEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob
|
||||
def perform; end
|
||||
end
|
||||
# rubocop: enable Style/Documentation
|
||||
end
|
||||
end
|
||||
|
||||
# rubocop: disable Layout/LineLength
|
||||
Gitlab::BackgroundMigration::BackfillEpicCacheCounts.prepend_mod_with('Gitlab::BackgroundMigration::BackfillEpicCacheCounts')
|
||||
# rubocop: enable Layout/LineLength
|
|
@ -0,0 +1,16 @@
|
|||
# RuboCop configuration adjustments during the transition time from Ruby 2.7 to Ruby 3.0.
|
||||
# This configuration should be removed after the transition has been completed.
|
||||
|
||||
# Disable cops for now since their behavior changed in Ruby 3.0.
|
||||
# See https://gitlab.com/gitlab-org/gitlab/-/jobs/3068345492
|
||||
#
|
||||
# Migration plan:
|
||||
# * Generate TODOs for these cops (with Ruby 3.0) right before the switch to Ruby 3.0
|
||||
# * Put these cops back in "grace period" to ensure `master` stability
|
||||
# * Remove "grace period" after the switch
|
||||
# * Incrementally fix TODOs
|
||||
#
|
||||
Style/MutableConstant:
|
||||
Enabled: false
|
||||
Style/RedundantFreeze:
|
||||
Enabled: false
|
|
@ -0,0 +1,32 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'spec_helper'
|
||||
require_migration!
|
||||
|
||||
RSpec.describe BackfillEpicCacheCounts, :migration do
|
||||
let(:migration) { described_class::MIGRATION }
|
||||
|
||||
describe '#up' do
|
||||
it 'schedules a batched background migration' do
|
||||
migrate!
|
||||
|
||||
expect(migration).to have_scheduled_batched_migration(
|
||||
table_name: :epics,
|
||||
column_name: :id,
|
||||
interval: described_class::DELAY_INTERVAL,
|
||||
batch_size: described_class::BATCH_SIZE,
|
||||
max_batch_size: described_class::MAX_BATCH_SIZE,
|
||||
sub_batch_size: described_class::SUB_BATCH_SIZE
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
describe '#down' do
|
||||
it 'deletes all batched migration records' do
|
||||
migrate!
|
||||
schema_migrate_down!
|
||||
|
||||
expect(migration).not_to have_scheduled_batched_migration
|
||||
end
|
||||
end
|
||||
end
|
|
@ -69,6 +69,8 @@ RSpec.describe WebHooks::LogExecutionService do
|
|||
subject(:service) { described_class.new(hook: project_hook, log_data: data, response_category: response_category) }
|
||||
|
||||
before do
|
||||
# stub LOCK_RETRY to be 0 in order for tests to run quicker
|
||||
stub_const("#{described_class.name}::LOCK_RETRY", 0)
|
||||
stub_exclusive_lease_taken(lease_key, timeout: described_class::LOCK_TTL)
|
||||
allow(project_hook).to receive(:executable?).and_return(executable)
|
||||
end
|
||||
|
|
Loading…
Reference in New Issue