Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-09-20 15:13:52 +00:00
parent 027f19b39c
commit b962adf4c3
31 changed files with 349 additions and 162 deletions

View File

@ -78,6 +78,9 @@ e2e:package-and-test:
SKIP_MESSAGE: Skipping package-and-test due to mr containing only quarantine changes!
trigger:
strategy: depend
forward:
yaml_variables: true
pipeline_variables: true
include:
- artifact: package-and-test-pipeline.yml
job: e2e-test-pipeline-generate

View File

@ -1862,7 +1862,6 @@ Layout/LineLength:
- 'ee/spec/controllers/registrations/welcome_controller_spec.rb'
- 'ee/spec/controllers/subscriptions/groups_controller_spec.rb'
- 'ee/spec/controllers/subscriptions_controller_spec.rb'
- 'ee/spec/controllers/trials_controller_spec.rb'
- 'ee/spec/elastic/migrate/20210201104800_migrate_notes_to_separate_index_spec.rb'
- 'ee/spec/elastic/migrate/20210421140400_add_new_data_to_merge_requests_documents_spec.rb'
- 'ee/spec/elastic/migrate/20210429154500_migrate_merge_requests_to_separate_index_spec.rb'

View File

@ -252,7 +252,6 @@ Rails/SkipsModelValidations:
- 'ee/spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb'
- 'ee/spec/controllers/groups/ldaps_controller_spec.rb'
- 'ee/spec/controllers/projects/merge_requests_controller_spec.rb'
- 'ee/spec/controllers/trials_controller_spec.rb'
- 'ee/spec/factories/import_states.rb'
- 'ee/spec/features/admin/admin_settings_spec.rb'
- 'ee/spec/features/epic_boards/epic_boards_sidebar_spec.rb'

View File

@ -13,7 +13,6 @@ RSpec/AnyInstanceOf:
- ee/spec/controllers/projects/path_locks_controller_spec.rb
- ee/spec/controllers/projects_controller_spec.rb
- ee/spec/controllers/subscriptions_controller_spec.rb
- ee/spec/controllers/trials_controller_spec.rb
- ee/spec/features/admin/admin_audit_logs_spec.rb
- ee/spec/features/admin/admin_reset_pipeline_minutes_spec.rb
- ee/spec/features/admin/admin_users_spec.rb

View File

@ -70,7 +70,6 @@ RSpec/ContextWording:
- 'ee/spec/controllers/security/vulnerabilities_controller_spec.rb'
- 'ee/spec/controllers/subscriptions_controller_spec.rb'
- 'ee/spec/controllers/trial_registrations_controller_spec.rb'
- 'ee/spec/controllers/trials_controller_spec.rb'
- 'ee/spec/controllers/users_controller_spec.rb'
- 'ee/spec/elastic/migrate/20220119120500_populate_commit_permissions_in_main_index_spec.rb'
- 'ee/spec/elastic/migrate/20220512150000_pause_indexing_for_unsupported_es_versions_spec.rb'

View File

@ -7,7 +7,6 @@ RSpec/ExpectInHook:
- 'ee/spec/controllers/projects/boards_controller_spec.rb'
- 'ee/spec/controllers/projects/settings/slacks_controller_spec.rb'
- 'ee/spec/controllers/subscriptions_controller_spec.rb'
- 'ee/spec/controllers/trials_controller_spec.rb'
- 'ee/spec/elastic/migrate/20220118150500_delete_orphaned_commits_spec.rb'
- 'ee/spec/elastic/migrate/20220119120500_populate_commit_permissions_in_main_index_spec.rb'
- 'ee/spec/features/billings/billing_plans_spec.rb'
@ -19,7 +18,6 @@ RSpec/ExpectInHook:
- 'ee/spec/features/registrations/trial_during_signup_flow_spec.rb'
- 'ee/spec/features/signup_spec.rb'
- 'ee/spec/features/trial_registrations/company_information_spec.rb'
- 'ee/spec/features/trials/capture_lead_spec.rb'
- 'ee/spec/features/trials/select_namespace_spec.rb'
- 'ee/spec/finders/license_template_finder_spec.rb'
- 'ee/spec/finders/projects/integrations/jira/issues_finder_spec.rb'

View File

@ -205,6 +205,12 @@ export default {
mrPipelinesDocsPath: helpPagePath('ci/pipelines/merge_request_pipelines.md', {
anchor: 'prerequisites',
}),
runPipelinesInTheParentProjectHelpPath: helpPagePath(
'/ci/pipelines/merge_request_pipelines.html',
{
anchor: 'run-pipelines-in-the-parent-project',
},
),
};
</script>
<template>
@ -321,10 +327,7 @@ export default {
s__('Pipelines|If you are unsure, please ask a project maintainer to review it for you.')
}}
</p>
<gl-link
href="/help/ci/pipelines/merge_request_pipelines.html#run-pipelines-in-the-parent-project-for-merge-requests-from-a-forked-project"
target="_blank"
>
<gl-link :href="$options.runPipelinesInTheParentProjectHelpPath" target="_blank">
{{ s__('Pipelines|More Information') }}
</gl-link>
</gl-modal>

View File

@ -1,6 +1,6 @@
import Vue from 'vue';
import { convertObjectPropsToCamelCase } from '~/lib/utils/common_utils';
import { secondsToHours } from '~/lib/utils/datetime_utility';
import { formatUtcOffset } from '~/lib/utils/datetime_utility';
import * as types from './mutation_types';
const formatTimezoneName = (freezePeriod, timezoneList) => {
@ -8,7 +8,7 @@ const formatTimezoneName = (freezePeriod, timezoneList) => {
return convertObjectPropsToCamelCase({
...freezePeriod,
cron_timezone: {
formattedTimezone: tz && `[UTC ${secondsToHours(tz.offset)}] ${tz.name}`,
formattedTimezone: tz && `[UTC ${formatUtcOffset(tz.offset)}] ${tz.name}`,
identifier: freezePeriod.cron_timezone,
},
});

View File

@ -277,15 +277,15 @@ export const secondsToDays = (seconds) => Math.round(seconds / 86400);
*
* @param {Number} offset UTC offset in seconds as a integer
*
* @return {String} the + or - offset in hours
* @return {String} the + or - offset in hours, e.g. `- 10`, `0`, `+ 4`
*/
export const secondsToHours = (offset) => {
export const formatUtcOffset = (offset) => {
const parsed = parseInt(offset, 10);
if (Number.isNaN(parsed) || parsed === 0) {
return `0`;
}
const num = offset / 3600;
return parseInt(num, 10) !== num ? num.toFixed(1) : num;
const prefix = offset > 0 ? '+' : '-';
return `${prefix} ${Math.abs(offset / 3600)}`;
};
/**

View File

@ -3,6 +3,7 @@ import '~/profile/gl_crop';
import Profile from '~/profile/profile';
import initSearchSettings from '~/search_settings';
import initPasswordPrompt from './password_prompt';
import { initTimezoneDropdown } from './init_timezone_dropdown';
// eslint-disable-next-line func-names
$(document).on('input.ssh_key', '#key_key', function () {
@ -21,3 +22,4 @@ new Profile(); // eslint-disable-line no-new
initSearchSettings();
initPasswordPrompt();
initTimezoneDropdown();

View File

@ -0,0 +1,40 @@
import Vue from 'vue';
import TimezoneDropdown from '~/vue_shared/components/timezone_dropdown.vue';
import { formatUtcOffset } from '~/lib/utils/datetime_utility';
const formatTimezone = (item) => `[UTC ${formatUtcOffset(item.offset)}] ${item.name}`;
export const initTimezoneDropdown = () => {
const el = document.querySelector('.js-timezone-dropdown');
if (!el) {
return null;
}
const { timezoneData, initialValue } = el.dataset;
const timezones = JSON.parse(timezoneData);
const initialTimezone = initialValue
? formatTimezone(timezones.find((timezone) => timezone.identifier === initialValue))
: undefined;
const timezoneDropdown = new Vue({
el,
data() {
return {
value: initialTimezone,
};
},
render(h) {
return h(TimezoneDropdown, {
props: {
value: this.value,
timezoneData: timezones,
name: 'user[timezone]',
},
class: 'gl-md-form-input-lg',
});
},
});
return timezoneDropdown;
};

View File

@ -5,9 +5,6 @@ import axios from '~/lib/utils/axios_utils';
import { parseBoolean } from '~/lib/utils/common_utils';
import { parseRailsFormFields } from '~/lib/utils/forms';
import { Rails } from '~/lib/utils/rails_ujs';
import TimezoneDropdown, {
formatTimezone,
} from '~/pages/projects/pipeline_schedules/shared/components/timezone_dropdown';
import UserProfileSetStatusWrapper from '~/set_status_modal/user_profile_set_status_wrapper.vue';
export default class Profile {
@ -17,15 +14,6 @@ export default class Profile {
this.setRepoRadio();
this.bindEvents();
this.initAvatarGlCrop();
this.$inputEl = $('#user_timezone');
this.timezoneDropdown = new TimezoneDropdown({
$inputEl: this.$inputEl,
$dropdownEl: $('.js-timezone-dropdown'),
displayFormat: (selectedItem) => formatTimezone(selectedItem),
allowEmpty: true,
});
}
initAvatarGlCrop() {

View File

@ -1,6 +1,6 @@
<script>
import { GlDropdown, GlDropdownItem, GlSearchBoxByType } from '@gitlab/ui';
import { secondsToHours } from '~/lib/utils/datetime_utility';
import { formatUtcOffset } from '~/lib/utils/datetime_utility';
import { __ } from '~/locale';
import autofocusonshow from '~/vue_shared/directives/autofocusonshow';
@ -20,6 +20,11 @@ export default {
required: true,
default: '',
},
name: {
type: String,
required: false,
default: '',
},
timezoneData: {
type: Array,
required: true,
@ -29,9 +34,10 @@ export default {
data() {
return {
searchTerm: '',
tzValue: this.value,
};
},
tranlations: {
translations: {
noResultsText: __('No matching results'),
},
computed: {
@ -48,41 +54,50 @@ export default {
);
},
selectedTimezoneLabel() {
return this.value || __('Select timezone');
return this.tzValue || __('Select timezone');
},
timezoneIdentifier() {
return this.tzValue
? this.timezones.find((timezone) => timezone.formattedTimezone === this.tzValue).identifier
: undefined;
},
},
methods: {
selectTimezone(selectedTimezone) {
this.tzValue = selectedTimezone.formattedTimezone;
this.$emit('input', selectedTimezone);
this.searchTerm = '';
},
isSelected(timezone) {
return this.value === timezone.formattedTimezone;
return this.tzValue === timezone.formattedTimezone;
},
formatTimezone(item) {
return `[UTC ${secondsToHours(item.offset)}] ${item.name}`;
return `[UTC ${formatUtcOffset(item.offset)}] ${item.name}`;
},
},
};
</script>
<template>
<gl-dropdown :text="selectedTimezoneLabel" block lazy menu-class="gl-w-full!" v-bind="$attrs">
<gl-search-box-by-type v-model.trim="searchTerm" v-autofocusonshow autofocus />
<gl-dropdown-item
v-for="timezone in filteredResults"
:key="timezone.formattedTimezone"
:is-checked="isSelected(timezone)"
is-check-item
@click="selectTimezone(timezone)"
>
{{ timezone.formattedTimezone }}
</gl-dropdown-item>
<gl-dropdown-item
v-if="!filteredResults.length"
class="gl-pointer-events-none"
data-testid="noMatchingResults"
>
{{ $options.tranlations.noResultsText }}
</gl-dropdown-item>
</gl-dropdown>
<div>
<input v-if="name" id="user_timezone" :name="name" :value="timezoneIdentifier" type="hidden" />
<gl-dropdown :text="selectedTimezoneLabel" block lazy menu-class="gl-w-full!" v-bind="$attrs">
<gl-search-box-by-type v-model.trim="searchTerm" v-autofocusonshow autofocus />
<gl-dropdown-item
v-for="timezone in filteredResults"
:key="timezone.formattedTimezone"
:is-checked="isSelected(timezone)"
is-check-item
@click="selectTimezone(timezone)"
>
{{ timezone.formattedTimezone }}
</gl-dropdown-item>
<gl-dropdown-item
v-if="!filteredResults.length"
class="gl-pointer-events-none"
data-testid="noMatchingResults"
>
{{ $options.translations.noResultsText }}
</gl-dropdown-item>
</gl-dropdown>
</div>
</template>

View File

@ -35,15 +35,15 @@ class GroupPolicy < Namespaces::GroupProjectNamespaceSharedPolicy
with_options scope: :subject, score: 0
condition(:request_access_enabled) { @subject.request_access_enabled }
condition(:create_projects_disabled) do
condition(:create_projects_disabled, scope: :subject) do
@subject.project_creation_level == ::Gitlab::Access::NO_ONE_PROJECT_ACCESS
end
condition(:developer_maintainer_access) do
condition(:developer_maintainer_access, scope: :subject) do
@subject.project_creation_level == ::Gitlab::Access::DEVELOPER_MAINTAINER_PROJECT_ACCESS
end
condition(:maintainer_can_create_group) do
condition(:maintainer_can_create_group, scope: :subject) do
@subject.subgroup_creation_level == ::Gitlab::Access::MAINTAINER_SUBGROUP_ACCESS
end
@ -51,7 +51,7 @@ class GroupPolicy < Namespaces::GroupProjectNamespaceSharedPolicy
group_projects_for(user: @user, group: @subject, only_owned: false).any? { |p| p.design_management_enabled? }
end
condition(:dependency_proxy_available) do
condition(:dependency_proxy_available, scope: :subject) do
@subject.dependency_proxy_feature_available?
end
@ -59,7 +59,7 @@ class GroupPolicy < Namespaces::GroupProjectNamespaceSharedPolicy
access_level(for_any_session: true) >= GroupMember::GUEST || valid_dependency_proxy_deploy_token
end
condition(:observability_enabled) do
condition(:observability_enabled, scope: :subject) do
Feature.enabled?(:observability_group_tab, @subject)
end
@ -80,9 +80,10 @@ class GroupPolicy < Namespaces::GroupProjectNamespaceSharedPolicy
with_scope :subject
condition(:has_project_with_service_desk_enabled) { @subject.has_project_with_service_desk_enabled? }
with_scope :subject
condition(:crm_enabled, score: 0, scope: :subject) { @subject.crm_enabled? }
condition(:group_runner_registration_allowed) do
condition(:group_runner_registration_allowed, scope: :global) do
Feature.disabled?(:runner_registration_control) || Gitlab::CurrentSettings.valid_runner_registrars.include?('group')
end

View File

@ -54,9 +54,8 @@
%h4.gl-mt-0= s_("Profiles|Time settings")
%p= s_("Profiles|Set your local time zone.")
.col-lg-8
%h5= _("Time zone")
= dropdown_tag(_("Select a time zone"), options: { toggle_class: 'gl-button btn js-timezone-dropdown input-lg gl-w-full!', title: _("Select a time zone"), filter: true, placeholder: s_("OfSearchInADropdown|Filter"), data: { data: timezone_data } } )
%input.hidden{ :type => 'hidden', :id => 'user_timezone', :name => 'user[timezone]', value: @user.timezone }
= f.label :user_timezone, _("Time zone")
.js-timezone-dropdown{ data: { timezone_data: timezone_data.to_json, initial_value: @user.timezone } }
.col-lg-12
%hr
.row.js-search-settings-section

View File

@ -1,5 +1,10 @@
# frozen_string_literal: true
# Explicitly set the JSON adapter used by MultiJson
# Currently we want this to default to the existing json gem
#
# This changes the default JSON adapter used by any gem dependencies
# we have that rely on MultiJson for their JSON handling. We set this
# to `oj` for a universal performance improvement in JSON handling
# across those gems.
MultiJson.use(:oj)

View File

@ -138,7 +138,14 @@ You can filter by merged attributes with:
GET /personal_access_tokens?revoked=true&created_before=2022-01-01
```
## Get single personal access token by ID
## Get single personal access token
Get a personal access token by either:
- Using the ID of the personal access token.
- Passing it to the API in a header.
### Using a personal access token ID
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362239) in GitLab 15.1.
@ -157,7 +164,7 @@ GET /personal_access_tokens/:id
curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/personal_access_tokens/<id>"
```
### Responses
#### Responses
> `404` HTTP status code [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93650) in GitLab 15.3.
@ -166,6 +173,38 @@ curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab
- The token with the specified ID doesn't exist.
- `404: Not Found` if the user is an administrator but the token with the specified ID doesn't exist.
### Using a request header
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/373999) in GitLab 15.5
Get a single personal access token by using passing the token in a header.
```plaintext
GET /personal_access_tokens/self
```
```shell
curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/personal_access_tokens/self"
```
Example response:
```json
{
"id": 4,
"name": "Test Token",
"revoked": false,
"created_at": "2020-07-23T14:31:47.729Z",
"scopes": [
"api"
],
"user_id": 3,
"last_used_at": "2021-10-06T17:58:37.550Z",
"active": true,
"expires_at": null
}
```
## Revoke a personal access token
Revoke a personal access token by either:

View File

@ -245,6 +245,7 @@ In progress.
- 2022-04-15: Partitioned pipeline data associations PoC [shipped](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84071).
- 2022-04-30: Additional [benchmarking started](https://gitlab.com/gitlab-org/gitlab/-/issues/361019) to evaluate impact.
- 2022-06-31: [Pipeline partitioning design](pipeline_partitioning.md) document [merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87683) merged.
- 2022-09-01: Engineering effort started to implement partitioning.
## Who

View File

@ -610,6 +610,40 @@ application-wide outage.
1. Make it possible to create partitions in an automatic way.
1. Deliver the new architecture to self-managed instances.
The diagram below visualizes this plan on Gantt chart. Please note that dates
on the chart below are just estimates to visualize the plan better, these are
not deadlines and can change at any time.
```mermaid
gantt
title CI Data Partitioning Timeline
dateFormat YYYY-MM-DD
axisFormat %Y-%m
section Phase 0
Build data partitioning strategy :done, 0_1, 2022-06-01, 90d
section Phase 1
Partition biggest CI tables :1_1, after 0_1, 140d
Biggest table partitioned :milestone, metadata, 2022-12-01, 1min
Tables larger than 100GB partitioned :milestone, 100gb, after 1_1, 1min
section Phase 2
Add paritioning keys to SQL queries :2_1, after 1_1, 120d
Emergency partition detachment possible :milestone, detachment, 2023-04-01, 1min
All SQL queries are routed to partitions :milestone, routing, after 2_1, 1min
section Phase 3
Build new data access patterns :3_1, 2023-03-01, 120d
New API endpoint created for inactive data :milestone, api1, 2023-05-01, 1min
Filtering added to existing API endpoints :milestone, api2, 2023-07-01, 1min
section Phase 4
Introduce time-decay mechanisms :4_1, 2023-06-01, 120d
Inactive partitions are not being read :milestone, part1, 2023-08-01, 1min
Performance of the database cluster improves :milestone, part2, 2023-09-01, 1min
section Phase 5
Introduce auto-partitioning mechanisms :5_1, 2023-07-01, 120d
New partitions are being created automatically :milestone, part3, 2023-10-01, 1min
Partitioning is made available on self-managed :milestone, part4, 2023-11-01, 1min
```
## Conclusions
We want to build a solid strategy for partitioning CI/CD data. We are aware of

View File

@ -17,11 +17,15 @@ and has become [one of the most beloved CI/CD solutions](https://about.gitlab.co
GitLab CI/CD has come a long way since the initial release, but the design of
the data storage for pipeline builds remains almost the same since 2012. We
store all the builds in PostgreSQL in `ci_builds` table, and because we are
creating more than [2 million builds each day on GitLab.com](https://docs.google.com/spreadsheets/d/17ZdTWQMnTHWbyERlvj1GA7qhw_uIfCoI5Zfrrsh95zU),
we are reaching database limits that are slowing our development velocity down.
creating more than 5 million builds each day on GitLab.com we are reaching
database limits that are slowing our development velocity down.
On February 1st, 2021, GitLab.com surpassed 1 billion CI/CD builds created and the number of
builds continues to grow exponentially.
On February 1st, 2021, GitLab.com surpassed 1 billion CI/CD builds created. In
February 2022 we reached 2 billion of CI/CD build stored in the database. The
number of builds continues to grow exponentially.
The screenshot below shows our forecast created at the beginning of 2021, that
turned out to be quite accurate.
![CI builds cumulative with forecast](ci_builds_cumulative_forecast.png)
@ -34,9 +38,9 @@ builds continues to grow exponentially.
The current state of CI/CD product architecture needs to be updated if we want
to sustain future growth.
### We are running out of the capacity to store primary keys
### We were running out of the capacity to store primary keys: DONE
The primary key in `ci_builds` table is an integer generated in a sequence.
The primary key in `ci_builds` table is an integer value, generated in a sequence.
Historically, Rails used to use [integer](https://www.postgresql.org/docs/14/datatype-numeric.html)
type when creating primary keys for a table. We did use the default when we
[created the `ci_builds` table in 2012](https://gitlab.com/gitlab-org/gitlab/-/blob/046b28312704f3131e72dcd2dbdacc5264d4aa62/db/ci/migrate/20121004165038_create_builds.rb).
@ -45,34 +49,32 @@ since the release of Rails 5. The framework is now using `bigint` type that is 8
bytes long, however we have not migrated primary keys for `ci_builds` table to
`bigint` yet.
We will run out of the capacity of the integer type to store primary keys in
`ci_builds` table before December 2021. When it happens without a viable
workaround or an emergency plan, GitLab.com will go down.
In early 2021 we had estimated that would run out of the capacity of the integer
type to store primary keys in `ci_builds` table before December 2021. If it had
happened without a viable workaround or an emergency plan, GitLab.com would go
down. `ci_builds` was just one of many tables that were running out of the
primary keys available in Int4 sequence.
`ci_builds` is just one of the tables that are running out of the primary keys
available in Int4 sequence. There are multiple other tables storing CI/CD data
that have the same problem.
Before October 2021, our Database team had managed to migrate all the risky
tables' primary keys to big integers.
Primary keys problem will be tackled by our Database Team.
See the [related Epic](https://gitlab.com/groups/gitlab-org/-/epics/5657) for more details.
**Status**: In October 2021, the primary keys in CI tables were migrated
to big integers. See the [related Epic](https://gitlab.com/groups/gitlab-org/-/epics/5657) for more details.
### Some CI/CD database tables are too large: IN PROGRESS
### The table is too large
There is more than two billion rows in `ci_builds` table. We store many
terabytes of data in that table, and the total size of indexes is measured in
terabytes as well.
There is more than a billion rows in `ci_builds` table. We store more than 2
terabytes of data in that table, and the total size of indexes is more than 1
terabyte (as of February 2021).
This amount of data contributes to a significant number of performance
problems we experience on our CI PostgreSQL database.
This amount of data contributes to a significant performance problems we
experience on our primary PostgreSQL database.
Most of the problem are related to how PostgreSQL database works internally,
Most of the problems are related to how PostgreSQL database works internally,
and how it is making use of resources on a node the database runs on. We are at
the limits of vertical scaling of the primary database nodes and we frequently
see a negative impact of the `ci_builds` table on the overall performance,
stability, scalability and predictability of the database GitLab.com depends
on.
the limits of vertical scaling of the CI primary database nodes and we
frequently see a negative impact of the `ci_builds` table on the overall
performance, stability, scalability and predictability of the CI database
GitLab.com depends on.
The size of the table also hinders development velocity because queries that
seem fine in the development environment may not work on GitLab.com. The
@ -90,41 +92,40 @@ environment.
We also expect a significant, exponential growth in the upcoming years.
One of the forecasts done using [Facebook's Prophet](https://facebook.github.io/prophet/)
shows that in the first half of
2024 we expect seeing 20M builds created on GitLab.com each day. In comparison
to around 2M we see created today, this is 10x growth our product might need to
sustain in upcoming years.
shows that in the first half of 2024 we expect seeing 20M builds created on
GitLab.com each day. In comparison to around 5M we see created today. This is
10x growth from numbers we saw in 2021.
![CI builds daily forecast](ci_builds_daily_forecast.png)
**Status**: As of October 2021 we reduced the growth rate of `ci_builds` table
by writing build options and variables to `ci_builds_metadata` table. We plan
to ship further improvements that will be described in a separate blueprint.
by writing build options and variables to `ci_builds_metadata` table. We are
also working on partitioning the largest CI/CD database tables using
[time decay pattern](../ci_data_decay/index.md).
### Queuing mechanisms are using the large table
### Queuing mechanisms were using the large table: DONE
Because of how large the table is, mechanisms that we use to build queues of
pending builds (there is more than one queue), are not very efficient. Pending
builds represent a small fraction of what we store in the `ci_builds` table,
yet we need to find them in this big dataset to determine an order in which we
want to process them.
Because of how large the table is, mechanisms that we used to build queues of
pending builds (there is more than one queue), were not very efficient. Pending
builds represented a small fraction of what we store in the `ci_builds` table,
yet we needed to find them in this big dataset to determine an order in which we
wanted to process them.
This mechanism is very inefficient, and it has been causing problems on the
production environment frequently. This usually results in a significant drop
of the CI/CD Apdex score, and sometimes even causes a significant performance
This mechanism was very inefficient, and it had been causing problems on the
production environment frequently. This usually resulted in a significant drop
of the CI/CD Apdex score, and sometimes even caused a significant performance
degradation in the production environment.
There are multiple other strategies that can improve performance and
reliability. We can use [Redis queuing](https://gitlab.com/gitlab-org/gitlab/-/issues/322972), or
[a separate table that will accelerate SQL queries used to build queues](https://gitlab.com/gitlab-org/gitlab/-/issues/322766)
and we want to explore them.
There were multiple other strategies that we considered to improve performance and
reliability. We evaluated using [Redis queuing](https://gitlab.com/gitlab-org/gitlab/-/issues/322972), or
[a separate table that would accelerate SQL queries used to build queues](https://gitlab.com/gitlab-org/gitlab/-/issues/322766).
We decided to proceed with the latter.
**Status**: As of October 2021 the new architecture
[has been implemented on GitLab.com](https://gitlab.com/groups/gitlab-org/-/epics/5909#note_680407908).
The following epic tracks making it generally available:
[Make the new pending builds architecture generally available](https://gitlab.com/groups/gitlab-org/-/epics/6954).
In October 2021 we finished shipping the new architecture of builds queuing
[on GitLab.com](https://gitlab.com/groups/gitlab-org/-/epics/5909#note_680407908).
We then made the new architecture [generally available](https://gitlab.com/groups/gitlab-org/-/epics/6954).
### Moving big amounts of data is challenging
### Moving big amounts of data is challenging: IN PROGRESS
We store a significant amount of data in `ci_builds` table. Some of the columns
in that table store a serialized user-provided data. Column `ci_builds.options`
@ -144,24 +145,27 @@ described in a separate architectural blueprint.
## Proposal
Making GitLab CI/CD product ready for the scale we expect to see in the
upcoming years is a multi-phase effort.
Below you can find the original proposal made in early 2021 about how we want
to move forward with CI Scaling effort:
First, we want to focus on things that are urgently needed right now. We need
to fix primary keys overflow risk and unblock other teams that are working on
database partitioning and sharding.
We want to improve known bottlenecks, like
builds queuing mechanisms that is using the large table, and other things that
are holding other teams back.
Extending CI/CD metrics is important to get a better sense of how the system
performs and to what growth should we expect. This will make it easier for us
to identify bottlenecks and perform more advanced capacity planning.
Next step is to better understand how we can leverage strong time-decay
characteristic of CI/CD data. This might help us to partition CI/CD dataset to
reduce the size of CI/CD database tables.
> Making GitLab CI/CD product ready for the scale we expect to see in the
> upcoming years is a multi-phase effort.
>
> First, we want to focus on things that are urgently needed right now. We need
> to fix primary keys overflow risk and unblock other teams that are working on
> database partitioning and sharding.
>
> We want to improve known bottlenecks, like
> builds queuing mechanisms that is using the large table, and other things that
> are holding other teams back.
>
> Extending CI/CD metrics is important to get a better sense of how the system
> performs and to what growth should we expect. This will make it easier for us
> to identify bottlenecks and perform more advanced capacity planning.
>
> Next step is to better understand how we can leverage strong time-decay
> characteristic of CI/CD data. This might help us to partition CI/CD dataset to
> reduce the size of CI/CD database tables.
## Iterations
@ -170,15 +174,12 @@ Work required to achieve our next CI/CD scaling target is tracked in the
1. ✓ Migrate primary keys to big integers on GitLab.com.
1. ✓ Implement the new architecture of builds queuing on GitLab.com.
1. [Make the new builds queuing architecture generally available](https://gitlab.com/groups/gitlab-org/-/epics/6954).
1. [Make the new builds queuing architecture generally available](https://gitlab.com/groups/gitlab-org/-/epics/6954).
1. [Partition CI/CD data using time-decay pattern](../ci_data_decay/index.md).
## Status
|-------------|--------------|
| Created at | 21.01.2021 |
| Approved at | 26.04.2021 |
| Updated at | 28.02.2022 |
Created at 21.01.2021, approved at 26.04.2021.
Status: In progress.

View File

@ -28,7 +28,7 @@ Value stream analytics is also available for [projects](../../analytics/value_st
> - Filtering [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13216) in GitLab 13.3
> - Horizontal stage path [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12196) in 13.0 and [feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/323982) in 13.12
Prerequisite:
Prerequisites:
- You must have at least the Reporter role to view value stream analytics for groups.
- You must create a [custom value stream](#create-a-value-stream-with-gitlab-default-stages). Value stream analytics only shows custom value streams created for your group.
@ -68,7 +68,9 @@ The table shows a list of related workflow items for the selected stage. Based o
The **Overview** dashboard in value stream analytics shows key metrics and DORA metrics of group performance. Based on the filter you select,
the dashboard automatically aggregates DORA metrics and displays the current status of the value stream. Select a DORA metric to view its chart.
To view deployment metrics, you must have a
Prerequisite:
- To view deployment metrics, you must have a
[production environment configured](../../../ci/environments/index.md#deployment-tier-of-environments).
To view the DORA metrics and key metrics:
@ -248,7 +250,7 @@ You can change the name of a project environment in your GitLab CI/CD configurat
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221202) in GitLab 13.3
When you create a value stream, you can use GitLab default stages and hide or re-order them to customize. You can also
When you create a value stream, you can use GitLab default stages and hide or re-order them. You can also
create custom stages in addition to those provided in the default template.
1. On the top bar, select **Main menu > Groups** and find your group.

View File

@ -120,7 +120,12 @@ To change the assignee on a task:
## Set a start and due date
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/365399) in GitLab 15.4.
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/365399) in GitLab 15.4 [with a flag](../administration/feature_flags.md) named `work_items_mvc_2`. Disabled by default.
FLAG:
On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `work_items_mvc_2`.
On GitLab.com, this feature is not available.
This feature is not ready for production use.
You can set a [start and due date](project/issues/due_dates.md) on a task.

View File

@ -263,7 +263,7 @@ module API
mount ::API::PackageFiles
mount ::API::Pages
mount ::API::PagesDomains
mount ::API::PersonalAccessTokens::SelfRevocation
mount ::API::PersonalAccessTokens::SelfInformation
mount ::API::PersonalAccessTokens
mount ::API::ProjectClusters
mount ::API::ProjectContainerRepositories

View File

@ -2,21 +2,25 @@
module API
class PersonalAccessTokens
class SelfRevocation < ::API::Base
class SelfInformation < ::API::Base
include APIGuard
feature_category :authentication_and_authorization
helpers ::API::Helpers::PersonalAccessTokensHelpers
# As any token regardless of `scope` should be able to revoke itself
# all availabe scopes are allowed for this API class.
# As any token regardless of `scope` should be able to view/revoke itself
# all available scopes are allowed for this API class.
# Please be aware of the permissive scope when adding new endpoints to this class.
allow_access_with_scope(Gitlab::Auth.all_available_scopes)
before { authenticate! }
resource :personal_access_tokens do
get 'self' do
present access_token, with: Entities::PersonalAccessToken
end
delete 'self' do
revoke_token(access_token)
end

View File

@ -36211,9 +36211,6 @@ msgstr ""
msgid "Select a template type"
msgstr ""
msgid "Select a time zone"
msgstr ""
msgid "Select a timezone"
msgstr ""

View File

@ -551,13 +551,13 @@ RSpec.describe 'User edit profile' do
it 'allows the user to select a time zone from a dropdown list of options' do
expect(page.find('.user-time-preferences .dropdown')).not_to have_css('.show')
page.find('.user-time-preferences .js-timezone-dropdown').click
page.find('.user-time-preferences .dropdown').click
expect(page.find('.user-time-preferences .dropdown')).to have_css('.show')
page.find("a", text: "Nuku'alofa").click
page.find("button", text: "Arizona").click
expect(page).to have_field(:user_timezone, with: 'Pacific/Tongatapu', type: :hidden)
expect(page).to have_field(:user_timezone, with: 'America/Phoenix', type: :hidden)
end
it 'timezone defaults to empty' do

View File

@ -1,10 +1,10 @@
import freezePeriodsFixture from 'test_fixtures/api/freeze-periods/freeze_periods.json';
import timezoneDataFixture from 'test_fixtures/timezones/short.json';
import { secondsToHours } from '~/lib/utils/datetime_utility';
import { formatUtcOffset } from '~/lib/utils/datetime_utility';
export { freezePeriodsFixture, timezoneDataFixture };
export const findTzByName = (identifier = '') =>
timezoneDataFixture.find(({ name }) => name.toLowerCase() === identifier.toLowerCase());
export const formatTz = ({ offset, name }) => `[UTC ${secondsToHours(offset)}] ${name}`;
export const formatTz = ({ offset, name }) => `[UTC ${formatUtcOffset(offset)}] ${name}`;

View File

@ -28,9 +28,9 @@ describe('Deploy freeze mutations', () => {
describe('RECEIVE_FREEZE_PERIODS_SUCCESS', () => {
it('should set freeze periods and format timezones from identifiers to names', () => {
const timezoneNames = {
'Europe/Berlin': '[UTC 2] Berlin',
'Europe/Berlin': '[UTC + 2] Berlin',
'Etc/UTC': '[UTC 0] UTC',
'America/New_York': '[UTC -4] Eastern Time (US & Canada)',
'America/New_York': '[UTC - 4] Eastern Time (US & Canada)',
};
mutations[types.RECEIVE_FREEZE_PERIODS_SUCCESS](stateCopy, freezePeriodsFixture);

View File

@ -1,4 +1,8 @@
import { getDateWithUTC, newDateAsLocaleTime } from '~/lib/utils/datetime/date_calculation_utility';
import {
getDateWithUTC,
newDateAsLocaleTime,
formatUtcOffset,
} from '~/lib/utils/datetime/date_calculation_utility';
describe('newDateAsLocaleTime', () => {
it.each`
@ -31,3 +35,22 @@ describe('getDateWithUTC', () => {
expect(getDateWithUTC(date)).toEqual(expected);
});
});
describe('formatUtcOffset', () => {
it.each`
offset | expected
${-32400} | ${'- 9'}
${'-12600'} | ${'- 3.5'}
${0} | ${'0'}
${'10800'} | ${'+ 3'}
${19800} | ${'+ 5.5'}
${0} | ${'0'}
${[]} | ${'0'}
${{}} | ${'0'}
${true} | ${'0'}
${null} | ${'0'}
${undefined} | ${'0'}
`('returns $expected given $offset', ({ offset, expected }) => {
expect(formatUtcOffset(offset)).toEqual(expected);
});
});

View File

@ -2,13 +2,13 @@
require 'spec_helper'
RSpec.describe API::PersonalAccessTokens::SelfRevocation do
RSpec.describe API::PersonalAccessTokens::SelfInformation do
let(:path) { '/personal_access_tokens/self' }
let(:token) { create(:personal_access_token, user: current_user) }
let_it_be(:current_user) { create(:user) }
describe 'DELETE /personal_access_tokens/self' do
let(:path) { '/personal_access_tokens/self' }
let(:token) { create(:personal_access_token, user: current_user) }
subject(:delete_token) { delete api(path, personal_access_token: token) }
shared_examples 'revoking token succeeds' do
@ -66,4 +66,37 @@ RSpec.describe API::PersonalAccessTokens::SelfRevocation do
end
end
end
describe 'GET /personal_access_tokens/self' do
Gitlab::Auth.all_available_scopes.each do |scope|
context "with a '#{scope}' scoped token" do
let(:token) { create(:personal_access_token, scopes: [scope], user: current_user) }
it 'shows token info' do
get api(path, personal_access_token: token)
expect(response).to have_gitlab_http_status(:ok)
expect(json_response['scopes']).to match_array([scope.to_s])
end
end
end
context 'when token is invalid' do
it 'returns 401' do
get api(path, personal_access_token: instance_double(PersonalAccessToken, token: 'invalidtoken'))
expect(response).to have_gitlab_http_status(:unauthorized)
end
end
context 'when token is expired' do
it 'returns 401' do
token = create(:personal_access_token, expires_at: 1.day.ago, user: current_user)
get api(path, personal_access_token: token)
expect(response).to have_gitlab_http_status(:unauthorized)
end
end
end
end

View File

@ -189,7 +189,6 @@
- './ee/spec/controllers/subscriptions_controller_spec.rb'
- './ee/spec/controllers/subscriptions/groups_controller_spec.rb'
- './ee/spec/controllers/trial_registrations_controller_spec.rb'
- './ee/spec/controllers/trials_controller_spec.rb'
- './ee/spec/controllers/users_controller_spec.rb'
- './ee/spec/db/production/license_spec.rb'
- './ee/spec/elastic_integration/global_search_spec.rb'
@ -533,7 +532,6 @@
- './ee/spec/features/trial_registrations/company_information_spec.rb'
- './ee/spec/features/trial_registrations/signin_spec.rb'
- './ee/spec/features/trial_registrations/signup_spec.rb'
- './ee/spec/features/trials/capture_lead_spec.rb'
- './ee/spec/features/trials/select_namespace_spec.rb'
- './ee/spec/features/trials/show_trial_banner_spec.rb'
- './ee/spec/features/users/arkose_labs_csp_spec.rb'