+
+
+
diff --git a/app/assets/javascripts/vue_shared/components/alert_details_table.vue b/app/assets/javascripts/vue_shared/components/alert_details_table.vue
index 395ed0f2033..c94e784c01e 100644
--- a/app/assets/javascripts/vue_shared/components/alert_details_table.vue
+++ b/app/assets/javascripts/vue_shared/components/alert_details_table.vue
@@ -1,6 +1,14 @@
diff --git a/app/assets/javascripts/vue_shared/components/sidebar/labels_select_vue/labels_select_root.vue b/app/assets/javascripts/vue_shared/components/sidebar/labels_select_vue/labels_select_root.vue
index 248e9929833..7aa464dd9b3 100644
--- a/app/assets/javascripts/vue_shared/components/sidebar/labels_select_vue/labels_select_root.vue
+++ b/app/assets/javascripts/vue_shared/components/sidebar/labels_select_vue/labels_select_root.vue
@@ -166,7 +166,11 @@ export default {
!state.showDropdownButton &&
!state.showDropdownContents
) {
- this.handleDropdownClose(state.labels.filter(label => label.touched));
+ let filterFn = label => label.touched;
+ if (this.isDropdownVariantEmbedded) {
+ filterFn = label => label.set;
+ }
+ this.handleDropdownClose(state.labels.filter(filterFn));
}
},
/**
@@ -186,7 +190,7 @@ export default {
].some(
className =>
target?.classList.contains(className) ||
- target?.parentElement.classList.contains(className),
+ target?.parentElement?.classList.contains(className),
);
const hadExceptionParent = ['.js-btn-back', '.js-labels-list'].some(
diff --git a/app/assets/stylesheets/pages/alert_management/details.scss b/app/assets/stylesheets/pages/alert_management/details.scss
index 531693a3bbe..a104c06c853 100644
--- a/app/assets/stylesheets/pages/alert_management/details.scss
+++ b/app/assets/stylesheets/pages/alert_management/details.scss
@@ -1,48 +1,4 @@
.alert-management-details {
- // these styles need to be deleted once GlTable component looks in GitLab same as in @gitlab/ui
- table {
- tr {
- td {
- @include gl-border-0;
- @include gl-p-5;
- border-color: transparent;
-
- &:not(:last-child) {
- border-bottom: 1px solid $table-border-color;
- }
-
- &:first-child {
- div {
- font-weight: bold;
- }
- }
-
- &:not(:first-child) {
- &::before {
- color: $gray-500;
- font-weight: normal !important;
- }
-
- div {
- color: $gray-500;
- }
- }
-
- @include media-breakpoint-up(sm) {
- div {
- text-align: left !important;
- }
- }
- }
-
- &:last-child {
- &::after {
- content: none !important;
- }
- }
- }
- }
-
@include media-breakpoint-down(xs) {
.alert-details-incident-button {
width: 100%;
diff --git a/app/uploaders/object_storage.rb b/app/uploaders/object_storage.rb
index d82271e4b92..f72d02d0bf0 100644
--- a/app/uploaders/object_storage.rb
+++ b/app/uploaders/object_storage.rb
@@ -210,6 +210,20 @@ module ObjectStorage
end
end
+ class OpenFile
+ extend Forwardable
+
+ # Explicitly exclude :path, because rubyzip uses that to detect "real" files.
+ def_delegators :@file, *(Zip::File::IO_METHODS - [:path])
+
+ # Even though :size is not in IO_METHODS, we do need it.
+ def_delegators :@file, :size
+
+ def initialize(file)
+ @file = file
+ end
+ end
+
# allow to configure and overwrite the filename
def filename
@filename || super || file&.filename # rubocop:disable Gitlab/ModuleWithInstanceVariables
@@ -259,6 +273,24 @@ module ObjectStorage
end
end
+ def use_open_file(&blk)
+ Tempfile.open(path) do |file|
+ file.unlink
+ file.binmode
+
+ if file_storage?
+ IO.copy_stream(path, file)
+ else
+ streamer = lambda { |chunk, _, _| file.write(chunk) }
+ Excon.get(url, response_block: streamer)
+ end
+
+ file.seek(0, IO::SEEK_SET)
+
+ yield OpenFile.new(file)
+ end
+ end
+
#
# Move the file to another store
#
diff --git a/app/views/admin/dev_ops_report/_no_data.html.haml b/app/views/admin/dev_ops_report/_no_data.html.haml
index 585aa878b0b..e540a4e2bce 100644
--- a/app/views/admin/dev_ops_report/_no_data.html.haml
+++ b/app/views/admin/dev_ops_report/_no_data.html.haml
@@ -3,5 +3,5 @@
= custom_icon('dev_ops_report_no_data')
%h4= _('Data is still calculating...')
%p
- = _('In order to gather accurate feature usage data, it can take 1 to 2 weeks to see your index.')
- = link_to _('Learn more'), help_page_path('user/admin_area/analytics/dev_ops_report'), target: '_blank'
+ = _('It may be several days before you see feature usage data.')
+ = link_to _('Our documentation includes an example DevOps Score report.'), help_page_path('user/admin_area/analytics/dev_ops_report'), target: '_blank'
diff --git a/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb b/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb
index 8580802fb7e..a9976c6e5cb 100644
--- a/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb
+++ b/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb
@@ -14,7 +14,7 @@ module Analytics
idempotent!
def perform
- return if Feature.disabled?(:store_instance_statistics_measurements)
+ return if Feature.disabled?(:store_instance_statistics_measurements, default_enabled: true)
recorded_at = Time.zone.now
measurement_identifiers = Analytics::InstanceStatistics::Measurement.identifiers
diff --git a/changelogs/unreleased/212595-ide-commit-errors.yml b/changelogs/unreleased/212595-ide-commit-errors.yml
new file mode 100644
index 00000000000..cfd28e3cb77
--- /dev/null
+++ b/changelogs/unreleased/212595-ide-commit-errors.yml
@@ -0,0 +1,5 @@
+---
+title: Fix error reporting for Web IDE commits
+merge_request: 42383
+author:
+type: fixed
diff --git a/changelogs/unreleased/229214-replace-LoadingButton-with-GlButton.yml b/changelogs/unreleased/229214-replace-LoadingButton-with-GlButton.yml
new file mode 100644
index 00000000000..3d14795226a
--- /dev/null
+++ b/changelogs/unreleased/229214-replace-LoadingButton-with-GlButton.yml
@@ -0,0 +1,5 @@
+---
+title: Replace LoadingButton with GlButton for the comment dismissal modal
+merge_request: 40882
+author:
+type: performance
diff --git a/changelogs/unreleased/enable-store-instance-statistics-ff-by-default.yml b/changelogs/unreleased/enable-store-instance-statistics-ff-by-default.yml
new file mode 100644
index 00000000000..25077bf65d8
--- /dev/null
+++ b/changelogs/unreleased/enable-store-instance-statistics-ff-by-default.yml
@@ -0,0 +1,5 @@
+---
+title: Store object counts periodically for instance statistics
+merge_request: 42433
+author:
+type: changed
diff --git a/changelogs/unreleased/mjang-devops-score-ui-text.yml b/changelogs/unreleased/mjang-devops-score-ui-text.yml
new file mode 100644
index 00000000000..20aa1d8eabe
--- /dev/null
+++ b/changelogs/unreleased/mjang-devops-score-ui-text.yml
@@ -0,0 +1,5 @@
+---
+title: Modify DevOps Score UI Text
+merge_request: 42256
+author:
+type: other
diff --git a/config/feature_flags/development/ci_new_artifact_file_reader.yml b/config/feature_flags/development/ci_new_artifact_file_reader.yml
new file mode 100644
index 00000000000..a6e9c67bd7e
--- /dev/null
+++ b/config/feature_flags/development/ci_new_artifact_file_reader.yml
@@ -0,0 +1,7 @@
+---
+name: ci_new_artifact_file_reader
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/40268
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/249588
+group: group::pipeline authoring
+type: development
+default_enabled: false
diff --git a/config/feature_flags/development/store_instance_statistics_measurements.yml b/config/feature_flags/development/store_instance_statistics_measurements.yml
index bc7b3400694..9483b9005df 100644
--- a/config/feature_flags/development/store_instance_statistics_measurements.yml
+++ b/config/feature_flags/development/store_instance_statistics_measurements.yml
@@ -4,4 +4,4 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41300
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/247871
group: group::analytics
type: development
-default_enabled: false
+default_enabled: true
diff --git a/doc/.vale/gitlab/Acronyms.yml b/doc/.vale/gitlab/Acronyms.yml
index 113c6012e4a..d26ce9810d7 100644
--- a/doc/.vale/gitlab/Acronyms.yml
+++ b/doc/.vale/gitlab/Acronyms.yml
@@ -76,6 +76,7 @@ exceptions:
- SCSS
- SDK
- SHA
+ - SLA
- SMTP
- SQL
- SSH
diff --git a/doc/administration/geo/disaster_recovery/index.md b/doc/administration/geo/disaster_recovery/index.md
index 442edfa5176..5aa90f02e6c 100644
--- a/doc/administration/geo/disaster_recovery/index.md
+++ b/doc/administration/geo/disaster_recovery/index.md
@@ -100,7 +100,7 @@ Note the following when promoting a secondary:
- If replication was paused on the secondary node, for example as a part of upgrading,
while you were running a version of GitLab lower than 13.4, you _must_
- [enable the node via the database](#while-promoting-the-secondary-i-got-an-error-activerecordrecordinvalid)
+ [enable the node via the database](../replication/troubleshooting.md#while-promoting-the-secondary-i-got-an-error-activerecordrecordinvalid)
before proceeding.
- A new **secondary** should not be added at this time. If you want to add a new
**secondary**, do this after you have completed the entire process of promoting
@@ -129,28 +129,20 @@ Note the following when promoting a secondary:
```
1. Promote the **secondary** node to the **primary** node.
-
- Before promoting a secondary node to primary, preflight checks should be run. They can be run separately or along with the promotion script.
-
+
To promote the secondary node to primary along with preflight checks:
```shell
gitlab-ctl promote-to-primary-node
```
- If you have already run the [preflight checks](planned_failover.md#preflight-checks) or don't want to run them, you can skip preflight checks with:
+ If you have already run the [preflight checks](planned_failover.md#preflight-checks) separately or don't want to run them, you can skip preflight checks with:
```shell
gitlab-ctl promote-to-primary-node --skip-preflight-check
```
- You can also run preflight checks separately:
-
- ```shell
- gitlab-ctl promotion-preflight-checks
- ```
-
- After all the checks are run, you will be asked for a final confirmation before the promotion to primary. To skip this confirmation, run:
+ You can also promote the secondary node to primary **without any further confirmation**, even when preflight checks fail:
```shell
gitlab-ctl promote-to-primary-node --force
@@ -421,33 +413,4 @@ for another **primary** node. All the old replication settings will be overwritt
## Troubleshooting
-### I followed the disaster recovery instructions and now two-factor auth is broken
-
-The setup instructions for Geo prior to 10.5 failed to replicate the
-`otp_key_base` secret, which is used to encrypt the two-factor authentication
-secrets stored in the database. If it differs between **primary** and **secondary**
-nodes, users with two-factor authentication enabled won't be able to log in
-after a failover.
-
-If you still have access to the old **primary** node, you can follow the
-instructions in the
-[Upgrading to GitLab 10.5](../replication/version_specific_updates.md#updating-to-gitlab-105)
-section to resolve the error. Otherwise, the secret is lost and you'll need to
-[reset two-factor authentication for all users](../../../security/two_factor_authentication.md#disabling-2fa-for-everyone).
-
-### While Promoting the secondary, I got an error `ActiveRecord::RecordInvalid`
-
-If you disabled a secondary node, either with the [replication pause task](../index.md#pausing-and-resuming-replication)
-(13.2) or via the UI (13.1 and earlier), you must first re-enable the
-node before you can continue. This is fixed in 13.4.
-
-From `gitlab-psql`, execute the following, replacing ``
-with the URL for your secondary server starting with `http` or `https` and ending with a `/`.
-
-```shell
-SECONDARY_URL="https:///"
-DATABASE_NAME="gitlabhq_production"
-sudo gitlab-psql -d "$DATABASE_NAME" -c "UPDATE geo_nodes SET enabled = true WHERE url = '$SECONDARY_URL';"
-```
-
-This should update 1 row.
+This section was moved to [another location](../replication/troubleshooting.md#fixing-errors-during-a-failover-or-when-promoting-a-secondary-to-a-primary-node).
diff --git a/doc/administration/geo/disaster_recovery/planned_failover.md b/doc/administration/geo/disaster_recovery/planned_failover.md
index 26eb39abcee..9b9c386652c 100644
--- a/doc/administration/geo/disaster_recovery/planned_failover.md
+++ b/doc/administration/geo/disaster_recovery/planned_failover.md
@@ -51,12 +51,6 @@ Run this command to list out all preflight checks and automatically check if rep
gitlab-ctl promotion-preflight-checks
```
-You can run this command in `force` mode to promote to primary even if preflight checks fail:
-
-```shell
-sudo gitlab-ctl promote-to-primary-node --force
-```
-
Each step is described in more detail below.
### Object storage
diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md
index 0d7ccb344bb..d0184c641a8 100644
--- a/doc/administration/geo/replication/troubleshooting.md
+++ b/doc/administration/geo/replication/troubleshooting.md
@@ -632,6 +632,23 @@ To double check this, you can do the following:
UPDATE geo_nodes SET enabled = 't' WHERE id = ID_FROM_ABOVE;
```
+### While Promoting the secondary, I got an error `ActiveRecord::RecordInvalid`
+
+If you disabled a secondary node, either with the [replication pause task](../index.md#pausing-and-resuming-replication)
+(13.2) or via the UI (13.1 and earlier), you must first re-enable the
+node before you can continue. This is fixed in 13.4.
+
+From `gitlab-psql`, execute the following, replacing ``
+with the URL for your secondary server starting with `http` or `https` and ending with a `/`.
+
+```shell
+SECONDARY_URL="https:///"
+DATABASE_NAME="gitlabhq_production"
+sudo gitlab-psql -d "$DATABASE_NAME" -c "UPDATE geo_nodes SET enabled = true WHERE url = '$SECONDARY_URL';"
+```
+
+This should update 1 row.
+
### Message: ``NoMethodError: undefined method `secondary?' for nil:NilClass``
When [promoting a **secondary** node](../disaster_recovery/index.md#step-3-promoting-a-secondary-node),
@@ -674,6 +691,20 @@ sudo /opt/gitlab/embedded/bin/gitlab-pg-ctl promote
GitLab 12.9 and later are [unaffected by this error](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5147).
+### Two-factor authentication is broken after a failover
+
+The setup instructions for Geo prior to 10.5 failed to replicate the
+`otp_key_base` secret, which is used to encrypt the two-factor authentication
+secrets stored in the database. If it differs between **primary** and **secondary**
+nodes, users with two-factor authentication enabled won't be able to log in
+after a failover.
+
+If you still have access to the old **primary** node, you can follow the
+instructions in the
+[Upgrading to GitLab 10.5](../replication/version_specific_updates.md#updating-to-gitlab-105)
+section to resolve the error. Otherwise, the secret is lost and you'll need to
+[reset two-factor authentication for all users](../../../security/two_factor_authentication.md#disabling-2fa-for-everyone).
+
## Expired artifacts
If you notice for some reason there are more artifacts on the Geo
diff --git a/doc/administration/geo/replication/version_specific_updates.md b/doc/administration/geo/replication/version_specific_updates.md
index 900d09bdd34..7932233e8cb 100644
--- a/doc/administration/geo/replication/version_specific_updates.md
+++ b/doc/administration/geo/replication/version_specific_updates.md
@@ -314,7 +314,7 @@ sudo gitlab-ctl reconfigure
```
If you do not perform this step, you may find that two-factor authentication
-[is broken following DR](../disaster_recovery/index.md#i-followed-the-disaster-recovery-instructions-and-now-two-factor-auth-is-broken).
+[is broken following DR](troubleshooting.md#two-factor-authentication-is-broken-after-a-failover).
To prevent SSH requests to the newly promoted **primary** node from failing
due to SSH host key mismatch when updating the **primary** node domain's DNS record
diff --git a/doc/ci/pipelines/img/ci_efficiency_pipeline_dag_critical_path.png b/doc/ci/pipelines/img/ci_efficiency_pipeline_dag_critical_path.png
new file mode 100644
index 00000000000..1715e8224ab
Binary files /dev/null and b/doc/ci/pipelines/img/ci_efficiency_pipeline_dag_critical_path.png differ
diff --git a/doc/ci/pipelines/img/ci_efficiency_pipeline_health_grafana_dashboard.png b/doc/ci/pipelines/img/ci_efficiency_pipeline_health_grafana_dashboard.png
new file mode 100644
index 00000000000..0956e76804e
Binary files /dev/null and b/doc/ci/pipelines/img/ci_efficiency_pipeline_health_grafana_dashboard.png differ
diff --git a/doc/ci/pipelines/pipeline_efficiency.md b/doc/ci/pipelines/pipeline_efficiency.md
new file mode 100644
index 00000000000..fbcc4321381
--- /dev/null
+++ b/doc/ci/pipelines/pipeline_efficiency.md
@@ -0,0 +1,251 @@
+---
+stage: Verify
+group: Continuous Integration
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers
+type: reference
+---
+
+# Pipeline Efficiency
+
+[CI/CD Pipelines](index.md) are the fundamental building blocks for [GitLab CI/CD](../README.md).
+Making pipelines more efficient helps you save developer time, which:
+
+- Speeds up your DevOps processes
+- Reduces costs
+- Shortens the development feedback loop
+
+It's common that new teams or projects start with slow and inefficient pipelines,
+and improve their configuration over time through trial and error. A better process is
+to use pipeline features that improve efficiency right away, and get a faster software
+development lifecycle earlier.
+
+First ensure you are familiar with [GitLab CI/CD fundamentals](../introduction/index.md)
+and understand the [quick start guide](../quick_start/README.md).
+
+## Identify bottlenecks and common failures
+
+The easiest indicators to check for inefficient pipelines are the runtimes of the jobs,
+stages, and the total runtime of the pipeline itself. The total pipeline duration is
+heavily influenced by the:
+
+- Total number of stages and jobs
+- Dependencies between jobs
+- The ["critical path"](#directed-acyclic-graphs-dag-visualization), which represents
+ the minimum and maximum pipeline duration
+
+Additional points to pay attention relate to [GitLab Runners](../runners/README.md):
+
+- Availability of the runners and the resources they are provisioned with
+- Build dependencies and their installation time
+- [Container image size](#docker-images)
+- Network latency and slow connections
+
+Pipelines frequently failing unnecessarily also causes slowdowns in the development
+lifecycle. You should look for problematic patterns with failed jobs:
+
+- Flaky unit tests which fail randomly, or produce unreliable test results.
+- Test coverage drops and code quality correlated to that behavior.
+- Failures that can be safely ignored, but that halt the pipeline instead.
+- Tests that fail at the end of a long pipeline, but could be in an earlier stage,
+ causing delayed feedback.
+
+## Pipeline analysis
+
+Analyze the performance of your pipeline to find ways to improve efficiency. Analysis
+can help identify possible blockers in the CI/CD infrastructure. This includes analyzing:
+
+- Job workloads
+- Bottlenecks in the execution times
+- The overall pipeline architecture
+
+It's important to understand and document the pipeline workflows, and discuss possible
+actions and changes. Refactoring pipelines may need careful interaction between teams
+in the DevSecOps lifecycle.
+
+Pipeline analysis can help identify issues with cost efficiency. For example, [runners](../runners/README.md)
+hosted with a paid cloud service may be provisioned with:
+
+- More resources than needed for CI/CD pipelines, wasting money.
+- Not enough resources, causing slow runtimes and wasting time.
+
+### Pipeline Insights
+
+The [Pipeline success and duration charts](index.md#pipeline-success-and-duration-charts)
+give information about pipeline runtime and failed job counts.
+
+Tests like [unit tests](../unit_test_reports.md), integration tests, end-to-end tests,
+[code quality](../../user/project/merge_requests/code_quality.md) tests, and others
+ensure that problems are automatically found by the CI/CD pipeline. There could be many
+pipeline stages involved causing long runtimes.
+
+You can improve runtimes by running jobs that test different things in parallel, in
+the same stage, reducing overall runtime. The downside is that you need more runners
+running simultaneously to support the parallel jobs.
+
+The [testing levels for GitLab](../../development/testing_guide/testing_levels.md)
+provide an example of a complex testing strategy with many components involved.
+
+### Directed Acyclic Graphs (DAG) visualization
+
+The [Directed Acyclic Graph](../directed_acyclic_graph/index.md) (DAG) visualization can help analyze the critical path in
+the pipeline and understand possible blockers.
+
+![CI Pipeline Critical Path with DAG](img/ci_efficiency_pipeline_dag_critical_path.png)
+
+### Pipeline Monitoring
+
+Global pipeline health is a key indicator to monitor along with job and pipeline duration.
+[CI/CD analytics](index.md#pipeline-success-and-duration-charts) give a visual
+representation of pipeline health.
+
+Instance administrators have access to additional [performance metrics and self-monitoring](../../administration/monitoring/index.md).
+
+You can fetch specific pipeline health metrics from the [API](../../api/README.md).
+External monitoring tools can poll the API and verify pipeline health or collect
+metrics for long term SLA analytics.
+
+For example, the [GitLab CI Pipelines Exporter](https://github.com/mvisonneau/gitlab-ci-pipelines-exporter)
+for Prometheus fetches metrics from the API. It can check branches in projects automatically
+and get the pipeline status and duration. In combination with a Grafana dashboard,
+this helps build an actionable view for your operations team. Metric graphs can also
+be embedded into incidents making problem resolving easier.
+
+![Grafana Dashboard for GitLab CI Pipelines Prometheus Exporter](img/ci_efficiency_pipeline_health_grafana_dashboard.png)
+
+Alternatively, you can use a monitoring tool that can execute scripts, like
+[`check_gitlab`](https://gitlab.com/6uellerBpanda/check_gitlab) for example.
+
+#### Runner monitoring
+
+You can also [monitor CI runners](https://docs.gitlab.com/runner/monitoring/) on
+their host systems, or in clusters like Kubernetes. This includes checking:
+
+- Disk and disk IO
+- CPU usage
+- Memory
+- Runner process resources
+
+The [Prometheus Node Exporter](https://prometheus.io/docs/guides/node-exporter/)
+can monitor runners on Linux hosts, and [`kube-state-metrics`](https://github.com/kubernetes/kube-state-metrics)
+runs in a Kubernetes cluster.
+
+You can also test [GitLab Runner auto-scaling](https://docs.gitlab.com/runner/configuration/autoscale.html)
+with cloud providers, and define offline times to reduce costs.
+
+#### Dashboards and incident management
+
+Use your existing monitoring tools and dashboards to integrate CI/CD pipeline monitoring,
+or build them from scratch. Ensure that the runtime data is actionable and useful
+in teams, and operations/SREs are able to identify problems early enough.
+[Incident management](../../operations/incident_management/index.md) can help here too,
+with embedded metric charts and all valuable details to analyze the problem.
+
+### Storage usage
+
+Review the storage use of the following to help analyze costs and efficiency:
+
+- [Job artifacts](job_artifacts.md) and their [`expire_in`](../yaml/README.md#artifactsexpire_in)
+ configuration. If kept for too long, storage usage grows and could slow pipelines down.
+- [Container registry](../../user/packages/container_registry/index.md) usage.
+- [Package registry](../../user/packages/package_registry/index.md) usage.
+
+## Pipeline configuration
+
+Make careful choices when configuring pipelines to speed up pipelines and reduce
+resource usage. This includes making use of GitLab CI/CD's built-in features that
+make pipelines run faster and more efficiently.
+
+### Reduce how often jobs run
+
+Try to find which jobs don't need to run in all situations, and use pipeline configuration
+to stop them from running:
+
+- Use the [`interruptible`](../yaml/README.md#interruptible) keyword to stop old pipelines
+ when they are superceded by a newer pipeline.
+- Use [`rules`](../yaml/README.md#rules) to skip tests that aren't needed. For example,
+ skip backend tests when only the frontend code is changed.
+- Run non-essential [scheduled pipelines](schedules.md) less frequently.
+
+### Fail fast
+
+Ensure that errors are detected early in the CI/CD pipeline. A job that takes a very long
+time to complete keeps a pipeline from returning a failed status until the job completes.
+
+Design pipelines so that jobs that can [fail fast](../../user/project/merge_requests/fail_fast_testing.md)
+run earlier. For example, add an early stage and move the syntax, style linting,
+Git commit message verification, and similar jobs in there.
+
+Decide if it's important for long jobs to run early, before fast feedback from
+faster jobs. The initial failures may make it clear that the rest of the pipeline
+shouldn't run, saving pipeline resources.
+
+### Directed Acyclic Graphs (DAG)
+
+In a basic configuration, jobs always wait for all other jobs in earlier stages to complete
+before running. This is the simplest configuration, but it's also the slowest in most
+cases. [Directed Acyclic Graphs](../directed_acyclic_graph/index.md) and
+[parent/child pipelines](../parent_child_pipelines.md) are more flexible and can
+be more efficient, but can also make pipelines harder to understand and analyze.
+
+### Caching
+
+Another optimization method is to use [caching](../caching/index.md) between jobs and stages,
+for example [`/node_modules` for NodeJS](../caching/index.md#caching-nodejs-dependencies).
+
+### Docker Images
+
+Downloading and initializing Docker images can be a large part of the overall runtime
+of jobs.
+
+If a Docker image is slowing down job execution, analyze the base image size and network
+connection to the registry. If GitLab is running in the cloud, look for a cloud container
+registry offered by the vendor. In addition to that, you can make use of the
+[GitLab container registry](../../user/packages/container_registry/index.md) which can be accessed
+by the GitLab instance faster than other registries.
+
+#### Optimize Docker images
+
+Build optimized Docker images because large Docker images use up a lot of space and
+take a long time to download with slower connection speeds. If possible, avoid using
+one large image for all jobs. Use multiple smaller images, each for a specific task,
+that download and run faster.
+
+Try to use custom Docker images with the software pre-installed. It's usually much
+faster to download a larger pre-configured image than to use a common image and install
+software on it each time.
+
+Methods to reduce Docker image size:
+
+- Use a small base image, for example `debian-slim`.
+- Do not install convenience tools like vim, curl, and so on, if they aren't strictly needed.
+- Create a dedicated development image.
+- Disable man pages and docs installed by packages to save space.
+- Reduce the `RUN` layers and combine software installation steps.
+- If using `apt`, add `--no-install-recommends` to avoid unnecessary packages.
+- Clean up caches and files that are no longer needed at the end. For example
+ `rm -rf /var/lib/apt/lists/*` for Debian and Ubuntu, or `yum clean all` for RHEL and CentOS.
+- Use tools like [dive](https://github.com/wagoodman/dive) or [DockerSlim](https://github.com/docker-slim/docker-slim)
+ to analyze and shrink images.
+
+To simplify Docker image management, you can create a dedicated group for managing
+[Docker images](../docker/README.md) and test, build and publish them with CI/CD pipelines.
+
+## Test, document, and learn
+
+Improving pipelines is an iterative process. Make small changes, monitor the effect,
+then iterate again. Many small improvements can add up to a large increase in pipeline
+efficiency.
+
+It can help to document the pipeline design and architecture. You can do this with
+[Mermaid charts in Markdown](../../user/markdown.md#mermaid) directly in the GitLab
+repository.
+
+Document CI/CD pipeline problems and incidents in issues, including research done
+and solutions found. This helps onboarding new team members, and also helps
+identify recurring problems with CI pipeline efficiency.
+
+### Learn More
+
+- [CI Monitoring Webcast Slides](https://docs.google.com/presentation/d/1ONwIIzRB7GWX-WOSziIIv8fz1ngqv77HO1yVfRooOHM/edit?usp=sharing)
+- [GitLab.com Monitoring Handbook](https://about.gitlab.com/handbook/engineering/monitoring/)
+- [Buildings dashboards for operational visibility](https://aws.amazon.com/builders-library/building-dashboards-for-operational-visibility/)
diff --git a/doc/development/contributing/index.md b/doc/development/contributing/index.md
index 7e10e152304..7550fe69546 100644
--- a/doc/development/contributing/index.md
+++ b/doc/development/contributing/index.md
@@ -86,6 +86,11 @@ If you would like to contribute to GitLab:
- Issues with the
[`~Accepting merge requests` label](issue_workflow.md#label-for-community-contributors)
are a great place to start.
+- Optimizing our tests is another great opportunity to contribute. You can use
+ [RSpec profiling statistics](https://gitlab-org.gitlab.io/rspec_profiling_stats/) to identify
+ slowest tests. These tests are good candidates for improving and checking if any of
+ [best practices](../testing_guide/best_practices.md)
+ could speed them up.
- Consult the [Contribution Flow](#contribution-flow) section to learn the process.
If you have any questions or need help visit [Getting Help](https://about.gitlab.com/get-help/) to
diff --git a/doc/user/search/advanced_search_syntax.md b/doc/user/search/advanced_search_syntax.md
index 42103e17d6c..804d4c540ac 100644
--- a/doc/user/search/advanced_search_syntax.md
+++ b/doc/user/search/advanced_search_syntax.md
@@ -47,13 +47,13 @@ Full details can be found in the [Elasticsearch documentation](https://www.elast
here's a quick guide:
- Searches look for all the words in a query, in any order - e.g.: searching
- issues for `display bug` will return all issues matching both those words, in any order.
-- To find the exact phrase (stemming still applies), use double quotes: `"display bug"`
-- To find bugs not mentioning display, use `-`: `bug -display`
-- To find a bug in display or sound, use `|`: `bug display | sound`
-- To group terms together, use parentheses: `bug | (display +sound)`
-- To match a partial word, use `*`: `bug find_by_*`
-- To find a term containing one of these symbols, use `\`: `argument \-last`
+ issues for [`display bug`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=issues&repository_ref=&search=display+bug&group_id=9970&project_id=278964) and [`bug display`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=issues&repository_ref=&search=bug+Display&group_id=9970&project_id=278964) will return the same results.
+- To find the exact phrase (stemming still applies), use double quotes: [`"display bug"`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=issues&repository_ref=&search=%22display+bug%22&group_id=9970&project_id=278964)
+- To find bugs not mentioning display, use `-`: [`bug -display`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=issues&repository_ref=&search=bug+-display&group_id=9970&project_id=278964)
+- To find a bug in display or banner, use `|`: [`bug display | banner`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=issues&repository_ref=&search=bug+display+%7C+banner&group_id=9970&project_id=278964)
+- To group terms together, use parentheses: [`bug | (display +banner)`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=issues&repository_ref=&search=bug+%7C+%28display+%2Bbanner%29&group_id=9970&project_id=278964)
+- To match a partial word, use `*`. In this example, I want to find bugs with any 500 errors. : [`bug error 50*`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=issues&repository_ref=&search=bug+error+50*&group_id=9970&project_id=278964)
+- To use one of symbols above literally, escape the symbol with a preceding `\`: [`argument \-last`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=blobs&repository_ref=&search=argument+%5C-last&group_id=9970&project_id=278964)
### Syntax search filters
@@ -68,11 +68,11 @@ To use them, simply add them to your query in the format `:`
Examples:
-- Finding a file with any content named `hello_world.rb`: `* filename:hello_world.rb`
-- Finding a file named `hello_world` with the text `whatever` inside of it: `whatever filename:hello_world`
-- Finding the text 'def create' inside files with the `.rb` extension: `def create extension:rb`
-- Finding the text `sha` inside files in a folder called `encryption`: `sha path:encryption`
-- Finding any file starting with `hello` containing `world` and with the `.js` extension: `world filename:hello* extension:js`
+- Finding a file with any content named `search_results.rb`: [`* filename:search_results.rb`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=blobs&repository_ref=&search=*+filename%3Asearch_results.rb&group_id=9970&project_id=278964)
+- Finding a file named `found_blob_spec.rb` with the text `CHANGELOG` inside of it: [`CHANGELOG filename:found_blob_spec.rb](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=blobs&repository_ref=&search=CHANGELOG+filename%3Afound_blob_spec.rb&group_id=9970&project_id=278964)
+- Finding the text `EpicLinks` inside files with the `.rb` extension: [`EpicLinks extension:rb`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=blobs&repository_ref=&search=EpicLinks+extension%3Arb&group_id=9970&project_id=278964)
+- Finding the text `Sidekiq` in a file, when that file is in a path that includes `elastic`: [`Sidekiq path:elastic`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=blobs&repository_ref=&search=Sidekiq+path%3Aelastic&group_id=9970&project_id=278964)
+- Syntax filters can be combined for complex filtering. Finding any file starting with `search` containing `eventHub` and with the `.js` extension: [`eventHub filename:search* extension:js`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=blobs&repository_ref=&search=eventHub+filename%3Asearch*+extension%3Ajs&group_id=9970&project_id=278964)
#### Excluding filters
@@ -86,7 +86,7 @@ Filters can be inversed to **filter out** results from the result set, by prefix
Examples:
-- Finding `rails` in all files but `Gemfile.lock`: `rails -filename:Gemfile.lock`
-- Finding `success` in all files excluding `.po|pot` files: `success -filename:*.po*`
-- Finding `import` excluding minified JavaScript (`.min.js`) files: `import -extension:min.js`
-- Finding `docs` for all files outside the `docs/` folder: `docs -path:docs/`
+- Finding `rails` in all files but `Gemfile.lock`: [`rails -filename:Gemfile.lock`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=blobs&repository_ref=&search=rails+-filename%3AGemfile.lock&group_id=9970&project_id=278964)
+- Finding `success` in all files excluding `.po|pot` files: [`success -filename:*.po*`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=blobs&repository_ref=&search=success+-filename%3A*.po*&group_id=9970&project_id=278964)
+- Finding `import` excluding minified JavaScript (`.min.js`) files: [`import -extension:min.js`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=blobs&repository_ref=&search=import+-extension%3Amin.js&group_id=9970&project_id=278964)
+- Finding `docs` for all files outside the `docs/` folder: [`docs -path:docs/`](https://gitlab.com/search?utf8=%E2%9C%93&snippets=&scope=blobs&repository_ref=&search=docs+-path%3Adocs%2F&group_id=9970&project_id=278964)
diff --git a/lib/gitlab/ci/artifact_file_reader.rb b/lib/gitlab/ci/artifact_file_reader.rb
index c2d17cc176e..6395a20ca99 100644
--- a/lib/gitlab/ci/artifact_file_reader.rb
+++ b/lib/gitlab/ci/artifact_file_reader.rb
@@ -45,6 +45,31 @@ module Gitlab
end
def read_zip_file!(file_path)
+ if ::Gitlab::Ci::Features.new_artifact_file_reader_enabled?(job.project)
+ read_with_new_artifact_file_reader(file_path)
+ else
+ read_with_legacy_artifact_file_reader(file_path)
+ end
+ end
+
+ def read_with_new_artifact_file_reader(file_path)
+ job.artifacts_file.use_open_file do |file|
+ zip_file = Zip::File.new(file, false, true)
+ entry = zip_file.find_entry(file_path)
+
+ unless entry
+ raise Error, "Path `#{file_path}` does not exist inside the `#{job.name}` artifacts archive!"
+ end
+
+ if entry.name_is_directory?
+ raise Error, "Path `#{file_path}` was expected to be a file but it was a directory!"
+ end
+
+ zip_file.read(entry)
+ end
+ end
+
+ def read_with_legacy_artifact_file_reader(file_path)
job.artifacts_file.use_file do |archive_path|
Zip::File.open(archive_path) do |zip_file|
entry = zip_file.find_entry(file_path)
diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb
index 13fb7ca868c..5ab89a56c13 100644
--- a/lib/gitlab/ci/features.rb
+++ b/lib/gitlab/ci/features.rb
@@ -78,6 +78,10 @@ module Gitlab
::Feature.enabled?(:ci_enable_live_trace, project) &&
::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: false)
end
+
+ def self.new_artifact_file_reader_enabled?(project)
+ ::Feature.enabled?(:ci_new_artifact_file_reader, project, default_enabled: false)
+ end
end
end
end
diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
index 3fd6a15dec4..53bf6daea4c 100644
--- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb
+++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
@@ -3,8 +3,6 @@
module Gitlab
module UsageDataCounters
module HLLRedisCounter
- include Gitlab::Utils::UsageData
-
DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks
DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days
DEFAULT_REDIS_SLOT = ''.freeze
@@ -33,6 +31,8 @@ module Gitlab
# * Track event: Gitlab::UsageDataCounters::HLLRedisCounter.track_event(user_id, 'g_compliance_dashboard')
# * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current)
class << self
+ include Gitlab::Utils::UsageData
+
def track_event(entity_id, event_name, time = Time.zone.now)
return unless Gitlab::CurrentSettings.usage_ping_enabled?
@@ -54,7 +54,7 @@ module Gitlab
keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date)
- Gitlab::Redis::HLL.count(keys: keys)
+ redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) }
end
def categories
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 0ea57afdc90..f38e9a97187 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -2678,9 +2678,6 @@ msgstr ""
msgid "An error occurred while checking group path. Please refresh and try again."
msgstr ""
-msgid "An error occurred while committing your changes."
-msgstr ""
-
msgid "An error occurred while creating the issue. Please try again."
msgstr ""
@@ -4114,7 +4111,7 @@ msgstr ""
msgid "Branch %{branch_name} was created. To set up auto deploy, choose a GitLab CI Yaml template and commit your changes. %{link_to_autodeploy_doc}"
msgstr ""
-msgid "Branch has changed"
+msgid "Branch changed"
msgstr ""
msgid "Branch is already taken"
@@ -4420,6 +4417,9 @@ msgstr ""
msgid "CLOSED (MOVED)"
msgstr ""
+msgid "CODEOWNERS rule violation"
+msgstr ""
+
msgid "CONTRIBUTING"
msgstr ""
@@ -7134,6 +7134,9 @@ msgstr ""
msgid "Could not change HEAD: branch '%{branch}' does not exist"
msgstr ""
+msgid "Could not commit. An unexpected error occurred."
+msgstr ""
+
msgid "Could not connect to FogBugz, check your URL"
msgstr ""
@@ -13324,9 +13327,6 @@ msgstr ""
msgid "In order to enable Service Desk for your instance, you must first set up incoming email."
msgstr ""
-msgid "In order to gather accurate feature usage data, it can take 1 to 2 weeks to see your index."
-msgstr ""
-
msgid "In order to personalize your experience with GitLab%{br_tag}we would like to know a bit more about you."
msgstr ""
@@ -14013,6 +14013,9 @@ msgstr ""
msgid "It looks like you have some draft commits in this branch."
msgstr ""
+msgid "It may be several days before you see feature usage data."
+msgstr ""
+
msgid "It must have a header row and at least two columns: the first column is the issue title and the second column is the issue description. The separator is automatically detected."
msgstr ""
@@ -17757,6 +17760,9 @@ msgstr ""
msgid "Other visibility settings have been disabled by the administrator."
msgstr ""
+msgid "Our documentation includes an example DevOps Score report."
+msgstr ""
+
msgid "Out-of-compliance with this project's policies and should be removed"
msgstr ""
@@ -25705,9 +25711,6 @@ msgstr ""
msgid "This board's scope is reduced"
msgstr ""
-msgid "This branch has changed since you started editing. Would you like to create a new branch?"
-msgstr ""
-
msgid "This chart could not be displayed"
msgstr ""
@@ -27056,6 +27059,9 @@ msgstr ""
msgid "Undo ignore"
msgstr ""
+msgid "Unexpected error"
+msgstr ""
+
msgid "Unfortunately, your email message to GitLab could not be processed."
msgstr ""
@@ -28715,6 +28721,9 @@ msgstr ""
msgid "Workflow Help"
msgstr ""
+msgid "Would you like to create a new branch?"
+msgstr ""
+
msgid "Write"
msgstr ""
diff --git a/scripts/utils.sh b/scripts/utils.sh
index 5218c1e2d98..9d188fc7b77 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -110,7 +110,7 @@ function get_job_id() {
let "page++"
done
- if [[ "${job_id}" == "" ]]; then
+ if [[ "${job_id}" == "null" ]]; then # jq prints "null" for non-existent attribute
echoerr "The '${job_name}' job ID couldn't be retrieved!"
else
echoinfo "The '${job_name}' job ID is ${job_id}"
@@ -142,7 +142,7 @@ function fail_pipeline_early() {
local dont_interrupt_me_job_id
dont_interrupt_me_job_id=$(get_job_id 'dont-interrupt-me' 'scope=success')
- if [[ "${dont_interrupt_me_job_id}" != "" ]]; then
+ if [[ -n "${dont_interrupt_me_job_id}" ]]; then
echoinfo "This pipeline cannot be interrupted due to \`dont-interrupt-me\` job ${dont_interrupt_me_job_id}"
else
echoinfo "Failing pipeline early for fast feedback due to test failures in rspec fail-fast."
diff --git a/spec/frontend/ide/components/commit_sidebar/form_spec.js b/spec/frontend/ide/components/commit_sidebar/form_spec.js
index 9245cefc183..56667d6b03d 100644
--- a/spec/frontend/ide/components/commit_sidebar/form_spec.js
+++ b/spec/frontend/ide/components/commit_sidebar/form_spec.js
@@ -1,10 +1,13 @@
import Vue from 'vue';
+import { getByText } from '@testing-library/dom';
import { createComponentWithStore } from 'helpers/vue_mount_component_helper';
import { projectData } from 'jest/ide/mock_data';
import waitForPromises from 'helpers/wait_for_promises';
import { createStore } from '~/ide/stores';
+import consts from '~/ide/stores/modules/commit/constants';
import CommitForm from '~/ide/components/commit_sidebar/form.vue';
import { leftSidebarViews } from '~/ide/constants';
+import { createCodeownersCommitError, createUnexpectedCommitError } from '~/ide/lib/errors';
describe('IDE commit form', () => {
const Component = Vue.extend(CommitForm);
@@ -259,21 +262,47 @@ describe('IDE commit form', () => {
});
});
- it('opens new branch modal if commitChanges throws an error', () => {
- vm.commitChanges.mockRejectedValue({ success: false });
+ it.each`
+ createError | props
+ ${() => createCodeownersCommitError('test message')} | ${{ actionPrimary: { text: 'Create new branch' } }}
+ ${createUnexpectedCommitError} | ${{ actionPrimary: null }}
+ `('opens error modal if commitError with $error', async ({ createError, props }) => {
+ jest.spyOn(vm.$refs.commitErrorModal, 'show');
- jest.spyOn(vm.$refs.createBranchModal, 'show').mockImplementation();
+ const error = createError();
+ store.state.commit.commitError = error;
- return vm
- .$nextTick()
- .then(() => {
- vm.$el.querySelector('.btn-success').click();
+ await vm.$nextTick();
- return vm.$nextTick();
- })
- .then(() => {
- expect(vm.$refs.createBranchModal.show).toHaveBeenCalled();
- });
+ expect(vm.$refs.commitErrorModal.show).toHaveBeenCalled();
+ expect(vm.$refs.commitErrorModal).toMatchObject({
+ actionCancel: { text: 'Cancel' },
+ ...props,
+ });
+ // Because of the legacy 'mountComponent' approach here, the only way to
+ // test the text of the modal is by viewing the content of the modal added to the document.
+ expect(document.body).toHaveText(error.messageHTML);
+ });
+ });
+
+ describe('with error modal with primary', () => {
+ beforeEach(() => {
+ jest.spyOn(vm.$store, 'dispatch').mockReturnValue(Promise.resolve());
+ });
+
+ it('updates commit action and commits', async () => {
+ store.state.commit.commitError = createCodeownersCommitError('test message');
+
+ await vm.$nextTick();
+
+ getByText(document.body, 'Create new branch').click();
+
+ await waitForPromises();
+
+ expect(vm.$store.dispatch.mock.calls).toEqual([
+ ['commit/updateCommitAction', consts.COMMIT_TO_NEW_BRANCH],
+ ['commit/commitChanges', undefined],
+ ]);
});
});
});
diff --git a/spec/frontend/ide/lib/errors_spec.js b/spec/frontend/ide/lib/errors_spec.js
new file mode 100644
index 00000000000..8c3fb378302
--- /dev/null
+++ b/spec/frontend/ide/lib/errors_spec.js
@@ -0,0 +1,70 @@
+import {
+ createUnexpectedCommitError,
+ createCodeownersCommitError,
+ createBranchChangedCommitError,
+ parseCommitError,
+} from '~/ide/lib/errors';
+
+const TEST_SPECIAL = '&special<';
+const TEST_SPECIAL_ESCAPED = '&special<';
+const TEST_MESSAGE = 'Test message.';
+const CODEOWNERS_MESSAGE =
+ 'Push to protected branches that contain changes to files matching CODEOWNERS is not allowed';
+const CHANGED_MESSAGE = 'Things changed since you started editing';
+
+describe('~/ide/lib/errors', () => {
+ const createResponseError = message => ({
+ response: {
+ data: {
+ message,
+ },
+ },
+ });
+
+ describe('createCodeownersCommitError', () => {
+ it('uses given message', () => {
+ expect(createCodeownersCommitError(TEST_MESSAGE)).toEqual({
+ title: 'CODEOWNERS rule violation',
+ messageHTML: TEST_MESSAGE,
+ canCreateBranch: true,
+ });
+ });
+
+ it('escapes special chars', () => {
+ expect(createCodeownersCommitError(TEST_SPECIAL)).toEqual({
+ title: 'CODEOWNERS rule violation',
+ messageHTML: TEST_SPECIAL_ESCAPED,
+ canCreateBranch: true,
+ });
+ });
+ });
+
+ describe('createBranchChangedCommitError', () => {
+ it.each`
+ message | expectedMessage
+ ${TEST_MESSAGE} | ${`${TEST_MESSAGE}
Would you like to create a new branch?`}
+ ${TEST_SPECIAL} | ${`${TEST_SPECIAL_ESCAPED}