Add timed incremental rollout to Auto DevOps

Auto DevOps deployment strategies now supports timed incremental rollout. We are deprecating the usage of INCREMENTAL_ROLLOUT_ENABLED environment variable in Auto DevOps template. The new behavior will be driven by the INCREMENTAL_ROLLOUT_MODE variable that can either be manual (same as INCREMENTAL_ROLLOUT_ENABLED) or timed. Rollout deployments will be executed using a 5 minute delay between each job.
2018-10-01 15:03:48 +02:00 · 2018-10-01 15:03:48 +02:00 · 94fc061936
parent 1a90632cc9
commit 94fc061936
8 changed files with 173 additions and 79 deletions
--- a/app/models/project_auto_devops.rb
+++ b/app/models/project_auto_devops.rb
@ -5,7 +5,8 @@ class ProjectAutoDevops < ActiveRecord::Base

  enum deploy_strategy: {
    continuous: 0,
-    manual: 1
+    manual: 1,
+    timed_incremental: 2
  }

  scope :enabled, -> { where(enabled: true) }
@ -30,10 +31,7 @@ class ProjectAutoDevops < ActiveRecord::Base
                         value: domain.presence || instance_domain)
      end

-      if manual?
-        variables.append(key: 'STAGING_ENABLED', value: '1')
-        variables.append(key: 'INCREMENTAL_ROLLOUT_ENABLED', value: '1')
-      end
+      variables.concat(deployment_strategy_default_variables)
    end
  end

@ -51,4 +49,16 @@ class ProjectAutoDevops < ActiveRecord::Base
      !project.public? &&
      !project.deploy_tokens.find_by(name: DeployToken::GITLAB_DEPLOY_TOKEN_NAME).present?
  end
+
+  def deployment_strategy_default_variables
+    Gitlab::Ci::Variables::Collection.new.tap do |variables|
+      if manual?
+        variables.append(key: 'STAGING_ENABLED', value: '1')
+        variables.append(key: 'INCREMENTAL_ROLLOUT_ENABLED', value: '1') # deprecated
+        variables.append(key: 'INCREMENTAL_ROLLOUT_MODE', value: 'manual')
+      elsif timed_incremental?
+        variables.append(key: 'INCREMENTAL_ROLLOUT_MODE', value: 'timed')
+      end
+    end
+  end
 end
--- a/app/views/projects/settings/ci_cd/_autodevops_form.html.haml
+++ b/app/views/projects/settings/ci_cd/_autodevops_form.html.haml
@ -39,10 +39,17 @@
                  = form.label :deploy_strategy_continuous, class: 'form-check-label' do
                    = s_('CICD|Continuous deployment to production')
                    = link_to icon('question-circle'), help_page_path('topics/autodevops/index.md', anchor: 'auto-deploy'), target: '_blank'
+
+                .form-check
+                  = form.radio_button :deploy_strategy, 'timed_incremental', class: 'form-check-input'
+                  = form.label :deploy_strategy_timed_incremental, class: 'form-check-label' do
+                    = s_('CICD|Continuous deployment to production using timed incremental rollout')
+                    = link_to icon('question-circle'), help_page_path('topics/autodevops/index.md', anchor: 'timed-incremental-rollout-to-production'), target: '_blank'
+
                .form-check
                  = form.radio_button :deploy_strategy, 'manual', class: 'form-check-input'
                  = form.label :deploy_strategy_manual, class: 'form-check-label' do
                    = s_('CICD|Automatic deployment to staging, manual deployment to production')
-                    = link_to icon('question-circle'), help_page_path('ci/environments.md', anchor: 'manually-deploying-to-environments'), target: '_blank'
+                    = link_to icon('question-circle'), help_page_path('topics/autodevops/index.md', anchor: 'incremental-rollout-to-production'), target: '_blank'

          = f.submit _('Save changes'), class: "btn btn-success prepend-top-15"
--- a/changelogs/unreleased/autodevops-timed-incremental-rollout.yml
+++ b/changelogs/unreleased/autodevops-timed-incremental-rollout.yml
@ -0,0 +1,5 @@
+---
+title: Add timed incremental rollout to Auto DevOps
+merge_request: 22023
+author:
+type: added
--- a/doc/topics/autodevops/index.md
+++ b/doc/topics/autodevops/index.md
@ -239,14 +239,19 @@ project's **Settings > CI/CD > Auto DevOps**.

 The available options are:

- **Continuous deployment to production** - enables [Auto Deploy](#auto-deploy)
-  by setting the [`STAGING_ENABLED`](#deploy-policy-for-staging-and-production-environments) and
-  [`INCREMENTAL_ROLLOUT_ENABLED`](#incremental-rollout-to-production) variables
-  to false.
- **Automatic deployment to staging, manual deployment to production** - sets the
+- **Continuous deployment to production**: Enables [Auto Deploy](#auto-deploy)
+  with `master` branch directly deployed to production.
+- **Continuous deployment to production using timed incremental rollout**: Sets the
+  [`INCREMENTAL_ROLLOUT_MODE`](#timed-incremental-rollout-to-production) variable
+  to `timed`, and production deployment will be executed with a 5 minute delay between
+  each increment in rollout.
+- **Automatic deployment to staging, manual deployment to production**: Sets the
  [`STAGING_ENABLED`](#deploy-policy-for-staging-and-production-environments) and
-  [`INCREMENTAL_ROLLOUT_ENABLED`](#incremental-rollout-to-production) variables
-  to true, and the user is responsible for manually deploying to staging and production.
+  [`INCREMENTAL_ROLLOUT_MODE`](#incremental-rollout-to-production) variables
+  to `1` and `manual`. This means:
+
+  - `master` branch is directly deployed to staging.
+  - Manual actions are provided for incremental rollout to production.

 ## Stages of Auto DevOps

@ -609,7 +614,7 @@ also be customized, and you can easily use a [custom buildpack](#custom-buildpac
 | `DB_MIGRATE`                 | From GitLab 11.4, this variable can be used to specify the command to run to migrate the application's PostgreSQL database. It runs inside the application pod. |
 | `STAGING_ENABLED`            | From GitLab 10.8, this variable can be used to define a [deploy policy for staging and production environments](#deploy-policy-for-staging-and-production-environments). |
 | `CANARY_ENABLED`             | From GitLab 11.0, this variable can be used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments). |
-| `INCREMENTAL_ROLLOUT_ENABLED`| From GitLab 10.8, this variable can be used to enable an [incremental rollout](#incremental-rollout-to-production) of your application for the production environment. |
+| `INCREMENTAL_ROLLOUT_MODE`| From GitLab 11.4, this variable, if present, can be used to enable an [incremental rollout](#incremental-rollout-to-production) of your application for the production environment.<br/>Set to: <ul><li>`manual`, for manual deployment jobs.</li><li>`timed`, for automatic rollout deployments with a 5 minute delay each one.</li></ul> |
 | `TEST_DISABLED`              | From GitLab 11.0, this variable can be used to disable the `test` job. If the variable is present, the job will not be created. |
 | `CODE_QUALITY_DISABLED`       | From GitLab 11.0, this variable can be used to disable the `codequality` job. If the variable is present, the job will not be created. |
 | `SAST_DISABLED`              | From GitLab 11.0, this variable can be used to disable the `sast` job. If the variable is present, the job will not be created. |
@ -730,9 +735,8 @@ to use an incremental rollout to replace just a few pods with the latest code.
 This will allow you to first check how the app is behaving, and later manually
 increasing the rollout up to 100%.

-If `INCREMENTAL_ROLLOUT_ENABLED` is defined in your project (e.g., set
-`INCREMENTAL_ROLLOUT_ENABLED` to `1` as a secret variable), then instead of the
-standard `production` job, 4 different
+If `INCREMENTAL_ROLLOUT_MODE` is set to `manual` in your project, then instead
+of the standard `production` job, 4 different
 [manual jobs](../../ci/pipelines.md#manual-actions-from-the-pipeline-graph)
 will be created:

@ -756,21 +760,45 @@ environment page.
 Below, you can see how the pipeline will look if the rollout or staging
 variables are defined.

- **Without `INCREMENTAL_ROLLOUT_ENABLED` and without `STAGING_ENABLED`**
+Without `INCREMENTAL_ROLLOUT_MODE` and without `STAGING_ENABLED`:

-    ![Staging and rollout disabled](img/rollout_staging_disabled.png)
+![Staging and rollout disabled](img/rollout_staging_disabled.png)

- **Without `INCREMENTAL_ROLLOUT_ENABLED` and with `STAGING_ENABLED`**
+Without `INCREMENTAL_ROLLOUT_MODE` and with `STAGING_ENABLED`:

-    ![Staging enabled](img/staging_enabled.png)
+![Staging enabled](img/staging_enabled.png)

- **With `INCREMENTAL_ROLLOUT_ENABLED` and without `STAGING_ENABLED`**
+With `INCREMENTAL_ROLLOUT_MODE` set to `manual` and without `STAGING_ENABLED`:

-    ![Rollout enabled](img/rollout_enabled.png)
+![Rollout enabled](img/rollout_enabled.png)

- **With `INCREMENTAL_ROLLOUT_ENABLED` and with `STAGING_ENABLED`**
+With `INCREMENTAL_ROLLOUT_MODE` set to `manual` and with `STAGING_ENABLED`

-    ![Rollout and staging enabled](img/rollout_staging_enabled.png)
+![Rollout and staging enabled](img/rollout_staging_enabled.png)
+
+CAUTION: **Caution:**
+Before GitLab 11.4 this feature was enabled by the presence of the
+`INCREMENTAL_ROLLOUT_ENABLED` environment variable.
+This configuration is deprecated and will be removed in the future.
+
+#### Timed incremental rollout to production **[PREMIUM]**
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/7545) in GitLab 11.4.
+
+TIP: **Tip:**
+You can also set this inside your [project's settings](#deployment-strategy).
+
+This configuration based on
+[incremental rollout to production](#incremental-rollout-to-production).
+
+Everything behaves the same way, except:
+
+- It's enabled by setting the `INCREMENTAL_ROLLOUT_MODE` variable to `timed`.
+- Instead of the standard `production` job, the following jobs with a 5 minute delay between each are created:
+    1. `timed rollout 10%`
+    1. `timed rollout 25%`
+    1. `timed rollout 50%`
+    1. `timed rollout 100%`

 ## Currently supported languages

--- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
@ -25,8 +25,9 @@
 # level, or manually added below.
 #
 # Continuous deployment to production is enabled by default.
-# If you want to deploy to staging first, or enable incremental rollouts,
-# set STAGING_ENABLED or INCREMENTAL_ROLLOUT_ENABLED environment variables.
+# If you want to deploy to staging first, set STAGING_ENABLED environment variable.
+# If you want to enable incremental rollout, either manual or time based,
+# set INCREMENTAL_ROLLOUT_TYPE environment variable to "manual" or "timed".
 # If you want to use canary deployments, set CANARY_ENABLED environment variable.
 #
 # If Auto DevOps fails to detect the proper buildpack, or if you want to
@ -61,6 +62,10 @@ stages:
  - staging
  - canary
  - production
+  - incremental rollout 10%
+  - incremental rollout 25%
+  - incremental rollout 50%
+  - incremental rollout 100%
  - performance
  - cleanup

@ -282,11 +287,6 @@ stop_review:
    variables:
      - $REVIEW_DISABLED

-# Keys that start with a dot (.) will not be processed by GitLab CI.
-# Staging and canary jobs are disabled by default, to enable them
-# remove the dot (.) before the job name.
-# https://docs.gitlab.com/ee/ci/yaml/README.html#hidden-keys
-
 # Staging deploys are disabled by default since
 # continuous deployment to production is enabled by default
 # If you prefer to automatically deploy to staging and
@ -368,6 +368,7 @@ production:
      - $STAGING_ENABLED
      - $CANARY_ENABLED
      - $INCREMENTAL_ROLLOUT_ENABLED
+      - $INCREMENTAL_ROLLOUT_MODE

 production_manual:
  <<: *production_template
@ -383,11 +384,11 @@ production_manual:
  except:
    variables:
      - $INCREMENTAL_ROLLOUT_ENABLED
+      - $INCREMENTAL_ROLLOUT_MODE

 # This job implements incremental rollout on for every push to `master`.

 .rollout: &rollout_template
-  stage: production
  script:
    - check_kube_domain
    - install_dependencies
@ -405,52 +406,77 @@ production_manual:
  artifacts:
    paths: [environment_url.txt]

-rollout 10%:
+.manual_rollout_template: &manual_rollout_template
  <<: *rollout_template
+  stage: production
+  when: manual
+  # This selectors are backward compatible mode with $INCREMENTAL_ROLLOUT_ENABLED (before 11.4)
+  only:
+    refs:
+      - master
+    kubernetes: active
+    variables:
+      - $INCREMENTAL_ROLLOUT_MODE == "manual"
+      - $INCREMENTAL_ROLLOUT_ENABLED
+  except:
+    variables:
+      - $INCREMENTAL_ROLLOUT_MODE == "timed"
+
+.timed_rollout_template: &timed_rollout_template
+  <<: *rollout_template
+  when: delayed
+  start_in: 5 minutes
+  only:
+    refs:
+      - master
+    kubernetes: active
+    variables:
+      - $INCREMENTAL_ROLLOUT_MODE == "timed"
+
+timed rollout 10%:
+  <<: *timed_rollout_template
+  stage: incremental rollout 10%
  variables:
    ROLLOUT_PERCENTAGE: 10
-  when: manual
-  only:
-    refs:
-      - master
-    kubernetes: active
-    variables:
-      - $INCREMENTAL_ROLLOUT_ENABLED

-rollout 25%:
-  <<: *rollout_template
+timed rollout 25%:
+  <<: *timed_rollout_template
+  stage: incremental rollout 25%
  variables:
    ROLLOUT_PERCENTAGE: 25
-  when: manual
-  only:
-    refs:
-      - master
-    kubernetes: active
-    variables:
-      - $INCREMENTAL_ROLLOUT_ENABLED

-rollout 50%:
-  <<: *rollout_template
+timed rollout 50%:
+  <<: *timed_rollout_template
+  stage: incremental rollout 50%
+  variables:
+    ROLLOUT_PERCENTAGE: 50
+
+timed rollout 100%:
+  <<: *timed_rollout_template
+  <<: *production_template
+  stage: incremental rollout 100%
+  variables:
+    ROLLOUT_PERCENTAGE: 100
+
+rollout 10%:
+  <<: *manual_rollout_template
+  variables:
+    ROLLOUT_PERCENTAGE: 10
+
+rollout 25%:
+  <<: *manual_rollout_template
+  variables:
+    ROLLOUT_PERCENTAGE: 25
+
+rollout 50%:
+  <<: *manual_rollout_template
  variables:
    ROLLOUT_PERCENTAGE: 50
-  when: manual
-  only:
-    refs:
-      - master
-    kubernetes: active
-    variables:
-      - $INCREMENTAL_ROLLOUT_ENABLED

 rollout 100%:
+  <<: *manual_rollout_template
  <<: *production_template
-  when: manual
  allow_failure: false
-  only:
-    refs:
-      - master
-    kubernetes: active
-    variables:
-      - $INCREMENTAL_ROLLOUT_ENABLED

 # ---------------------------------------------------------------------------

--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@ -1081,6 +1081,9 @@ msgstr ""
 msgid "CICD|Continuous deployment to production"
 msgstr ""

+msgid "CICD|Continuous deployment to production using timed incremental rollout"
+msgstr ""
+
 msgid "CICD|Default to Auto DevOps pipeline"
 msgstr ""

--- a/spec/factories/project_auto_devops.rb
+++ b/spec/factories/project_auto_devops.rb
@ -5,8 +5,16 @@ FactoryBot.define do
    domain "example.com"
    deploy_strategy :continuous

-    trait :manual do
-      deploy_strategy :manual
+    trait :continuous_deployment do
+      deploy_strategy ProjectAutoDevops.deploy_strategies[:continuous] # rubocop:disable FactoryBot/DynamicAttributeDefinedStatically
+    end
+
+    trait :manual_deployment do
+      deploy_strategy ProjectAutoDevops.deploy_strategies[:manual] # rubocop:disable FactoryBot/DynamicAttributeDefinedStatically
+    end
+
+    trait :timed_incremental_deployment do
+      deploy_strategy ProjectAutoDevops.deploy_strategies[:timed_incremental] # rubocop:disable FactoryBot/DynamicAttributeDefinedStatically
    end

    trait :disabled do
--- a/spec/models/project_auto_devops_spec.rb
+++ b/spec/models/project_auto_devops_spec.rb
@ -70,24 +70,31 @@ describe ProjectAutoDevops do
    end

    context 'when deploy_strategy is manual' do
-      let(:domain) { 'example.com' }
-
-      before do
-        auto_devops.deploy_strategy = 'manual'
+      let(:auto_devops) { build_stubbed(:project_auto_devops, :manual_deployment, project: project) }
+      let(:expected_variables) do
+        [
+          { key: 'INCREMENTAL_ROLLOUT_MODE', value: 'manual' },
+          { key: 'STAGING_ENABLED', value: '1' },
+          { key: 'INCREMENTAL_ROLLOUT_ENABLED', value: '1' }
+        ]
      end

-      it do
-        expect(auto_devops.predefined_variables.map { |var| var[:key] })
-          .to include("STAGING_ENABLED", "INCREMENTAL_ROLLOUT_ENABLED")
-      end
+      it { expect(auto_devops.predefined_variables).to include(*expected_variables) }
    end

    context 'when deploy_strategy is continuous' do
-      let(:domain) { 'example.com' }
+      let(:auto_devops) { build_stubbed(:project_auto_devops, :continuous_deployment, project: project) }

-      before do
-        auto_devops.deploy_strategy = 'continuous'
+      it do
+        expect(auto_devops.predefined_variables.map { |var| var[:key] })
+          .not_to include("STAGING_ENABLED", "INCREMENTAL_ROLLOUT_ENABLED")
      end
+    end
+
+    context 'when deploy_strategy is timed_incremental' do
+      let(:auto_devops) { build_stubbed(:project_auto_devops, :timed_incremental_deployment, project: project) }
+
+      it { expect(auto_devops.predefined_variables).to include(key: 'INCREMENTAL_ROLLOUT_MODE', value: 'timed') }

      it do
        expect(auto_devops.predefined_variables.map { |var| var[:key] })