From e0df05cfbc7f03e8fce27c5e93fda17f634721e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Wed, 3 Apr 2019 22:51:00 +0200 Subject: [PATCH] Require all templates to use default stages Our templates are free to override the 'stages:'. However, they should follow the convention to always include 'build', 'test', 'deploy' which are defaults when not defined. --- ...ll-templates-to-include-default-stages.yml | 5 ++ lib/gitlab/ci/config.rb | 8 +-- lib/gitlab/ci/config/external/file/local.rb | 4 +- .../templates/Android-Fastlane.gitlab-ci.yml | 1 + lib/gitlab/ci/templates/Android.gitlab-ci.yml | 4 -- .../ci/templates/Auto-DevOps.gitlab-ci.yml | 1 + lib/gitlab/ci/templates/Chef.gitlab-ci.yml | 6 ++- lib/gitlab/ci/templates/Go.gitlab-ci.yml | 1 + lib/gitlab/ci/templates/Mono.gitlab-ci.yml | 1 + .../ci/templates/OpenShift.gitlab-ci.yml | 2 + lib/gitlab/ci/templates/Packer.gitlab-ci.yml | 2 + .../ci/templates/Serverless.gitlab-ci.yml | 1 + lib/gitlab/ci/templates/Swift.gitlab-ci.yml | 2 + .../ci/templates/Terraform.gitlab-ci.yml | 1 + .../ci/templates/iOS-Fastlane.gitlab-ci.yml | 2 + lib/gitlab/template/base_template.rb | 6 +++ .../lib/gitlab/ci/templates/templates_spec.rb | 52 +++++++------------ spec/lib/gitlab/ci/yaml_processor_spec.rb | 13 +++-- 18 files changed, 63 insertions(+), 49 deletions(-) create mode 100644 changelogs/unreleased/require-all-templates-to-include-default-stages.yml diff --git a/changelogs/unreleased/require-all-templates-to-include-default-stages.yml b/changelogs/unreleased/require-all-templates-to-include-default-stages.yml new file mode 100644 index 00000000000..de8b07160ba --- /dev/null +++ b/changelogs/unreleased/require-all-templates-to-include-default-stages.yml @@ -0,0 +1,5 @@ +--- +title: Require all templates to use default stages +merge_request: 26954 +author: +type: fixed diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 15643fa03ac..f187e456993 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -73,17 +73,13 @@ module Gitlab def build_config(config, project:, sha:, user:) initial_config = Gitlab::Config::Loader::Yaml.new(config).load! - if project - process_external_files(initial_config, project: project, sha: sha, user: user) - else - initial_config - end + process_external_files(initial_config, project: project, sha: sha, user: user) end def process_external_files(config, project:, sha:, user:) Config::External::Processor.new(config, project: project, - sha: sha || project.repository.root_ref_sha, + sha: sha || project&.repository&.root_ref_sha, user: user, expandset: Set.new).perform end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index 229a06451e8..cac321ec4a6 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -21,7 +21,9 @@ module Gitlab private def validate_content! - if content.nil? + if context.project&.repository.nil? + errors.push("Local file `#{location}` does not have project!") + elsif content.nil? errors.push("Local file `#{location}` does not exist!") elsif content.blank? errors.push("Local file `#{location}` is empty!") diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml index 8e767b22360..120272200c6 100644 --- a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml @@ -6,6 +6,7 @@ stages: - environment - build - test + - deploy - internal - alpha - beta diff --git a/lib/gitlab/ci/templates/Android.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.gitlab-ci.yml index c169e3f7686..2be7f933462 100644 --- a/lib/gitlab/ci/templates/Android.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android.gitlab-ci.yml @@ -24,10 +24,6 @@ before_script: - yes | android-sdk-linux/tools/bin/sdkmanager --licenses - set -o pipefail -stages: - - build - - test - lintDebug: stage: build script: diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index f786022beb0..611b25c2113 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -60,6 +60,7 @@ variables: stages: - build - test + - deploy # dummy stage to follow the template guidelines - review - dast - staging diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml index 33507aa58e4..1e14aa8bea9 100644 --- a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -14,9 +14,11 @@ variables: KITCHEN_LOCAL_YAML: ".kitchen.dokken.yml" stages: + - build - lint - - unit + - test - functional + - deploy foodcritic: stage: lint @@ -29,7 +31,7 @@ cookstyle: - chef exec cookstyle . chefspec: - stage: unit + stage: test script: - chef exec rspec spec diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml index d572d7a1edc..55fda1a4799 100644 --- a/lib/gitlab/ci/templates/Go.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -18,6 +18,7 @@ before_script: stages: - test - build + - deploy format: stage: test diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml index 86d62b93313..10fb6be6c39 100644 --- a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -15,6 +15,7 @@ image: mono:latest stages: + - build - test - deploy diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml index 290b9997084..61a925e0d2d 100644 --- a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -1,7 +1,9 @@ image: ayufan/openshift-cli stages: + - build # dummy stage to follow the template guidelines - test + - deploy # dummy stage to follow the template guidelines - review - staging - production diff --git a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml index fa296057c72..83e179f37c3 100644 --- a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml @@ -9,6 +9,8 @@ before_script: stages: - validate + - build + - test - deploy validate: diff --git a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml index 4f3d08d98fe..4e0d3de9389 100644 --- a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml @@ -4,6 +4,7 @@ image: alpine:latest stages: - build + - test - deploy .serverless:build:image: diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml index ba8a802ba4f..9aa4abd4791 100644 --- a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -2,7 +2,9 @@ # This file assumes an own GitLab CI runner, setup on a macOS system. stages: - build + - test - archive + - deploy build_project: stage: build diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 7160fce26a8..f374bc7e26a 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -24,6 +24,7 @@ before_script: stages: - validate - build + - test - deploy validate: diff --git a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml index df6ac4d340d..a4a9e96c1d2 100644 --- a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml @@ -12,6 +12,8 @@ stages: - build + - test + - deploy variables: LC_ALL: "en_US.UTF-8" diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 0b4cc571dc0..14c9a3e0389 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -16,6 +16,12 @@ module Gitlab end alias_method :key, :name + def full_name + Pathname.new(@path) + .relative_path_from(self.class.base_dir) + .to_s + end + def content @finder.read(@path) end diff --git a/spec/lib/gitlab/ci/templates/templates_spec.rb b/spec/lib/gitlab/ci/templates/templates_spec.rb index 4e3681cd943..b52064b3036 100644 --- a/spec/lib/gitlab/ci/templates/templates_spec.rb +++ b/spec/lib/gitlab/ci/templates/templates_spec.rb @@ -3,46 +3,32 @@ require 'spec_helper' describe "CI YML Templates" do - ABSTRACT_TEMPLATES = %w[Serverless].freeze - # These templates depend on the presence of the `project` - # param to enable processing of `include:` within CI config. - PROJECT_DEPENDENT_TEMPLATES = %w[Auto-DevOps DAST].freeze + using RSpec::Parameterized::TableSyntax - def self.concrete_templates - Gitlab::Template::GitlabCiYmlTemplate.all.reject do |template| - ABSTRACT_TEMPLATES.include?(template.name) - end + subject { Gitlab::Ci::YamlProcessor.new(content) } + + where(:template_name) do + Gitlab::Template::GitlabCiYmlTemplate.all.map(&:full_name) end - def self.abstract_templates - Gitlab::Template::GitlabCiYmlTemplate.all.select do |template| - ABSTRACT_TEMPLATES.include?(template.name) + with_them do + let(:content) do + <<~EOS + include: + - template: #{template_name} + + concrete_build_implemented_by_a_user: + stage: test + script: do something + EOS end - end - describe 'concrete templates with CI/CD jobs' do - concrete_templates.each do |template| - it "#{template.name} template should be valid" do - # Trigger processing of included files - project = create(:project, :test_repo) if PROJECT_DEPENDENT_TEMPLATES.include?(template.name) - - expect { Gitlab::Ci::YamlProcessor.new(template.content, project: project) } - .not_to raise_error - end + it 'is valid' do + expect { subject }.not_to raise_error end - end - describe 'abstract templates without concrete jobs defined' do - abstract_templates.each do |template| - it "#{template.name} template should be valid after being implemented" do - content = template.content + <<~EOS - concrete_build_implemented_by_a_user: - stage: build - script: do something - EOS - - expect { Gitlab::Ci::YamlProcessor.new(content) }.not_to raise_error - end + it 'require default stages to be included' do + expect(subject.stages).to include(*Gitlab::Ci::Config::Entry::Stages.default) end end end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 63a0d54dcfc..518dd59d4a9 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -618,8 +618,8 @@ module Gitlab context "when an array is provided" do let(:include_content) { ["/local.gitlab-ci.yml"] } - it "does not return any error" do - expect { subject }.not_to raise_error + it "returns a validation error" do + expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /does not have project/) end end @@ -635,11 +635,18 @@ module Gitlab let(:include_content) do [ 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml', - '/templates/.after-script-template.yml', { template: 'Auto-DevOps.gitlab-ci.yml' } ] end + before do + WebMock.stub_request(:get, 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml') + .to_return( + status: 200, + headers: { 'Content-Type' => 'application/json' }, + body: 'prepare: { script: ls -al }') + end + it "does not return any error" do expect { subject }.not_to raise_error end