From 1a53f017b4c2106da3425f3dcbe40fb95fd44bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Fri, 7 Sep 2018 21:37:44 +0200 Subject: [PATCH] Make Repository#blob_data_at public CE mirror of 17de13ada1a98da060802c55889489a512183cd1 --- app/models/repository.rb | 20 ++-- doc/ci/yaml/README.md | 98 +++++++++++++++++++ lib/gitlab/ci/external_files/external_file.rb | 4 +- spec/lib/gitlab/ci/config_spec.rb | 22 +++++ .../ci/external_files/external_file_spec.rb | 2 +- .../gitlab/ci/external_files/mapper_spec.rb | 2 +- .../ci/external_files/processor_spec.rb | 2 +- 7 files changed, 132 insertions(+), 18 deletions(-) diff --git a/app/models/repository.rb b/app/models/repository.rb index 4b479f047ed..96d25e38cbe 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -980,10 +980,6 @@ class Repository blob_data_at(sha, '.gitlab/route-map.yml') end - def fetch_file_for(sha, path_to_file) - blob_data_at(sha, path_to_file) - end - def gitlab_ci_yml_for(sha, path = '.gitlab-ci.yml') blob_data_at(sha, path) end @@ -1003,14 +999,6 @@ class Repository remote_branch: merge_request.target_branch) end - def blob_data_at(sha, path) - blob = blob_at(sha, path) - return unless blob - - blob.load_all_data! - blob.data - end - def squash(user, merge_request) raw.squash(user, merge_request.id, branch: merge_request.target_branch, start_sha: merge_request.diff_start_sha, @@ -1019,6 +1007,14 @@ class Repository message: merge_request.title) end + def blob_data_at(sha, path) + blob = blob_at(sha, path) + return unless blob + + blob.load_all_data! + blob.data + end + private # TODO Generice finder, later split this on finders by Ref or Oid diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index d89705e8ead..418d5b88375 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -966,6 +966,104 @@ Additionally, if you have a job that unconditionally recreates the cache without reference to its previous contents, you can use `policy: push` in that job to skip the download step. +### include + +From 10.5 we can use `include` keyword to allow the inclusion of external yml files. + +```yaml +# Content of https://gitlab.com/awesome-project/raw/master/.gitlab-ci-before-script-template.yml +before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - ruby -v + - which ruby + - gem install bundler --no-ri --no-rdoc + - bundle install --jobs $(nproc) "${FLAGS[@]}" +``` + +```yaml +include: 'https://gitlab.com/awesome-project/raw/master/.gitlab-ci-before-script-template.yml' + +rspec: + script: + - bundle exec rspec + +rubocop: + script: + - bundle exec rubocop +``` + +In the above example `.gitlab-ci-before-script-template.yml` content will be automatically fetched and evaluated as it were a single local file. + +`include` supports two types of files: + - **local** to the same repository, referenced using the relative path, or + - **remote** in a different location, accessed using HTTP(S) protocol, referenced using the full URL + +Also, `include` supports a single string or an array of different values, so + +```yaml +include: '/templates/.gitlab-ci-templates.yml' +``` + +and + +```yaml +include: + - 'https://gitlab.com/same-group/another-project/raw/master/.gitlab-ci-templates.yml' + - '/templates/.gitlab-ci-templates.yml' +``` + +are both valid use cases. + +#### Restrictions + +- We can only use files that are currently tracked by Git on the default repository branch, so when using a **local file** make sure it's on the latest commit of your default branch, otherwise feel free to use a remote location. +- Since external files defined on `include` are evaluated first, the configuration on this files will take precedence over the content of `.gitlab-ci.yml`, for example: + +```yaml +# Content of http://company.com/default-gitlab-ci.yml +image: php:5-fpm-alpine + +job2: + script: php -v +``` + + +```yaml +include: + - http://company.com/default-gitlab-ci.yml + +image: ruby:2.1 + +job1: + script: ruby -v +``` + +In this case both, `job1` and `job2` will be executed with `php:5-fpm-alpine` + + + +#### Examples + +**Example of local files included** + +```yaml +include: '/templates/.gitlab-ci-templates.yml' +``` + +**Example of remote files included** + +```yaml +include: 'https://gitlab.com/awesome-project/raw/master/.gitlab-ci-templates.yml' +``` + +**Example of multiple files included** + +```yaml +include: + - 'https://gitlab.com/same-group/another-project/raw/master/.gitlab-ci-templates.yml' + - '/templates/.gitlab-ci-templates.yml' +``` + ## `artifacts` > **Notes:** diff --git a/lib/gitlab/ci/external_files/external_file.rb b/lib/gitlab/ci/external_files/external_file.rb index f481e9b0a39..4fffdc5f716 100644 --- a/lib/gitlab/ci/external_files/external_file.rb +++ b/lib/gitlab/ci/external_files/external_file.rb @@ -1,5 +1,3 @@ -require 'open-uri' - module Gitlab module Ci module ExternalFiles @@ -32,7 +30,7 @@ module Gitlab end def local_file_content - project.repository.fetch_file_for(sha, value) + project.repository.blob_data_at(sha, value) end def sha diff --git a/spec/lib/gitlab/ci/config_spec.rb b/spec/lib/gitlab/ci/config_spec.rb index 6e1edf76c19..fe9d3b3a570 100644 --- a/spec/lib/gitlab/ci/config_spec.rb +++ b/spec/lib/gitlab/ci/config_spec.rb @@ -192,4 +192,26 @@ describe Gitlab::Ci::Config do ) end end + + context "when both external files and gitlab_ci defined the same key" do + let(:yml) do + <<~HEREDOC + include: + - https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml + + image: ruby:2.2 + HEREDOC + end + + let(:http_file_content) do + <<~HEREDOC + image: php:5-fpm-alpine + HEREDOC + end + + it 'should take precedence' do + allow(HTTParty).to receive(:get).and_return(http_file_content) + expect(config.to_hash).to eq({ image: 'php:5-fpm-alpine' }) + end + end end diff --git a/spec/lib/gitlab/ci/external_files/external_file_spec.rb b/spec/lib/gitlab/ci/external_files/external_file_spec.rb index 822ed2970bf..b149b4200d8 100644 --- a/spec/lib/gitlab/ci/external_files/external_file_spec.rb +++ b/spec/lib/gitlab/ci/external_files/external_file_spec.rb @@ -1,4 +1,4 @@ -require 'rails_helper' +require 'fast_spec_helper' describe Gitlab::Ci::ExternalFiles::ExternalFile do let(:project) { create(:project, :repository) } diff --git a/spec/lib/gitlab/ci/external_files/mapper_spec.rb b/spec/lib/gitlab/ci/external_files/mapper_spec.rb index ab1bc242c19..84957c1f6b4 100644 --- a/spec/lib/gitlab/ci/external_files/mapper_spec.rb +++ b/spec/lib/gitlab/ci/external_files/mapper_spec.rb @@ -1,4 +1,4 @@ -require 'rails_helper' +require 'fast_spec_helper' describe Gitlab::Ci::ExternalFiles::Mapper do let(:project) { create(:project, :repository) } diff --git a/spec/lib/gitlab/ci/external_files/processor_spec.rb b/spec/lib/gitlab/ci/external_files/processor_spec.rb index a3db566f428..36eec0b90d3 100644 --- a/spec/lib/gitlab/ci/external_files/processor_spec.rb +++ b/spec/lib/gitlab/ci/external_files/processor_spec.rb @@ -1,4 +1,4 @@ -require 'rails_helper' +require 'fast_spec_helper' describe Gitlab::Ci::ExternalFiles::Processor do let(:project) { create(:project, :repository) }