Merge branch 'feature/add-support-for-services-configuration' into 'master'

Add support for services configuration in .gitlab-ci.yml

See merge request !8578
This commit is contained in:
Grzegorz Bizon 2017-06-14 07:57:47 +00:00
commit b29bf62602
18 changed files with 548 additions and 140 deletions

View file

@ -0,0 +1,4 @@
---
title: Add support for image and services configuration in .gitlab-ci.yml
merge_request: 8578
author:

View file

@ -804,7 +804,11 @@ module API
end
class Image < Grape::Entity
expose :name
expose :name, :entrypoint
end
class Service < Image
expose :alias, :command
end
class Artifacts < Grape::Entity
@ -848,7 +852,7 @@ module API
expose :variables
expose :steps, using: Step
expose :image, using: Image
expose :services, using: Image
expose :services, using: Service
expose :artifacts, using: Artifacts
expose :cache, using: Cache
expose :credentials, using: Credentials

View file

@ -45,7 +45,21 @@ module Ci
expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? }
expose :options do |model|
model.options
# This part ensures that output of old API is still the same after adding support
# for extended docker configuration options, used by new API
#
# I'm leaving this here, not in the model, because it should be removed at the same time
# when old API will be removed (planned for August 2017).
model.options.dup.tap do |options|
options[:image] = options[:image][:name] if options[:image].is_a?(Hash)
options[:services].map! do |service|
if service.is_a?(Hash)
service[:name]
else
service
end
end
end
end
expose :timeout do |model|

View file

@ -2,7 +2,7 @@ module Gitlab
module Ci
module Build
class Image
attr_reader :name
attr_reader :alias, :command, :entrypoint, :name
class << self
def from_image(job)
@ -21,7 +21,14 @@ module Gitlab
end
def initialize(image)
@name = image
if image.is_a?(String)
@name = image
elsif image.is_a?(Hash)
@alias = image[:alias]
@command = image[:command]
@entrypoint = image[:entrypoint]
@name = image[:name]
end
end
def valid?

View file

@ -8,8 +8,36 @@ module Gitlab
class Image < Node
include Validatable
ALLOWED_KEYS = %i[name entrypoint].freeze
validations do
validates :config, type: String
validates :config, hash_or_string: true
validates :config, allowed_keys: ALLOWED_KEYS
validates :name, type: String, presence: true
validates :entrypoint, type: String, allow_nil: true
end
def hash?
@config.is_a?(Hash)
end
def string?
@config.is_a?(String)
end
def name
value[:name]
end
def entrypoint
value[:entrypoint]
end
def value
return { name: @config } if string?
return @config if hash?
{}
end
end
end

View file

@ -0,0 +1,34 @@
module Gitlab
module Ci
class Config
module Entry
##
# Entry that represents a configuration of Docker service.
#
class Service < Image
include Validatable
ALLOWED_KEYS = %i[name entrypoint command alias].freeze
validations do
validates :config, hash_or_string: true
validates :config, allowed_keys: ALLOWED_KEYS
validates :name, type: String, presence: true
validates :entrypoint, type: String, allow_nil: true
validates :command, type: String, allow_nil: true
validates :alias, type: String, allow_nil: true
end
def alias
value[:alias]
end
def command
value[:command]
end
end
end
end
end
end

View file

@ -9,7 +9,30 @@ module Gitlab
include Validatable
validations do
validates :config, array_of_strings: true
validates :config, type: Array
end
def compose!(deps = nil)
super do
@entries = []
@config.each do |config|
@entries << Entry::Factory.new(Entry::Service)
.value(config || {})
.create!
end
@entries.each do |entry|
entry.compose!(deps)
end
end
end
def value
@entries.map(&:value)
end
def descendants
@entries
end
end
end

View file

@ -44,6 +44,14 @@ module Gitlab
end
end
class HashOrStringValidator < ActiveModel::EachValidator
def validate_each(record, attribute, value)
unless value.is_a?(Hash) || value.is_a?(String)
record.errors.add(attribute, 'should be a hash or a string')
end
end
end
class KeyValidator < ActiveModel::EachValidator
include LegacyValidationHelpers

View file

@ -194,8 +194,8 @@ FactoryGirl.define do
trait :extended_options do
options do
{
image: 'ruby:2.1',
services: ['postgres'],
image: { name: 'ruby:2.1', entrypoint: '/bin/sh' },
services: ['postgres', { name: 'docker:dind', entrypoint: '/bin/sh', command: 'sleep 30', alias: 'docker' }],
after_script: %w(ls date),
artifacts: {
name: 'artifacts_file',

View file

@ -596,62 +596,117 @@ module Ci
end
describe "Image and service handling" do
it "returns image and service when defined" do
config = YAML.dump({
image: "ruby:2.1",
services: ["mysql"],
before_script: ["pwd"],
rspec: { script: "rspec" }
})
context "when extended docker configuration is used" do
it "returns image and service when defined" do
config = YAML.dump({ image: { name: "ruby:2.1" },
services: ["mysql", { name: "docker:dind", alias: "docker" }],
before_script: ["pwd"],
rspec: { script: "rspec" } })
config_processor = GitlabCiYamlProcessor.new(config, path)
config_processor = GitlabCiYamlProcessor.new(config, path)
expect(config_processor.builds_for_stage_and_ref("test", "master").size).to eq(1)
expect(config_processor.builds_for_stage_and_ref("test", "master").first).to eq({
stage: "test",
stage_idx: 1,
name: "rspec",
commands: "pwd\nrspec",
coverage_regex: nil,
tag_list: [],
options: {
image: "ruby:2.1",
services: ["mysql"]
},
allow_failure: false,
when: "on_success",
environment: nil,
yaml_variables: []
})
expect(config_processor.builds_for_stage_and_ref("test", "master").size).to eq(1)
expect(config_processor.builds_for_stage_and_ref("test", "master").first).to eq({
stage: "test",
stage_idx: 1,
name: "rspec",
commands: "pwd\nrspec",
coverage_regex: nil,
tag_list: [],
options: {
image: { name: "ruby:2.1" },
services: [{ name: "mysql" }, { name: "docker:dind", alias: "docker" }]
},
allow_failure: false,
when: "on_success",
environment: nil,
yaml_variables: []
})
end
it "returns image and service when overridden for job" do
config = YAML.dump({ image: "ruby:2.1",
services: ["mysql"],
before_script: ["pwd"],
rspec: { image: { name: "ruby:2.5" },
services: [{ name: "postgresql", alias: "db-pg" }, "docker:dind"], script: "rspec" } })
config_processor = GitlabCiYamlProcessor.new(config, path)
expect(config_processor.builds_for_stage_and_ref("test", "master").size).to eq(1)
expect(config_processor.builds_for_stage_and_ref("test", "master").first).to eq({
stage: "test",
stage_idx: 1,
name: "rspec",
commands: "pwd\nrspec",
coverage_regex: nil,
tag_list: [],
options: {
image: { name: "ruby:2.5" },
services: [{ name: "postgresql", alias: "db-pg" }, { name: "docker:dind" }]
},
allow_failure: false,
when: "on_success",
environment: nil,
yaml_variables: []
})
end
end
it "returns image and service when overridden for job" do
config = YAML.dump({
image: "ruby:2.1",
services: ["mysql"],
before_script: ["pwd"],
rspec: { image: "ruby:2.5", services: ["postgresql"], script: "rspec" }
})
context "when etended docker configuration is not used" do
it "returns image and service when defined" do
config = YAML.dump({ image: "ruby:2.1",
services: ["mysql", "docker:dind"],
before_script: ["pwd"],
rspec: { script: "rspec" } })
config_processor = GitlabCiYamlProcessor.new(config, path)
config_processor = GitlabCiYamlProcessor.new(config, path)
expect(config_processor.builds_for_stage_and_ref("test", "master").size).to eq(1)
expect(config_processor.builds_for_stage_and_ref("test", "master").first).to eq({
stage: "test",
stage_idx: 1,
name: "rspec",
commands: "pwd\nrspec",
coverage_regex: nil,
tag_list: [],
options: {
image: "ruby:2.5",
services: ["postgresql"]
},
allow_failure: false,
when: "on_success",
environment: nil,
yaml_variables: []
})
expect(config_processor.builds_for_stage_and_ref("test", "master").size).to eq(1)
expect(config_processor.builds_for_stage_and_ref("test", "master").first).to eq({
stage: "test",
stage_idx: 1,
name: "rspec",
commands: "pwd\nrspec",
coverage_regex: nil,
tag_list: [],
options: {
image: { name: "ruby:2.1" },
services: [{ name: "mysql" }, { name: "docker:dind" }]
},
allow_failure: false,
when: "on_success",
environment: nil,
yaml_variables: []
})
end
it "returns image and service when overridden for job" do
config = YAML.dump({ image: "ruby:2.1",
services: ["mysql"],
before_script: ["pwd"],
rspec: { image: "ruby:2.5", services: ["postgresql", "docker:dind"], script: "rspec" } })
config_processor = GitlabCiYamlProcessor.new(config, path)
expect(config_processor.builds_for_stage_and_ref("test", "master").size).to eq(1)
expect(config_processor.builds_for_stage_and_ref("test", "master").first).to eq({
stage: "test",
stage_idx: 1,
name: "rspec",
commands: "pwd\nrspec",
coverage_regex: nil,
tag_list: [],
options: {
image: { name: "ruby:2.5" },
services: [{ name: "postgresql" }, { name: "docker:dind" }]
},
allow_failure: false,
when: "on_success",
environment: nil,
yaml_variables: []
})
end
end
end
@ -884,8 +939,8 @@ module Ci
coverage_regex: nil,
tag_list: [],
options: {
image: "ruby:2.1",
services: ["mysql"],
image: { name: "ruby:2.1" },
services: [{ name: "mysql" }],
artifacts: {
name: "custom_name",
paths: ["logs/", "binaries/"],
@ -1261,7 +1316,7 @@ EOT
config = YAML.dump({ image: ["test"], rspec: { script: "test" } })
expect do
GitlabCiYamlProcessor.new(config, path)
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "image config should be a string")
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "image config should be a hash or a string")
end
it "returns errors if job name is blank" do
@ -1282,35 +1337,35 @@ EOT
config = YAML.dump({ rspec: { script: "test", image: ["test"] } })
expect do
GitlabCiYamlProcessor.new(config, path)
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "jobs:rspec:image config should be a string")
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "jobs:rspec:image config should be a hash or a string")
end
it "returns errors if services parameter is not an array" do
config = YAML.dump({ services: "test", rspec: { script: "test" } })
expect do
GitlabCiYamlProcessor.new(config, path)
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "services config should be an array of strings")
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "services config should be a array")
end
it "returns errors if services parameter is not an array of strings" do
config = YAML.dump({ services: [10, "test"], rspec: { script: "test" } })
expect do
GitlabCiYamlProcessor.new(config, path)
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "services config should be an array of strings")
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "service config should be a hash or a string")
end
it "returns errors if job services parameter is not an array" do
config = YAML.dump({ rspec: { script: "test", services: "test" } })
expect do
GitlabCiYamlProcessor.new(config, path)
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "jobs:rspec:services config should be an array of strings")
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "jobs:rspec:services config should be a array")
end
it "returns errors if job services parameter is not an array of strings" do
config = YAML.dump({ rspec: { script: "test", services: [10, "test"] } })
expect do
GitlabCiYamlProcessor.new(config, path)
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "jobs:rspec:services config should be an array of strings")
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "service config should be a hash or a string")
end
it "returns error if job configuration is invalid" do
@ -1324,7 +1379,7 @@ EOT
config = YAML.dump({ extra: { script: 'rspec', services: "test" } })
expect do
GitlabCiYamlProcessor.new(config, path)
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "jobs:extra:services config should be an array of strings")
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "jobs:extra:services config should be a array")
end
it "returns errors if there are no jobs defined" do

View file

@ -10,12 +10,28 @@ describe Gitlab::Ci::Build::Image do
let(:image_name) { 'ruby:2.1' }
let(:job) { create(:ci_build, options: { image: image_name } ) }
it 'fabricates an object of the proper class' do
is_expected.to be_kind_of(described_class)
context 'when image is defined as string' do
it 'fabricates an object of the proper class' do
is_expected.to be_kind_of(described_class)
end
it 'populates fabricated object with the proper name attribute' do
expect(subject.name).to eq(image_name)
end
end
it 'populates fabricated object with the proper name attribute' do
expect(subject.name).to eq(image_name)
context 'when image is defined as hash' do
let(:entrypoint) { '/bin/sh' }
let(:job) { create(:ci_build, options: { image: { name: image_name, entrypoint: entrypoint } } ) }
it 'fabricates an object of the proper class' do
is_expected.to be_kind_of(described_class)
end
it 'populates fabricated object with the proper attributes' do
expect(subject.name).to eq(image_name)
expect(subject.entrypoint).to eq(entrypoint)
end
end
context 'when image name is empty' do
@ -41,10 +57,39 @@ describe Gitlab::Ci::Build::Image do
let(:service_image_name) { 'postgres' }
let(:job) { create(:ci_build, options: { services: [service_image_name] }) }
it 'fabricates an non-empty array of objects' do
is_expected.to be_kind_of(Array)
is_expected.not_to be_empty
expect(subject.first.name).to eq(service_image_name)
context 'when service is defined as string' do
it 'fabricates an non-empty array of objects' do
is_expected.to be_kind_of(Array)
is_expected.not_to be_empty
end
it 'populates fabricated objects with the proper name attributes' do
expect(subject.first).to be_kind_of(described_class)
expect(subject.first.name).to eq(service_image_name)
end
end
context 'when service is defined as hash' do
let(:service_entrypoint) { '/bin/sh' }
let(:service_alias) { 'db' }
let(:service_command) { 'sleep 30' }
let(:job) do
create(:ci_build, options: { services: [{ name: service_image_name, entrypoint: service_entrypoint,
alias: service_alias, command: service_command }] })
end
it 'fabricates an non-empty array of objects' do
is_expected.to be_kind_of(Array)
is_expected.not_to be_empty
expect(subject.first).to be_kind_of(described_class)
end
it 'populates fabricated objects with the proper attributes' do
expect(subject.first.name).to eq(service_image_name)
expect(subject.first.entrypoint).to eq(service_entrypoint)
expect(subject.first.alias).to eq(service_alias)
expect(subject.first.command).to eq(service_command)
end
end
context 'when service image name is empty' do

View file

@ -95,13 +95,13 @@ describe Gitlab::Ci::Config::Entry::Global do
describe '#image_value' do
it 'returns valid image' do
expect(global.image_value).to eq 'ruby:2.2'
expect(global.image_value).to eq(name: 'ruby:2.2')
end
end
describe '#services_value' do
it 'returns array of services' do
expect(global.services_value).to eq ['postgres:9.1', 'mysql:5.5']
expect(global.services_value).to eq [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }]
end
end
@ -150,8 +150,8 @@ describe Gitlab::Ci::Config::Entry::Global do
script: %w[rspec ls],
before_script: %w(ls pwd),
commands: "ls\npwd\nrspec\nls",
image: 'ruby:2.2',
services: ['postgres:9.1', 'mysql:5.5'],
image: { name: 'ruby:2.2' },
services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }],
stage: 'test',
cache: { key: 'k', untracked: true, paths: ['public/'] },
variables: { 'VAR' => 'value' },
@ -161,8 +161,8 @@ describe Gitlab::Ci::Config::Entry::Global do
before_script: [],
script: %w[spinach],
commands: 'spinach',
image: 'ruby:2.2',
services: ['postgres:9.1', 'mysql:5.5'],
image: { name: 'ruby:2.2' },
services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }],
stage: 'test',
cache: { key: 'k', untracked: true, paths: ['public/'] },
variables: {},

View file

@ -3,43 +3,104 @@ require 'spec_helper'
describe Gitlab::Ci::Config::Entry::Image do
let(:entry) { described_class.new(config) }
describe 'validation' do
context 'when entry config value is correct' do
let(:config) { 'ruby:2.2' }
context 'when configuration is a string' do
let(:config) { 'ruby:2.2' }
describe '#value' do
it 'returns image string' do
expect(entry.value).to eq 'ruby:2.2'
end
end
describe '#errors' do
it 'does not append errors' do
expect(entry.errors).to be_empty
end
end
describe '#valid?' do
it 'is valid' do
expect(entry).to be_valid
end
describe '#value' do
it 'returns image hash' do
expect(entry.value).to eq({ name: 'ruby:2.2' })
end
end
context 'when entry value is not correct' do
let(:config) { ['ruby:2.2'] }
describe '#errors' do
it 'saves errors' do
expect(entry.errors)
.to include 'image config should be a string'
end
describe '#errors' do
it 'does not append errors' do
expect(entry.errors).to be_empty
end
end
describe '#valid?' do
it 'is not valid' do
expect(entry).not_to be_valid
end
describe '#valid?' do
it 'is valid' do
expect(entry).to be_valid
end
end
describe '#image' do
it "returns image's name" do
expect(entry.name).to eq 'ruby:2.2'
end
end
describe '#entrypoint' do
it "returns image's entrypoint" do
expect(entry.entrypoint).to be_nil
end
end
end
context 'when configuration is a hash' do
let(:config) { { name: 'ruby:2.2', entrypoint: '/bin/sh' } }
describe '#value' do
it 'returns image hash' do
expect(entry.value).to eq(config)
end
end
describe '#errors' do
it 'does not append errors' do
expect(entry.errors).to be_empty
end
end
describe '#valid?' do
it 'is valid' do
expect(entry).to be_valid
end
end
describe '#image' do
it "returns image's name" do
expect(entry.name).to eq 'ruby:2.2'
end
end
describe '#entrypoint' do
it "returns image's entrypoint" do
expect(entry.entrypoint).to eq '/bin/sh'
end
end
end
context 'when entry value is not correct' do
let(:config) { ['ruby:2.2'] }
describe '#errors' do
it 'saves errors' do
expect(entry.errors)
.to include 'image config should be a hash or a string'
end
end
describe '#valid?' do
it 'is not valid' do
expect(entry).not_to be_valid
end
end
end
context 'when unexpected key is specified' do
let(:config) { { name: 'ruby:2.2', non_existing: 'test' } }
describe '#errors' do
it 'saves errors' do
expect(entry.errors)
.to include 'image config contains unknown keys: non_existing'
end
end
describe '#valid?' do
it 'is not valid' do
expect(entry).not_to be_valid
end
end
end

View file

@ -104,7 +104,7 @@ describe Gitlab::Ci::Config::Entry::Job do
end
it 'overrides global config' do
expect(entry[:image].value).to eq 'some_image'
expect(entry[:image].value).to eq(name: 'some_image')
expect(entry[:cache].value).to eq(key: 'test')
end
end

View file

@ -0,0 +1,117 @@
require 'spec_helper'
describe Gitlab::Ci::Config::Entry::Service do
let(:entry) { described_class.new(config) }
before { entry.compose! }
context 'when configuration is a string' do
let(:config) { 'postgresql:9.5' }
describe '#valid?' do
it 'is valid' do
expect(entry).to be_valid
end
end
describe '#value' do
it 'returns valid hash' do
expect(entry.value).to include(name: 'postgresql:9.5')
end
end
describe '#image' do
it "returns service's image name" do
expect(entry.name).to eq 'postgresql:9.5'
end
end
describe '#alias' do
it "returns service's alias" do
expect(entry.alias).to be_nil
end
end
describe '#command' do
it "returns service's command" do
expect(entry.command).to be_nil
end
end
end
context 'when configuration is a hash' do
let(:config) do
{ name: 'postgresql:9.5', alias: 'db', command: 'cmd', entrypoint: '/bin/sh' }
end
describe '#valid?' do
it 'is valid' do
expect(entry).to be_valid
end
end
describe '#value' do
it 'returns valid hash' do
expect(entry.value).to eq config
end
end
describe '#image' do
it "returns service's image name" do
expect(entry.name).to eq 'postgresql:9.5'
end
end
describe '#alias' do
it "returns service's alias" do
expect(entry.alias).to eq 'db'
end
end
describe '#command' do
it "returns service's command" do
expect(entry.command).to eq 'cmd'
end
end
describe '#entrypoint' do
it "returns service's entrypoint" do
expect(entry.entrypoint).to eq '/bin/sh'
end
end
end
context 'when entry value is not correct' do
let(:config) { ['postgresql:9.5'] }
describe '#errors' do
it 'saves errors' do
expect(entry.errors)
.to include 'service config should be a hash or a string'
end
end
describe '#valid?' do
it 'is not valid' do
expect(entry).not_to be_valid
end
end
end
context 'when unexpected key is specified' do
let(:config) { { name: 'postgresql:9.5', non_existing: 'test' } }
describe '#errors' do
it 'saves errors' do
expect(entry.errors)
.to include 'service config contains unknown keys: non_existing'
end
end
describe '#valid?' do
it 'is not valid' do
expect(entry).not_to be_valid
end
end
end
end

View file

@ -3,37 +3,30 @@ require 'spec_helper'
describe Gitlab::Ci::Config::Entry::Services do
let(:entry) { described_class.new(config) }
describe 'validations' do
context 'when entry config value is correct' do
let(:config) { ['postgres:9.1', 'mysql:5.5'] }
before { entry.compose! }
describe '#value' do
it 'returns array of services as is' do
expect(entry.value).to eq config
end
end
context 'when configuration is valid' do
let(:config) { ['postgresql:9.5', { name: 'postgresql:9.1', alias: 'postgres_old' }] }
describe '#valid?' do
it 'is valid' do
expect(entry).to be_valid
end
describe '#valid?' do
it 'is valid' do
expect(entry).to be_valid
end
end
context 'when entry value is not correct' do
let(:config) { 'ls' }
describe '#errors' do
it 'saves errors' do
expect(entry.errors)
.to include 'services config should be an array of strings'
end
describe '#value' do
it 'returns valid array' do
expect(entry.value).to eq([{ name: 'postgresql:9.5' }, { name: 'postgresql:9.1', alias: 'postgres_old' }])
end
end
end
describe '#valid?' do
it 'is not valid' do
expect(entry).not_to be_valid
end
context 'when configuration is invalid' do
let(:config) { 'postgresql:9.5' }
describe '#valid?' do
it 'is invalid' do
expect(entry).not_to be_valid
end
end
end

View file

@ -356,8 +356,11 @@ describe API::Runner do
expect(json_response['token']).to eq(job.token)
expect(json_response['job_info']).to eq(expected_job_info)
expect(json_response['git_info']).to eq(expected_git_info)
expect(json_response['image']).to eq({ 'name' => 'ruby:2.1' })
expect(json_response['services']).to eq([{ 'name' => 'postgres' }])
expect(json_response['image']).to eq({ 'name' => 'ruby:2.1', 'entrypoint' => '/bin/sh' })
expect(json_response['services']).to eq([{ 'name' => 'postgres', 'entrypoint' => nil,
'alias' => nil, 'command' => nil },
{ 'name' => 'docker:dind', 'entrypoint' => '/bin/sh',
'alias' => 'docker', 'command' => 'sleep 30' }])
expect(json_response['steps']).to eq(expected_steps)
expect(json_response['artifacts']).to eq(expected_artifacts)
expect(json_response['cache']).to eq(expected_cache)

View file

@ -137,6 +137,18 @@ describe Ci::API::Builds do
end
end
end
context 'when docker configuration options are used' do
let!(:build) { create(:ci_build, :extended_options, pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0) }
it 'starts a build' do
register_builds info: { platform: :darwin }
expect(response).to have_http_status(201)
expect(json_response['options']['image']).to eq('ruby:2.1')
expect(json_response['options']['services']).to eq(['postgres', 'docker:dind'])
end
end
end
context 'when builds are finished' do