Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2019-10-16 21:07:22 +00:00
parent 8e45d25f7d
commit e924e9e7cb
26 changed files with 477 additions and 30 deletions

17
app/models/aws/role.rb Normal file
View File

@ -0,0 +1,17 @@
# frozen_string_literal: true
module Aws
class Role < ApplicationRecord
self.table_name = 'aws_roles'
belongs_to :user, inverse_of: :aws_role
validates :role_external_id, uniqueness: true, length: { in: 1..64 }
validates :role_arn,
length: 1..2048,
format: {
with: Gitlab::Regex.aws_arn_regex,
message: Gitlab::Regex.aws_arn_regex_message
}
end
end

View File

@ -35,6 +35,7 @@ module Clusters
# we force autosave to happen when we save `Cluster` model
has_one :provider_gcp, class_name: 'Clusters::Providers::Gcp', autosave: true
has_one :provider_aws, class_name: 'Clusters::Providers::Aws', autosave: true
has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', inverse_of: :cluster, autosave: true
@ -96,14 +97,20 @@ module Clusters
enum provider_type: {
user: 0,
gcp: 1
gcp: 1,
aws: 2
}
scope :enabled, -> { where(enabled: true) }
scope :disabled, -> { where(enabled: false) }
scope :user_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:user]) }
scope :gcp_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:gcp]) }
scope :user_provided, -> { where(provider_type: :user) }
scope :gcp_provided, -> { where(provider_type: :gcp) }
scope :aws_provided, -> { where(provider_type: :aws) }
scope :gcp_installed, -> { gcp_provided.joins(:provider_gcp).merge(Clusters::Providers::Gcp.with_status(:created)) }
scope :aws_installed, -> { aws_provided.joins(:provider_aws).merge(Clusters::Providers::Aws.with_status(:created)) }
scope :managed, -> { where(managed: true) }
scope :default_environment, -> { where(environment_scope: DEFAULT_ENVIRONMENT) }
@ -140,7 +147,11 @@ module Clusters
end
def provider
return provider_gcp if gcp?
if gcp?
provider_gcp
elsif aws?
provider_aws
end
end
def platform

View File

@ -42,6 +42,10 @@ module Clusters
def on_creation?
scheduled? || creating?
end
def assign_operation_id(_)
# Implemented by individual providers if operation ID is supported.
end
end
end
end

View File

@ -0,0 +1,47 @@
# frozen_string_literal: true
module Clusters
module Providers
class Aws < ApplicationRecord
include Clusters::Concerns::ProviderStatus
self.table_name = 'cluster_providers_aws'
belongs_to :cluster, inverse_of: :provider_aws, class_name: 'Clusters::Cluster'
belongs_to :created_by_user, class_name: 'User'
default_value_for :region, 'us-east-1'
default_value_for :num_nodes, 3
default_value_for :instance_type, 'm5.large'
attr_encrypted :secret_access_key,
mode: :per_attribute_iv,
key: Settings.attr_encrypted_db_key_base_truncated,
algorithm: 'aes-256-gcm'
validates :role_arn,
length: 1..2048,
format: {
with: Gitlab::Regex.aws_arn_regex,
message: Gitlab::Regex.aws_arn_regex_message
}
validates :num_nodes,
numericality: {
only_integer: true,
greater_than: 0
}
validates :key_name, :region, :instance_type, :security_group_id, length: { in: 1..255 }
validates :subnet_ids, presence: true
def nullify_credentials
assign_attributes(
access_key_id: nil,
secret_access_key: nil,
session_token: nil
)
end
end
end
end

View File

@ -122,9 +122,13 @@ class JiraService < IssueTrackerService
end
alias_method :original_url, :url
def url
original_url&.chomp('/')
original_url&.delete_suffix('/')
end
alias_method :original_api_url, :api_url
def api_url
original_api_url&.delete_suffix('/')
end
def execute(push)

View File

@ -99,6 +99,7 @@ class User < ApplicationRecord
has_many :u2f_registrations, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
has_many :chat_names, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
has_one :user_synced_attributes_metadata, autosave: true
has_one :aws_role, class_name: 'Aws::Role'
# Groups
has_many :members

View File

@ -0,0 +1,5 @@
---
title: 'JIRA Integration API URL works having a trailing slash'
merge_request: 18526
author:
type: fixed

View File

@ -0,0 +1,5 @@
---
title: Add database tables to store AWS roles and cluster providers
merge_request: 17057
author:
type: added

View File

@ -0,0 +1,33 @@
# frozen_string_literal: true
class CreateClusterProvidersAws < ActiveRecord::Migration[5.2]
DOWNTIME = false
def change
create_table :cluster_providers_aws do |t|
t.references :cluster, null: false, type: :bigint, index: { unique: true }, foreign_key: { on_delete: :cascade }
t.references :created_by_user, type: :integer, foreign_key: { on_delete: :nullify, to_table: :users }
t.integer :num_nodes, null: false
t.integer :status, null: false
t.timestamps_with_timezone null: false
t.string :key_name, null: false, limit: 255
t.string :role_arn, null: false, limit: 2048
t.string :region, null: false, limit: 255
t.string :vpc_id, null: false, limit: 255
t.string :subnet_ids, null: false, array: true, default: [], limit: 255
t.string :security_group_id, null: false, limit: 255
t.string :instance_type, null: false, limit: 255
t.string :access_key_id, limit: 255
t.string :encrypted_secret_access_key_iv, limit: 255
t.text :encrypted_secret_access_key
t.text :session_token
t.text :status_reason
t.index [:cluster_id, :status]
end
end
end

View File

@ -0,0 +1,21 @@
# frozen_string_literal: true
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class CreateAwsRoles < ActiveRecord::Migration[5.2]
DOWNTIME = false
def change
create_table :aws_roles, id: false do |t|
t.references :user, primary_key: true, default: nil, type: :integer, index: { unique: true }, foreign_key: { on_delete: :cascade }
t.timestamps_with_timezone null: false
t.string :role_arn, null: false, limit: 2048
t.string :role_external_id, null: false, limit: 64
t.index :role_external_id, unique: true
end
end
end

View File

@ -467,6 +467,15 @@ ActiveRecord::Schema.define(version: 2019_10_16_072826) do
t.index ["user_id", "name"], name: "index_award_emoji_on_user_id_and_name"
end
create_table "aws_roles", primary_key: "user_id", id: :integer, default: nil, force: :cascade do |t|
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.string "role_arn", limit: 2048, null: false
t.string "role_external_id", limit: 64, null: false
t.index ["role_external_id"], name: "index_aws_roles_on_role_external_id", unique: true
t.index ["user_id"], name: "index_aws_roles_on_user_id", unique: true
end
create_table "badges", id: :serial, force: :cascade do |t|
t.string "link_url", null: false
t.string "image_url", null: false
@ -968,6 +977,30 @@ ActiveRecord::Schema.define(version: 2019_10_16_072826) do
t.index ["project_id"], name: "index_cluster_projects_on_project_id"
end
create_table "cluster_providers_aws", force: :cascade do |t|
t.bigint "cluster_id", null: false
t.integer "created_by_user_id"
t.integer "num_nodes", null: false
t.integer "status", null: false
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.string "key_name", limit: 255, null: false
t.string "role_arn", limit: 2048, null: false
t.string "region", limit: 255, null: false
t.string "vpc_id", limit: 255, null: false
t.string "subnet_ids", limit: 255, default: [], null: false, array: true
t.string "security_group_id", limit: 255, null: false
t.string "instance_type", limit: 255, null: false
t.string "access_key_id", limit: 255
t.string "encrypted_secret_access_key_iv", limit: 255
t.text "encrypted_secret_access_key"
t.text "session_token"
t.text "status_reason"
t.index ["cluster_id", "status"], name: "index_cluster_providers_aws_on_cluster_id_and_status"
t.index ["cluster_id"], name: "index_cluster_providers_aws_on_cluster_id", unique: true
t.index ["created_by_user_id"], name: "index_cluster_providers_aws_on_created_by_user_id"
end
create_table "cluster_providers_gcp", id: :serial, force: :cascade do |t|
t.integer "cluster_id", null: false
t.integer "status"
@ -3933,6 +3966,7 @@ ActiveRecord::Schema.define(version: 2019_10_16_072826) do
add_foreign_key "approval_project_rules_users", "users", on_delete: :cascade
add_foreign_key "approvals", "merge_requests", name: "fk_310d714958", on_delete: :cascade
add_foreign_key "approver_groups", "namespaces", column: "group_id", on_delete: :cascade
add_foreign_key "aws_roles", "users", on_delete: :cascade
add_foreign_key "badges", "namespaces", column: "group_id", on_delete: :cascade
add_foreign_key "badges", "projects", on_delete: :cascade
add_foreign_key "board_assignees", "boards", on_delete: :cascade
@ -3996,6 +4030,8 @@ ActiveRecord::Schema.define(version: 2019_10_16_072826) do
add_foreign_key "cluster_platforms_kubernetes", "clusters", on_delete: :cascade
add_foreign_key "cluster_projects", "clusters", on_delete: :cascade
add_foreign_key "cluster_projects", "projects", on_delete: :cascade
add_foreign_key "cluster_providers_aws", "clusters", on_delete: :cascade
add_foreign_key "cluster_providers_aws", "users", column: "created_by_user_id", on_delete: :nullify
add_foreign_key "cluster_providers_gcp", "clusters", on_delete: :cascade
add_foreign_key "clusters", "projects", column: "management_project_id", name: "fk_f05c5e5a42", on_delete: :nullify
add_foreign_key "clusters", "users", on_delete: :nullify

View File

@ -50,7 +50,7 @@ module Gitlab
# @param [Teammate] person
# @return [Boolean]
def valid_person?(person)
!mr_author?(person) && !person.out_of_office?
!mr_author?(person) && person.available?
end
# @param [Teammate] person

View File

@ -43,12 +43,22 @@ module Gitlab
nil # better no status than a crashing Danger
end
# @return [Boolean]
def available?
!out_of_office? && has_capacity?
end
private
# @return [Boolean]
def out_of_office?
status&.dig("message")&.match?(/OOO/i) || false
end
private
# @return [Boolean]
def has_capacity?
status&.dig("emoji") != 'red_circle'
end
def has_capability?(project, category, kind, labels)
case category

View File

@ -119,6 +119,15 @@ module Gitlab
def breakline_regex
@breakline_regex ||= /\r\n|\r|\n/
end
# https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html
def aws_arn_regex
/\Aarn:\S+\z/
end
def aws_arn_regex_message
"must be a valid Amazon Resource Name"
end
end
end

View File

@ -19,6 +19,7 @@ describe 'Database schema' do
approver_groups: %w[target_id],
audit_events: %w[author_id entity_id],
award_emoji: %w[awardable_id user_id],
aws_roles: %w[role_external_id],
boards: %w[milestone_id],
chat_names: %w[chat_id service_id team_id user_id],
chat_teams: %w[team_id],
@ -26,6 +27,7 @@ describe 'Database schema' do
ci_pipelines: %w[user_id],
ci_runner_projects: %w[runner_id],
ci_trigger_requests: %w[commit_id],
cluster_providers_aws: %w[security_group_id vpc_id access_key_id],
cluster_providers_gcp: %w[gcp_project_id operation_id],
deploy_keys_projects: %w[deploy_key_id],
deployments: %w[deployable_id environment_id user_id],

View File

@ -0,0 +1,10 @@
# frozen_string_literal: true
FactoryBot.define do
factory :aws_role, class: Aws::Role do
user
role_arn { 'arn:aws:iam::123456789012:role/role-name' }
sequence(:role_external_id) { |n| "external-id-#{n}" }
end
end

View File

@ -53,6 +53,14 @@ FactoryBot.define do
platform_kubernetes factory: [:cluster_platform_kubernetes, :configured]
end
trait :provided_by_aws do
provider_type { :aws }
platform_type { :kubernetes }
provider_aws factory: [:cluster_provider_aws, :created]
platform_kubernetes factory: [:cluster_platform_kubernetes, :configured]
end
trait :providing_by_gcp do
provider_type { :gcp }
provider_gcp factory: [:cluster_provider_gcp, :creating]

View File

@ -0,0 +1,38 @@
# frozen_string_literal: true
FactoryBot.define do
factory :cluster_provider_aws, class: Clusters::Providers::Aws do
cluster
created_by_user factory: :user
role_arn { 'arn:aws:iam::123456789012:role/role-name' }
vpc_id { 'vpc-00000000000000000' }
subnet_ids { %w(subnet-00000000000000000 subnet-11111111111111111) }
security_group_id { 'sg-00000000000000000' }
key_name { 'user' }
trait :scheduled do
access_key_id { 'access_key_id' }
secret_access_key { 'secret_access_key' }
session_token { 'session_token' }
end
trait :creating do
after(:build) do |provider|
provider.make_creating
end
end
trait :created do
after(:build) do |provider|
provider.make_created
end
end
trait :errored do
after(:build) do |provider|
provider.make_errored('An error occurred')
end
end
end
end

View File

@ -104,11 +104,13 @@ describe Gitlab::Danger::Roulette do
let(:person2) { Gitlab::Danger::Teammate.new('username' => 'godfat') }
let(:author) { Gitlab::Danger::Teammate.new('username' => 'filipa') }
let(:ooo) { Gitlab::Danger::Teammate.new('username' => 'jacopo-beschi') }
let(:no_capacity) { Gitlab::Danger::Teammate.new('username' => 'uncharged') }
before do
stub_person_message(person1, 'making GitLab magic')
stub_person_message(person2, 'making GitLab magic')
stub_person_message(ooo, 'OOO till 15th')
stub_person_status(person1, message: 'making GitLab magic')
stub_person_status(person2, message: 'making GitLab magic')
stub_person_status(ooo, message: 'OOO till 15th')
stub_person_status(no_capacity, message: 'At capacity for the next few days', emoji: 'red_circle')
# we don't stub Filipa, as she is the author and
# we should not fire request checking for her
@ -131,10 +133,14 @@ describe Gitlab::Danger::Roulette do
expect(subject.spin_for_person([author], random: Random.new)).to be_nil
end
it 'excludes person with no capacity' do
expect(subject.spin_for_person([no_capacity], random: Random.new)).to be_nil
end
private
def stub_person_message(person, message)
body = { message: message }.to_json
def stub_person_status(person, message: 'dummy message', emoji: 'unicorn')
body = { message: message, emoji: emoji }.to_json
WebMock
.stub_request(:get, "https://gitlab.com/api/v4/users/#{person.username}/status")

View File

@ -135,17 +135,17 @@ describe Gitlab::Danger::Teammate do
end
end
describe '#out_of_office?' do
describe '#available?' do
using RSpec::Parameterized::TableSyntax
let(:capabilities) { ['dry head'] }
where(:status, :result) do
nil | false
{} | false
{ message: 'dear reader' } | false
{ message: 'OOO: massage' } | true
{ message: 'love it SOOO much' } | true
{} | true
{ message: 'dear reader' } | true
{ message: 'OOO: massage' } | false
{ message: 'love it SOOO much' } | false
{ emoji: 'red_circle' } | false
end
with_them do
@ -154,7 +154,15 @@ describe Gitlab::Danger::Teammate do
.and_return(status&.stringify_keys)
end
it { expect(subject.out_of_office?).to be result }
it { expect(subject.available?).to be result }
end
it 'returns true if request fails' do
expect(Gitlab::Danger::RequestHelper).to receive(:http_get_json)
.exactly(2).times
.and_raise(Gitlab::Danger::RequestHelper::HTTPError.new)
expect(subject.available?).to be true
end
end
end

View File

@ -64,4 +64,15 @@ describe Gitlab::Regex do
it { is_expected.not_to match('.my/image') }
it { is_expected.not_to match('my/image.') }
end
describe '.aws_account_id_regex' do
subject { described_class.aws_arn_regex }
it { is_expected.to match('arn:aws:iam::123456789012:role/role-name') }
it { is_expected.to match('arn:aws:s3:::bucket/key') }
it { is_expected.to match('arn:aws:ec2:us-east-1:123456789012:volume/vol-1') }
it { is_expected.to match('arn:aws:rds:us-east-1:123456789012:pg:prod') }
it { is_expected.not_to match('123456789012') }
it { is_expected.not_to match('role/role-name') }
end
end

View File

@ -0,0 +1,34 @@
# frozen_string_literal: true
require 'spec_helper'
describe Aws::Role do
it { is_expected.to belong_to(:user) }
it { is_expected.to validate_length_of(:role_external_id).is_at_least(1).is_at_most(64) }
describe 'custom validations' do
subject { role.valid? }
context ':role_arn' do
let(:role) { build(:aws_role, role_arn: role_arn) }
context 'length is zero' do
let(:role_arn) { '' }
it { is_expected.to be_falsey }
end
context 'length is longer than 2048' do
let(:role_arn) { '1' * 2049 }
it { is_expected.to be_falsey }
end
context 'ARN is valid' do
let(:role_arn) { 'arn:aws:iam::123456789012:role/test-role' }
it { is_expected.to be_truthy }
end
end
end
end

View File

@ -17,6 +17,7 @@ describe Clusters::Cluster, :use_clean_rails_memory_store_caching do
it { is_expected.to have_many(:cluster_groups) }
it { is_expected.to have_many(:groups) }
it { is_expected.to have_one(:provider_gcp) }
it { is_expected.to have_one(:provider_aws) }
it { is_expected.to have_one(:platform_kubernetes) }
it { is_expected.to have_one(:application_helm) }
it { is_expected.to have_one(:application_ingress) }
@ -108,6 +109,31 @@ describe Clusters::Cluster, :use_clean_rails_memory_store_caching do
it { is_expected.to contain_exactly(cluster) }
end
describe '.aws_provided' do
subject { described_class.aws_provided }
let!(:cluster) { create(:cluster, :provided_by_aws) }
before do
create(:cluster, :provided_by_user)
end
it { is_expected.to contain_exactly(cluster) }
end
describe '.aws_installed' do
subject { described_class.aws_installed }
let!(:cluster) { create(:cluster, :provided_by_aws) }
before do
errored_cluster = create(:cluster, :provided_by_aws)
errored_cluster.provider.make_errored!("Error message")
end
it { is_expected.to contain_exactly(cluster) }
end
describe '.managed' do
subject do
described_class.managed
@ -398,7 +424,14 @@ describe Clusters::Cluster, :use_clean_rails_memory_store_caching do
it 'returns a provider' do
is_expected.to eq(cluster.provider_gcp)
expect(subject.class.name.deconstantize).to eq(Clusters::Providers.to_s)
end
end
context 'when provider is aws' do
let(:cluster) { create(:cluster, :provided_by_aws) }
it 'returns a provider' do
is_expected.to eq(cluster.provider_aws)
end
end

View File

@ -0,0 +1,76 @@
# frozen_string_literal: true
require 'spec_helper'
describe Clusters::Providers::Aws do
it { is_expected.to belong_to(:cluster) }
it { is_expected.to belong_to(:created_by_user) }
it { is_expected.to validate_length_of(:key_name).is_at_least(1).is_at_most(255) }
it { is_expected.to validate_length_of(:region).is_at_least(1).is_at_most(255) }
it { is_expected.to validate_length_of(:instance_type).is_at_least(1).is_at_most(255) }
it { is_expected.to validate_length_of(:security_group_id).is_at_least(1).is_at_most(255) }
it { is_expected.to validate_presence_of(:subnet_ids) }
include_examples 'provider status', :cluster_provider_aws
describe 'default_value_for' do
let(:provider) { build(:cluster_provider_aws) }
it "sets default values" do
expect(provider.region).to eq('us-east-1')
expect(provider.num_nodes).to eq(3)
expect(provider.instance_type).to eq('m5.large')
end
end
describe 'custom validations' do
subject { provider.valid? }
context ':num_nodes' do
let(:provider) { build(:cluster_provider_aws, num_nodes: num_nodes) }
context 'contains non-digit characters' do
let(:num_nodes) { 'A3' }
it { is_expected.to be_falsey }
end
context 'is blank' do
let(:num_nodes) { nil }
it { is_expected.to be_falsey }
end
context 'is less than 1' do
let(:num_nodes) { 0 }
it { is_expected.to be_falsey }
end
context 'is a positive integer' do
let(:num_nodes) { 3 }
it { is_expected.to be_truthy }
end
end
end
describe '#nullify_credentials' do
let(:provider) { create(:cluster_provider_aws, :scheduled) }
subject { provider.nullify_credentials }
before do
expect(provider.access_key_id).to be_present
expect(provider.secret_access_key).to be_present
end
it 'removes access_key_id and secret_access_key' do
subject
expect(provider.access_key_id).to be_nil
expect(provider.secret_access_key).to be_nil
end
end
end

View File

@ -15,20 +15,22 @@ describe JiraService do
let(:transition_id) { 'test27' }
describe '#options' do
let(:service) do
described_class.create(
let(:options) do
{
project: create(:project),
active: true,
username: 'username ',
username: 'username',
password: 'test',
jira_issue_transition_id: 24,
url: 'http://jira.test.com/path/'
)
}
end
let(:service) { described_class.create(options) }
it 'sets the URL properly' do
# jira-ruby gem parses the URI and handles trailing slashes
# fine: https://github.com/sumoheavy/jira-ruby/blob/v1.4.1/lib/jira/http_client.rb#L59
# jira-ruby gem parses the URI and handles trailing slashes fine:
# https://github.com/sumoheavy/jira-ruby/blob/v1.7.0/lib/jira/http_client.rb#L62
expect(service.options[:site]).to eq('http://jira.test.com/')
end
@ -36,14 +38,30 @@ describe JiraService do
expect(service.options[:context_path]).to eq('/path')
end
it 'leaves out trailing whitespaces in username' do
expect(service.options[:username]).to eq('username')
context 'username with trailing whitespaces' do
before do
options.merge!(username: 'username ')
end
it 'leaves out trailing whitespaces in username' do
expect(service.options[:username]).to eq('username')
end
end
it 'provides additional cookies to allow basic auth with oracle webgate' do
expect(service.options[:use_cookies]).to eq(true)
expect(service.options[:additional_cookies]).to eq(['OBBasicAuth=fromDialog'])
end
context 'using api URL' do
before do
options.merge!(api_url: 'http://jira.test.com/api_path/')
end
it 'leaves out trailing slashes in context' do
expect(service.options[:context_path]).to eq('/api_path')
end
end
end
describe 'Associations' do

View File

@ -17,7 +17,7 @@ shared_examples 'provider status' do |factory|
let(:provider) { build(factory) }
let(:operation_id) { 'operation-xxx' }
it 'calls #operation_id on the provider' do
it 'calls #assign_operation_id on the provider' do
expect(provider).to receive(:assign_operation_id).with(operation_id).and_call_original
provider.make_creating(operation_id)