From e43b2e81dab3cade773d479f2ae56478e3113207 Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Thu, 27 Oct 2016 17:39:32 -0200 Subject: [PATCH 01/97] Added MR Road map --- lib/container_registry/ROADMAP.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 lib/container_registry/ROADMAP.md diff --git a/lib/container_registry/ROADMAP.md b/lib/container_registry/ROADMAP.md new file mode 100644 index 00000000000..e0a20776404 --- /dev/null +++ b/lib/container_registry/ROADMAP.md @@ -0,0 +1,7 @@ +## Road map + +### Initial thoughts + +- Determine if image names will be persisted or fetched from API +- If persisted, how to update the stored names upon modification +- If fetched, how to fetch only images of a given project From dcd4beb8eb7bb7d0c2f720ef85c3da9f97a3dfe6 Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Wed, 2 Nov 2016 00:33:35 -0200 Subject: [PATCH 02/97] Multi-level container image names backend implementation - Adds Registry events API endpoint - Adds container_images_repository and container_images models - Changes JWT authentication to allow multi-level scopes - Adds services for container image maintenance --- app/models/container_image.rb | 58 +++++++++++++++++++ app/models/container_images_repository.rb | 26 +++++++++ app/models/project.rb | 1 + ...ntainer_registry_authentication_service.rb | 9 ++- .../container_images/create_service.rb | 16 +++++ .../container_images/destroy_service.rb | 11 ++++ .../container_images/push_service.rb | 26 +++++++++ .../create_service.rb | 7 +++ ...3736_create_container_images_repository.rb | 31 ++++++++++ .../20161031013926_create_container_image.rb | 32 ++++++++++ lib/api/api.rb | 1 + lib/api/registry_events.rb | 52 +++++++++++++++++ lib/container_registry/client.rb | 4 ++ lib/container_registry/tag.rb | 8 +-- 14 files changed, 275 insertions(+), 7 deletions(-) create mode 100644 app/models/container_image.rb create mode 100644 app/models/container_images_repository.rb create mode 100644 app/services/container_images_repositories/container_images/create_service.rb create mode 100644 app/services/container_images_repositories/container_images/destroy_service.rb create mode 100644 app/services/container_images_repositories/container_images/push_service.rb create mode 100644 app/services/container_images_repositories/create_service.rb create mode 100644 db/migrate/20161029153736_create_container_images_repository.rb create mode 100644 db/migrate/20161031013926_create_container_image.rb create mode 100644 lib/api/registry_events.rb diff --git a/app/models/container_image.rb b/app/models/container_image.rb new file mode 100644 index 00000000000..dcc4a7af629 --- /dev/null +++ b/app/models/container_image.rb @@ -0,0 +1,58 @@ +class ContainerImage < ActiveRecord::Base + belongs_to :container_images_repository + + delegate :registry, :registry_path_with_namespace, :client, to: :container_images_repository + + validates :manifest, presence: true + + before_validation :update_token, on: :create + def update_token + paths = container_images_repository.allowed_paths << name_with_namespace + token = Auth::ContainerRegistryAuthenticationService.full_access_token(paths) + client.update_token(token) + end + + def path + [registry.path, name_with_namespace].compact.join('/') + end + + def name_with_namespace + [registry_path_with_namespace, name].compact.join('/') + end + + def tag(tag) + ContainerRegistry::Tag.new(self, tag) + end + + def manifest + @manifest ||= client.repository_tags(name_with_namespace) + end + + def tags + return @tags if defined?(@tags) + return [] unless manifest && manifest['tags'] + + @tags = manifest['tags'].map do |tag| + ContainerRegistry::Tag.new(self, tag) + end + end + + def blob(config) + ContainerRegistry::Blob.new(self, config) + end + + def delete_tags + return unless tags + + tags.all?(&:delete) + end + + def self.split_namespace(full_path) + image_name = full_path.split('/').last + namespace = full_path.gsub(/(.*)(#{Regexp.escape('/' + image_name)})/, '\1') + if namespace.count('/') < 1 + namespace, image_name = full_path, "" + end + return namespace, image_name + end +end diff --git a/app/models/container_images_repository.rb b/app/models/container_images_repository.rb new file mode 100644 index 00000000000..99e94d2a6d0 --- /dev/null +++ b/app/models/container_images_repository.rb @@ -0,0 +1,26 @@ +class ContainerImagesRepository < ActiveRecord::Base + + belongs_to :project + + has_many :container_images, dependent: :destroy + + delegate :client, to: :registry + + def registry_path_with_namespace + project.path_with_namespace.downcase + end + + def allowed_paths + @allowed_paths ||= [registry_path_with_namespace] + + container_images.map { |i| i.name_with_namespace } + end + + def registry + @registry ||= begin + token = Auth::ContainerRegistryAuthenticationService.full_access_token(allowed_paths) + url = Gitlab.config.registry.api_url + host_port = Gitlab.config.registry.host_port + ContainerRegistry::Registry.new(url, token: token, path: host_port) + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 411299eef63..703e24eb79a 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -157,6 +157,7 @@ class Project < ActiveRecord::Base has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" has_one :project_feature, dependent: :destroy has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete + has_one :container_images_repository, dependent: :destroy has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 5cb7a86a5ee..6b83b38fa4d 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -16,7 +16,7 @@ module Auth { token: authorized_token(scope).encoded } end - def self.full_access_token(*names) + def self.full_access_token(names) registry = Gitlab.config.registry token = JSONWebToken::RSAToken.new(registry.key) token.issuer = registry.issuer @@ -61,7 +61,12 @@ module Auth end def process_repository_access(type, name, actions) - requested_project = Project.find_by_full_path(name) + # Strips image name due to lack of + # per image authentication. + # Removes only last occurence in light + # of future nested groups + namespace, _ = ContainerImage::split_namespace(name) + requested_project = Project.find_by_full_path(namespace) return unless requested_project actions = actions.select do |action| diff --git a/app/services/container_images_repositories/container_images/create_service.rb b/app/services/container_images_repositories/container_images/create_service.rb new file mode 100644 index 00000000000..0c2c69d5183 --- /dev/null +++ b/app/services/container_images_repositories/container_images/create_service.rb @@ -0,0 +1,16 @@ +module ContainerImagesRepositories + module ContainerImages + class CreateService < BaseService + def execute + @container_image = container_images_repository.container_images.create(params) + @container_image if @container_image.valid? + end + + private + + def container_images_repository + @container_images_repository ||= project.container_images_repository + end + end + end +end diff --git a/app/services/container_images_repositories/container_images/destroy_service.rb b/app/services/container_images_repositories/container_images/destroy_service.rb new file mode 100644 index 00000000000..91b8cfeea47 --- /dev/null +++ b/app/services/container_images_repositories/container_images/destroy_service.rb @@ -0,0 +1,11 @@ +module ContainerImagesRepositories + module ContainerImages + class DestroyService < BaseService + def execute(container_image) + return false unless container_image + + container_image.destroy + end + end + end +end diff --git a/app/services/container_images_repositories/container_images/push_service.rb b/app/services/container_images_repositories/container_images/push_service.rb new file mode 100644 index 00000000000..2731cf1d52e --- /dev/null +++ b/app/services/container_images_repositories/container_images/push_service.rb @@ -0,0 +1,26 @@ +module ContainerImagesRepositories + module ContainerImages + class PushService < BaseService + def execute(container_image_name, event) + find_or_create_container_image(container_image_name).valid? + end + + private + + def find_or_create_container_image(container_image_name) + options = {name: container_image_name} + container_images.find_by(options) || + ::ContainerImagesRepositories::ContainerImages::CreateService.new(project, + current_user, options).execute + end + + def container_images_repository + @container_images_repository ||= project.container_images_repository + end + + def container_images + @container_images ||= container_images_repository.container_images + end + end + end +end diff --git a/app/services/container_images_repositories/create_service.rb b/app/services/container_images_repositories/create_service.rb new file mode 100644 index 00000000000..7e9dd3abe5f --- /dev/null +++ b/app/services/container_images_repositories/create_service.rb @@ -0,0 +1,7 @@ +module ContainerImagesRepositories + class CreateService < BaseService + def execute + project.container_images_repository || ::ContainerImagesRepository.create(project: project) + end + end +end diff --git a/db/migrate/20161029153736_create_container_images_repository.rb b/db/migrate/20161029153736_create_container_images_repository.rb new file mode 100644 index 00000000000..d93180b1674 --- /dev/null +++ b/db/migrate/20161029153736_create_container_images_repository.rb @@ -0,0 +1,31 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreateContainerImagesRepository < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index" or "add_column_with_default" + # you must disable the use of transactions as these methods can not run in an + # existing transaction. When using "add_concurrent_index" make sure that this + # method is the _only_ method called in the migration, any other changes + # should go in a separate migration. This ensures that upon failure _only_ the + # index creation fails and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + create_table :container_images_repositories do |t| + t.integer :project_id, null: false + end + end +end diff --git a/db/migrate/20161031013926_create_container_image.rb b/db/migrate/20161031013926_create_container_image.rb new file mode 100644 index 00000000000..94feae280a6 --- /dev/null +++ b/db/migrate/20161031013926_create_container_image.rb @@ -0,0 +1,32 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreateContainerImage < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index" or "add_column_with_default" + # you must disable the use of transactions as these methods can not run in an + # existing transaction. When using "add_concurrent_index" make sure that this + # method is the _only_ method called in the migration, any other changes + # should go in a separate migration. This ensures that upon failure _only_ the + # index creation fails and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + create_table :container_images do |t| + t.integer :container_images_repository_id + t.string :name + end + end +end diff --git a/lib/api/api.rb b/lib/api/api.rb index a0282ff8deb..ed775f898d2 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -84,6 +84,7 @@ module API mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings + mount ::API::RegistryEvents mount ::API::Pipelines mount ::API::ProjectHooks mount ::API::Projects diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb new file mode 100644 index 00000000000..c0473051424 --- /dev/null +++ b/lib/api/registry_events.rb @@ -0,0 +1,52 @@ +module API + # RegistryEvents API + class RegistryEvents < Grape::API + # before { authenticate! } + + content_type :json, 'application/vnd.docker.distribution.events.v1+json' + + params do + requires :events, type: Array, desc: 'The ID of a project' do + requires :id, type: String, desc: 'The ID of the event' + requires :timestamp, type: String, desc: 'Timestamp of the event' + requires :action, type: String, desc: 'Action performed by event' + requires :target, type: Hash, desc: 'Target of the event' do + optional :mediaType, type: String, desc: 'Media type of the target' + optional :size, type: Integer, desc: 'Size in bytes of the target' + requires :digest, type: String, desc: 'Digest of the target' + requires :repository, type: String, desc: 'Repository of target' + optional :url, type: String, desc: 'Url of the target' + optional :tag, type: String, desc: 'Tag of the target' + end + requires :request, type: Hash, desc: 'Request of the event' do + requires :id, type: String, desc: 'The ID of the request' + optional :addr, type: String, desc: 'IP Address of the request client' + optional :host, type: String, desc: 'Hostname of the registry instance' + requires :method, type: String, desc: 'Request method' + requires :useragent, type: String, desc: 'UserAgent header of the request' + end + requires :actor, type: Hash, desc: 'Actor that initiated the event' do + optional :name, type: String, desc: 'Actor name' + end + requires :source, type: Hash, desc: 'Source of the event' do + optional :addr, type: String, desc: 'Hostname of source registry node' + optional :instanceID, type: String, desc: 'Source registry node instanceID' + end + end + end + resource :registry_events do + post do + params['events'].each do |event| + repository = event['target']['repository'] + + if event['action'] == 'push' and !!event['target']['tag'] + namespace, container_image_name = ContainerImage::split_namespace(repository) + ::ContainerImagesRepositories::ContainerImages::PushService.new( + Project::find_with_namespace(namespace), current_user + ).execute(container_image_name, event) + end + end + end + end + end +end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 2edddb84fc3..2cbb7bfb67d 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -15,6 +15,10 @@ module ContainerRegistry @options = options end + def update_token(token) + @options[:token] = token + end + def repository_tags(name) response_body faraday.get("/v2/#{name}/tags/list") end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 59040199920..68dd87c979d 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -22,9 +22,7 @@ module ContainerRegistry end def manifest - return @manifest if defined?(@manifest) - - @manifest = client.repository_manifest(repository.name, name) + @manifest ||= client.repository_manifest(repository.name_with_namespace, name) end def path @@ -40,7 +38,7 @@ module ContainerRegistry def digest return @digest if defined?(@digest) - @digest = client.repository_tag_digest(repository.name, name) + @digest = client.repository_tag_digest(repository.name_with_namespace, name) end def config_blob @@ -82,7 +80,7 @@ module ContainerRegistry def delete return unless digest - client.delete_repository_tag(repository.name, digest) + client.delete_repository_tag(repository.name_with_namespace, digest) end end end From eed0b85ad084ad4d13cc26907102063d9372fe75 Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Wed, 23 Nov 2016 14:50:30 -0200 Subject: [PATCH 03/97] First iteration of container_image view - Fixes project, container_image and tag deletion - Removed container_images_repository [ci skip] --- .../stylesheets/pages/container_registry.scss | 16 +++++++++ .../projects/container_registry_controller.rb | 28 +++++++++++---- app/models/container_image.rb | 20 +++++++---- app/models/container_images_repository.rb | 26 -------------- app/models/project.rb | 20 ++++++----- .../container_images/destroy_service.rb | 32 +++++++++++++++++ .../container_images/create_service.rb | 16 --------- .../container_images/destroy_service.rb | 11 ------ .../container_images/push_service.rb | 26 -------------- .../create_service.rb | 7 ---- app/services/projects/destroy_service.rb | 10 ------ .../container_registry/_image.html.haml | 35 +++++++++++++++++++ .../container_registry/_tag.html.haml | 2 +- .../container_registry/index.html.haml | 20 +++-------- ...3736_create_container_images_repository.rb | 31 ---------------- .../20161031013926_create_container_image.rb | 2 +- lib/api/registry_events.rb | 16 +++++++-- 17 files changed, 149 insertions(+), 169 deletions(-) create mode 100644 app/assets/stylesheets/pages/container_registry.scss delete mode 100644 app/models/container_images_repository.rb create mode 100644 app/services/container_images/destroy_service.rb delete mode 100644 app/services/container_images_repositories/container_images/create_service.rb delete mode 100644 app/services/container_images_repositories/container_images/destroy_service.rb delete mode 100644 app/services/container_images_repositories/container_images/push_service.rb delete mode 100644 app/services/container_images_repositories/create_service.rb create mode 100644 app/views/projects/container_registry/_image.html.haml delete mode 100644 db/migrate/20161029153736_create_container_images_repository.rb diff --git a/app/assets/stylesheets/pages/container_registry.scss b/app/assets/stylesheets/pages/container_registry.scss new file mode 100644 index 00000000000..7d68eae3c97 --- /dev/null +++ b/app/assets/stylesheets/pages/container_registry.scss @@ -0,0 +1,16 @@ +/** + * Container Registry + */ + +.container-image { + border-bottom: 1px solid #f0f0f0; +} + +.container-image-head { + padding: 0px 16px; + line-height: 4; +} + +.table.tags { + margin-bottom: 0px; +} diff --git a/app/controllers/projects/container_registry_controller.rb b/app/controllers/projects/container_registry_controller.rb index d1f46497207..54bcb5f504a 100644 --- a/app/controllers/projects/container_registry_controller.rb +++ b/app/controllers/projects/container_registry_controller.rb @@ -5,17 +5,22 @@ class Projects::ContainerRegistryController < Projects::ApplicationController layout 'project' def index - @tags = container_registry_repository.tags + @images = project.container_images end def destroy url = namespace_project_container_registry_index_path(project.namespace, project) - if tag.delete - redirect_to url + if tag + delete_tag(url) else - redirect_to url, alert: 'Failed to remove tag' + if image.destroy + redirect_to url + else + redirect_to url, alert: 'Failed to remove image' + end end + end private @@ -24,11 +29,20 @@ class Projects::ContainerRegistryController < Projects::ApplicationController render_404 unless Gitlab.config.registry.enabled end - def container_registry_repository - @container_registry_repository ||= project.container_registry_repository + def delete_tag(url) + if tag.delete + image.destroy if image.tags.empty? + redirect_to url + else + redirect_to url, alert: 'Failed to remove tag' + end + end + + def image + @image ||= project.container_images.find_by(id: params[:id]) end def tag - @tag ||= container_registry_repository.tag(params[:id]) + @tag ||= image.tag(params[:tag]) if params[:tag].present? end end diff --git a/app/models/container_image.rb b/app/models/container_image.rb index dcc4a7af629..7721c53a6fc 100644 --- a/app/models/container_image.rb +++ b/app/models/container_image.rb @@ -1,23 +1,28 @@ class ContainerImage < ActiveRecord::Base - belongs_to :container_images_repository + belongs_to :project - delegate :registry, :registry_path_with_namespace, :client, to: :container_images_repository + delegate :container_registry, :container_registry_allowed_paths, + :container_registry_path_with_namespace, to: :project + + delegate :client, to: :container_registry validates :manifest, presence: true + before_destroy :delete_tags + before_validation :update_token, on: :create def update_token - paths = container_images_repository.allowed_paths << name_with_namespace + paths = container_registry_allowed_paths << name_with_namespace token = Auth::ContainerRegistryAuthenticationService.full_access_token(paths) client.update_token(token) end def path - [registry.path, name_with_namespace].compact.join('/') + [container_registry.path, name_with_namespace].compact.join('/') end def name_with_namespace - [registry_path_with_namespace, name].compact.join('/') + [container_registry_path_with_namespace, name].compact.join('/') end def tag(tag) @@ -44,7 +49,10 @@ class ContainerImage < ActiveRecord::Base def delete_tags return unless tags - tags.all?(&:delete) + digests = tags.map {|tag| tag.digest }.to_set + digests.all? do |digest| + client.delete_repository_tag(name_with_namespace, digest) + end end def self.split_namespace(full_path) diff --git a/app/models/container_images_repository.rb b/app/models/container_images_repository.rb deleted file mode 100644 index 99e94d2a6d0..00000000000 --- a/app/models/container_images_repository.rb +++ /dev/null @@ -1,26 +0,0 @@ -class ContainerImagesRepository < ActiveRecord::Base - - belongs_to :project - - has_many :container_images, dependent: :destroy - - delegate :client, to: :registry - - def registry_path_with_namespace - project.path_with_namespace.downcase - end - - def allowed_paths - @allowed_paths ||= [registry_path_with_namespace] + - container_images.map { |i| i.name_with_namespace } - end - - def registry - @registry ||= begin - token = Auth::ContainerRegistryAuthenticationService.full_access_token(allowed_paths) - url = Gitlab.config.registry.api_url - host_port = Gitlab.config.registry.host_port - ContainerRegistry::Registry.new(url, token: token, path: host_port) - end - end -end diff --git a/app/models/project.rb b/app/models/project.rb index 703e24eb79a..afaf2095a4c 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -157,7 +157,7 @@ class Project < ActiveRecord::Base has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" has_one :project_feature, dependent: :destroy has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete - has_one :container_images_repository, dependent: :destroy + has_many :container_images, dependent: :destroy has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id @@ -405,15 +405,19 @@ class Project < ActiveRecord::Base path_with_namespace.downcase end - def container_registry_repository + def container_registry_allowed_paths + @container_registry_allowed_paths ||= [container_registry_path_with_namespace] + + container_images.map { |i| i.name_with_namespace } + end + + def container_registry return unless Gitlab.config.registry.enabled - @container_registry_repository ||= begin - token = Auth::ContainerRegistryAuthenticationService.full_access_token(container_registry_path_with_namespace) + @container_registry ||= begin + token = Auth::ContainerRegistryAuthenticationService.full_access_token(container_registry_allowed_paths) url = Gitlab.config.registry.api_url host_port = Gitlab.config.registry.host_port - registry = ContainerRegistry::Registry.new(url, token: token, path: host_port) - registry.repository(container_registry_path_with_namespace) + ContainerRegistry::Registry.new(url, token: token, path: host_port) end end @@ -424,9 +428,9 @@ class Project < ActiveRecord::Base end def has_container_registry_tags? - return unless container_registry_repository + return unless container_images - container_registry_repository.tags.any? + container_images.first.tags.any? end def commit(ref = 'HEAD') diff --git a/app/services/container_images/destroy_service.rb b/app/services/container_images/destroy_service.rb new file mode 100644 index 00000000000..bc5b53fd055 --- /dev/null +++ b/app/services/container_images/destroy_service.rb @@ -0,0 +1,32 @@ +module ContainerImages + class DestroyService < BaseService + + class DestroyError < StandardError; end + + def execute(container_image) + @container_image = container_image + + return false unless can?(current_user, :remove_project, project) + + ContainerImage.transaction do + container_image.destroy! + + unless remove_container_image_tags + raise_error('Failed to remove container image tags. Please try again or contact administrator') + end + end + + true + end + + private + + def raise_error(message) + raise DestroyError.new(message) + end + + def remove_container_image_tags + container_image.delete_tags + end + end +end diff --git a/app/services/container_images_repositories/container_images/create_service.rb b/app/services/container_images_repositories/container_images/create_service.rb deleted file mode 100644 index 0c2c69d5183..00000000000 --- a/app/services/container_images_repositories/container_images/create_service.rb +++ /dev/null @@ -1,16 +0,0 @@ -module ContainerImagesRepositories - module ContainerImages - class CreateService < BaseService - def execute - @container_image = container_images_repository.container_images.create(params) - @container_image if @container_image.valid? - end - - private - - def container_images_repository - @container_images_repository ||= project.container_images_repository - end - end - end -end diff --git a/app/services/container_images_repositories/container_images/destroy_service.rb b/app/services/container_images_repositories/container_images/destroy_service.rb deleted file mode 100644 index 91b8cfeea47..00000000000 --- a/app/services/container_images_repositories/container_images/destroy_service.rb +++ /dev/null @@ -1,11 +0,0 @@ -module ContainerImagesRepositories - module ContainerImages - class DestroyService < BaseService - def execute(container_image) - return false unless container_image - - container_image.destroy - end - end - end -end diff --git a/app/services/container_images_repositories/container_images/push_service.rb b/app/services/container_images_repositories/container_images/push_service.rb deleted file mode 100644 index 2731cf1d52e..00000000000 --- a/app/services/container_images_repositories/container_images/push_service.rb +++ /dev/null @@ -1,26 +0,0 @@ -module ContainerImagesRepositories - module ContainerImages - class PushService < BaseService - def execute(container_image_name, event) - find_or_create_container_image(container_image_name).valid? - end - - private - - def find_or_create_container_image(container_image_name) - options = {name: container_image_name} - container_images.find_by(options) || - ::ContainerImagesRepositories::ContainerImages::CreateService.new(project, - current_user, options).execute - end - - def container_images_repository - @container_images_repository ||= project.container_images_repository - end - - def container_images - @container_images ||= container_images_repository.container_images - end - end - end -end diff --git a/app/services/container_images_repositories/create_service.rb b/app/services/container_images_repositories/create_service.rb deleted file mode 100644 index 7e9dd3abe5f..00000000000 --- a/app/services/container_images_repositories/create_service.rb +++ /dev/null @@ -1,7 +0,0 @@ -module ContainerImagesRepositories - class CreateService < BaseService - def execute - project.container_images_repository || ::ContainerImagesRepository.create(project: project) - end - end -end diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index 9716a1780a9..ba410b79e8c 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -31,10 +31,6 @@ module Projects project.team.truncate project.destroy! - unless remove_registry_tags - raise_error('Failed to remove project container registry. Please try again or contact administrator') - end - unless remove_repository(repo_path) raise_error('Failed to remove project repository. Please try again or contact administrator') end @@ -68,12 +64,6 @@ module Projects end end - def remove_registry_tags - return true unless Gitlab.config.registry.enabled - - project.container_registry_repository.delete_tags - end - def raise_error(message) raise DestroyError.new(message) end diff --git a/app/views/projects/container_registry/_image.html.haml b/app/views/projects/container_registry/_image.html.haml new file mode 100644 index 00000000000..b1d62e34a97 --- /dev/null +++ b/app/views/projects/container_registry/_image.html.haml @@ -0,0 +1,35 @@ +- expanded = false +.container-image.js-toggle-container + .container-image-head + = link_to "#", class: "js-toggle-button" do + - if expanded + = icon("chevron-up") + - else + = icon("chevron-down") + + = escape_once(image.name) + = clipboard_button(clipboard_text: "docker pull #{image.path}") + .controls.hidden-xs.pull-right + = link_to namespace_project_container_registry_path(@project.namespace, @project, image.id), class: 'btn btn-remove has-tooltip', title: "Remove", data: { confirm: "Are you sure?" }, method: :delete do + = icon("trash cred") + + + .container-image-tags.js-toggle-content{ class: ("hide" unless expanded) } + - if image.tags.blank? + %li + .nothing-here-block No tags in Container Registry for this container image. + + - else + .table-holder + %table.table.tags + %thead + %tr + %th Name + %th Image ID + %th Size + %th Created + - if can?(current_user, :update_container_image, @project) + %th + + - image.tags.each do |tag| + = render 'tag', tag: tag diff --git a/app/views/projects/container_registry/_tag.html.haml b/app/views/projects/container_registry/_tag.html.haml index 10822b6184c..00345ec26de 100644 --- a/app/views/projects/container_registry/_tag.html.haml +++ b/app/views/projects/container_registry/_tag.html.haml @@ -25,5 +25,5 @@ - if can?(current_user, :update_container_image, @project) %td.content .controls.hidden-xs.pull-right - = link_to namespace_project_container_registry_path(@project.namespace, @project, tag.name), class: 'btn btn-remove has-tooltip', title: "Remove", data: { confirm: "Are you sure?" }, method: :delete do + = link_to namespace_project_container_registry_path(@project.namespace, @project, { id: tag.repository.id, tag: tag.name} ), class: 'btn btn-remove has-tooltip', title: "Remove", data: { confirm: "Are you sure?" }, method: :delete do = icon("trash cred") diff --git a/app/views/projects/container_registry/index.html.haml b/app/views/projects/container_registry/index.html.haml index 993da27310f..f074ce6be6d 100644 --- a/app/views/projects/container_registry/index.html.haml +++ b/app/views/projects/container_registry/index.html.haml @@ -19,21 +19,9 @@ %br docker push #{escape_once(@project.container_registry_repository_url)} - - if @tags.blank? - %li - .nothing-here-block No images in Container Registry for this project. + - if @images.blank? + .nothing-here-block No container images in Container Registry for this project. - else - .table-holder - %table.table.tags - %thead - %tr - %th Name - %th Image ID - %th Size - %th Created - - if can?(current_user, :update_container_image, @project) - %th - - - @tags.each do |tag| - = render 'tag', tag: tag + - @images.each do |image| + = render 'image', image: image diff --git a/db/migrate/20161029153736_create_container_images_repository.rb b/db/migrate/20161029153736_create_container_images_repository.rb deleted file mode 100644 index d93180b1674..00000000000 --- a/db/migrate/20161029153736_create_container_images_repository.rb +++ /dev/null @@ -1,31 +0,0 @@ -# See http://doc.gitlab.com/ce/development/migration_style_guide.html -# for more information on how to write migrations for GitLab. - -class CreateContainerImagesRepository < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - # Set this constant to true if this migration requires downtime. - DOWNTIME = false - - # When a migration requires downtime you **must** uncomment the following - # constant and define a short and easy to understand explanation as to why the - # migration requires downtime. - # DOWNTIME_REASON = '' - - # When using the methods "add_concurrent_index" or "add_column_with_default" - # you must disable the use of transactions as these methods can not run in an - # existing transaction. When using "add_concurrent_index" make sure that this - # method is the _only_ method called in the migration, any other changes - # should go in a separate migration. This ensures that upon failure _only_ the - # index creation fails and can be retried or reverted easily. - # - # To disable transactions uncomment the following line and remove these - # comments: - # disable_ddl_transaction! - - def change - create_table :container_images_repositories do |t| - t.integer :project_id, null: false - end - end -end diff --git a/db/migrate/20161031013926_create_container_image.rb b/db/migrate/20161031013926_create_container_image.rb index 94feae280a6..85c0913b8f3 100644 --- a/db/migrate/20161031013926_create_container_image.rb +++ b/db/migrate/20161031013926_create_container_image.rb @@ -25,7 +25,7 @@ class CreateContainerImage < ActiveRecord::Migration def change create_table :container_images do |t| - t.integer :container_images_repository_id + t.integer :project_id t.string :name end end diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb index c0473051424..dc7279d2b75 100644 --- a/lib/api/registry_events.rb +++ b/lib/api/registry_events.rb @@ -41,9 +41,19 @@ module API if event['action'] == 'push' and !!event['target']['tag'] namespace, container_image_name = ContainerImage::split_namespace(repository) - ::ContainerImagesRepositories::ContainerImages::PushService.new( - Project::find_with_namespace(namespace), current_user - ).execute(container_image_name, event) + project = Project::find_with_namespace(namespace) + + if project + container_image = project.container_images.find_or_create_by(name: container_image_name) + + if container_image.valid? + puts('Valid!') + else + render_api_error!({ error: "Failed to create container image!" }, 400) + end + else + not_found!('Project') + end end end end From 246df2bd1151d39a04ef553064144eb75ee3e980 Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Tue, 13 Dec 2016 23:42:43 -0200 Subject: [PATCH 04/97] Adding registry endpoint authorization --- .../admin/application_settings_controller.rb | 6 ++++ .../admin/container_registry_controller.rb | 11 +++++++ app/models/application_setting.rb | 6 ++++ .../admin/container_registry/show.html.haml | 31 +++++++++++++++++++ app/views/admin/dashboard/_head.html.haml | 4 +++ config/routes/admin.rb | 2 ++ ...ry_access_token_to_application_settings.rb | 29 +++++++++++++++++ doc/administration/container_registry.md | 22 +++++++++++-- doc/ci/docker/using_docker_build.md | 8 ++--- doc/user/project/container_registry.md | 19 ++++++------ lib/api/helpers.rb | 10 ++++++ lib/api/registry_events.rb | 2 +- 12 files changed, 133 insertions(+), 17 deletions(-) create mode 100644 app/controllers/admin/container_registry_controller.rb create mode 100644 app/views/admin/container_registry/show.html.haml create mode 100644 db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb index b0f5d4a9933..fb6df1a06d2 100644 --- a/app/controllers/admin/application_settings_controller.rb +++ b/app/controllers/admin/application_settings_controller.rb @@ -29,6 +29,12 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController redirect_to :back end + def reset_container_registry_token + @application_setting.reset_container_registry_access_token! + flash[:notice] = 'New container registry access token has been generated!' + redirect_to :back + end + def clear_repository_check_states RepositoryCheck::ClearWorker.perform_async diff --git a/app/controllers/admin/container_registry_controller.rb b/app/controllers/admin/container_registry_controller.rb new file mode 100644 index 00000000000..265c032c67d --- /dev/null +++ b/app/controllers/admin/container_registry_controller.rb @@ -0,0 +1,11 @@ +class Admin::ContainerRegistryController < Admin::ApplicationController + def show + @access_token = container_registry_access_token + end + + private + + def container_registry_access_token + current_application_settings.container_registry_access_token + end +end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 74b358d8c40..b94a71e1ea7 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -4,6 +4,7 @@ class ApplicationSetting < ActiveRecord::Base add_authentication_token_field :runners_registration_token add_authentication_token_field :health_check_access_token + add_authentication_token_field :container_registry_access_token CACHE_KEY = 'application_setting.last' DOMAIN_LIST_SEPARATOR = %r{\s*[,;]\s* # comma or semicolon, optionally surrounded by whitespace @@ -141,6 +142,7 @@ class ApplicationSetting < ActiveRecord::Base before_save :ensure_runners_registration_token before_save :ensure_health_check_access_token + before_save :ensure_container_registry_access_token after_commit do Rails.cache.write(CACHE_KEY, self) @@ -276,6 +278,10 @@ class ApplicationSetting < ActiveRecord::Base ensure_health_check_access_token! end + def container_registry_access_token + ensure_container_registry_access_token! + end + def sidekiq_throttling_enabled? return false unless sidekiq_throttling_column_exists? diff --git a/app/views/admin/container_registry/show.html.haml b/app/views/admin/container_registry/show.html.haml new file mode 100644 index 00000000000..8803eddda69 --- /dev/null +++ b/app/views/admin/container_registry/show.html.haml @@ -0,0 +1,31 @@ +- @no_container = true += render "admin/dashboard/head" + +%div{ class: container_class } + + %p.prepend-top-default + %span + To properly configure the Container Registry you should add the following + access token to the Docker Registry config.yml as follows: + %pre + %code + :plain + notifications: + endpoints: + - ... + headers: + X-Registry-Token: [#{@access_token}] + %br + Access token is + %code{ id: 'registry-token' } #{@access_token} + + .bs-callout.clearfix + .pull-left + %p + You can reset container registry access token by pressing the button below. + %p + = button_to reset_container_registry_token_admin_application_settings_path, + method: :put, class: 'btn btn-default', + data: { confirm: 'Are you sure you want to reset container registry token?' } do + = icon('refresh') + Reset container registry access token diff --git a/app/views/admin/dashboard/_head.html.haml b/app/views/admin/dashboard/_head.html.haml index 7893c1dee97..dbd039547fa 100644 --- a/app/views/admin/dashboard/_head.html.haml +++ b/app/views/admin/dashboard/_head.html.haml @@ -27,3 +27,7 @@ = link_to admin_runners_path, title: 'Runners' do %span Runners + = nav_link path: 'container_registry#show' do + = link_to admin_container_registry_path, title: 'Registry' do + %span + Registry diff --git a/config/routes/admin.rb b/config/routes/admin.rb index 8e99239f350..b09c05826a7 100644 --- a/config/routes/admin.rb +++ b/config/routes/admin.rb @@ -58,6 +58,7 @@ namespace :admin do resource :background_jobs, controller: 'background_jobs', only: [:show] resource :system_info, controller: 'system_info', only: [:show] resources :requests_profiles, only: [:index, :show], param: :name, constraints: { name: /.+\.html/ } + resource :container_registry, controller: 'container_registry', only: [:show] resources :projects, only: [:index] @@ -88,6 +89,7 @@ namespace :admin do resources :services, only: [:index, :edit, :update] put :reset_runners_token put :reset_health_check_token + put :reset_container_registry_token put :clear_repository_check_states end diff --git a/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb b/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb new file mode 100644 index 00000000000..f89f9b00a5f --- /dev/null +++ b/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb @@ -0,0 +1,29 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddContainerRegistryAccessTokenToApplicationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index" or "add_column_with_default" + # you must disable the use of transactions as these methods can not run in an + # existing transaction. When using "add_concurrent_index" make sure that this + # method is the _only_ method called in the migration, any other changes + # should go in a separate migration. This ensures that upon failure _only_ the + # index creation fails and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + add_column :application_settings, :container_registry_access_token, :string + end +end diff --git a/doc/administration/container_registry.md b/doc/administration/container_registry.md index a6300e18dc0..14795601246 100644 --- a/doc/administration/container_registry.md +++ b/doc/administration/container_registry.md @@ -76,7 +76,7 @@ you modify its settings. Read the upstream documentation on how to achieve that. At the absolute minimum, make sure your [Registry configuration][registry-auth] has `container_registry` as the service and `https://gitlab.example.com/jwt/auth` -as the realm: +as the realm. ``` auth: @@ -87,6 +87,23 @@ auth: rootcertbundle: /root/certs/certbundle ``` +Also a notification endpoint must be configured with the token from +Admin Area -> Overview -> Registry (`/admin/container_registry`) like in the following sample: + +``` +notifications: + endpoints: + - name: listener + url: https://gitlab.example.com/api/v3/registry_events + headers: + X-Registry-Token: [57Cx95fc2zHFh93VTiGD] + timeout: 500ms + threshold: 5 + backoff: 1s +``` + +Check the [Registry endpoint configuration][registry-endpoint] for details. + ## Container Registry domain configuration There are two ways you can configure the Registry's external domain. @@ -477,7 +494,7 @@ configurable in future releases. **GitLab 8.8 ([source docs][8-8-docs])** - GitLab Container Registry feature was introduced. - +i [reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure [restart gitlab]: restart_gitlab.md#installations-from-source [wildcard certificate]: https://en.wikipedia.org/wiki/Wildcard_certificate @@ -487,6 +504,7 @@ configurable in future releases. [storage-config]: https://docs.docker.com/registry/configuration/#storage [registry-http-config]: https://docs.docker.com/registry/configuration/#http [registry-auth]: https://docs.docker.com/registry/configuration/#auth +[registry-endpoint]: https://docs.docker.com/registry/notifications/#/configuration [token-config]: https://docs.docker.com/registry/configuration/#token [8-8-docs]: https://gitlab.com/gitlab-org/gitlab-ce/blob/8-8-stable/doc/administration/container_registry.md [registry-ssl]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/registry-ssl diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index 8620984d40d..6ae6269b28a 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -299,8 +299,8 @@ could look like: stage: build script: - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN registry.example.com - - docker build -t registry.example.com/group/project:latest . - - docker push registry.example.com/group/project:latest + - docker build -t registry.example.com/group/project/image:latest . + - docker push registry.example.com/group/project/image:latest ``` You have to use the special `gitlab-ci-token` user created for you in order to @@ -350,8 +350,8 @@ stages: - deploy variables: - CONTAINER_TEST_IMAGE: registry.example.com/my-group/my-project:$CI_BUILD_REF_NAME - CONTAINER_RELEASE_IMAGE: registry.example.com/my-group/my-project:latest + CONTAINER_TEST_IMAGE: registry.example.com/my-group/my-project/my-image:$CI_BUILD_REF_NAME + CONTAINER_RELEASE_IMAGE: registry.example.com/my-group/my-project/my-image:latest before_script: - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN registry.example.com diff --git a/doc/user/project/container_registry.md b/doc/user/project/container_registry.md index 91b35c73b34..eada8e04227 100644 --- a/doc/user/project/container_registry.md +++ b/doc/user/project/container_registry.md @@ -10,6 +10,7 @@ - Starting from GitLab 8.12, if you have 2FA enabled in your account, you need to pass a personal access token instead of your password in order to login to GitLab's Container Registry. +- Multiple level image names support was added in GitLab ?8.15? With the Docker Container Registry integrated into GitLab, every project can have its own space to store its Docker images. @@ -54,26 +55,23 @@ sure that you are using the Registry URL with the namespace and project name that is hosted on GitLab: ``` -docker build -t registry.example.com/group/project . -docker push registry.example.com/group/project +docker build -t registry.example.com/group/project/image . +docker push registry.example.com/group/project/image ``` Your image will be named after the following scheme: ``` -// +/// ``` -As such, the name of the image is unique, but you can differentiate the images -using tags. - ## Use images from GitLab Container Registry To download and run a container from images hosted in GitLab Container Registry, use `docker run`: ``` -docker run [options] registry.example.com/group/project [arguments] +docker run [options] registry.example.com/group/project/image [arguments] ``` For more information on running Docker containers, visit the @@ -87,7 +85,8 @@ and click **Registry** in the project menu. This view will show you all tags in your project and will easily allow you to delete them. -![Container Registry panel](img/container_registry_panel.png) +![Container Registry panel](image-needs-update) +[//]: # (img/container_registry_panel.png) ## Build and push images using GitLab CI @@ -136,7 +135,7 @@ A user attempted to enable an S3-backed Registry. The `docker login` step went fine. However, when pushing an image, the output showed: ``` -The push refers to a repository [s3-testing.myregistry.com:4567/root/docker-test] +The push refers to a repository [s3-testing.myregistry.com:4567/root/docker-test/docker-image] dc5e59c14160: Pushing [==================================================>] 14.85 kB 03c20c1a019a: Pushing [==================================================>] 2.048 kB a08f14ef632e: Pushing [==================================================>] 2.048 kB @@ -229,7 +228,7 @@ a container image. You may need to run as root to do this. For example: ```sh docker login s3-testing.myregistry.com:4567 -docker push s3-testing.myregistry.com:4567/root/docker-test +docker push s3-testing.myregistry.com:4567/root/docker-test/docker-image ``` In the example above, we see the following trace on the mitmproxy window: diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a1db2099693..0fd2b1587e3 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -111,6 +111,16 @@ module API end end + def authenticate_container_registry_access_token! + token = request.headers['X-Registry-Token'] + unless token.present? && ActiveSupport::SecurityUtils.variable_size_secure_compare( + token, + current_application_settings.container_registry_access_token + ) + unauthorized! + end + end + def authenticated_as_admin! authenticate! forbidden! unless current_user.is_admin? diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb index dc7279d2b75..e52433339eb 100644 --- a/lib/api/registry_events.rb +++ b/lib/api/registry_events.rb @@ -1,7 +1,7 @@ module API # RegistryEvents API class RegistryEvents < Grape::API - # before { authenticate! } + before { authenticate_container_registry_access_token! } content_type :json, 'application/vnd.docker.distribution.events.v1+json' From e4fa80f3b67f1ef30c262cd4df28516ccff6336a Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Fri, 16 Dec 2016 01:24:05 -0200 Subject: [PATCH 05/97] Fixes broken and missing tests --- .../stylesheets/pages/container_registry.scss | 6 +- .../projects/container_registry_controller.rb | 1 - app/models/container_image.rb | 4 +- app/models/namespace.rb | 8 +- app/models/project.rb | 18 ++--- ...ntainer_registry_authentication_service.rb | 3 +- .../container_images/destroy_service.rb | 1 - app/services/projects/transfer_service.rb | 6 +- .../container_registry/_image.html.haml | 2 +- .../container_registry/_tag.html.haml | 2 +- .../container_registry/index.html.haml | 4 +- db/schema.rb | 6 ++ lib/api/registry_events.rb | 4 +- lib/container_registry/blob.rb | 4 +- lib/container_registry/registry.rb | 4 - lib/container_registry/repository.rb | 48 ------------ spec/factories/container_images.rb | 21 ++++++ spec/features/container_registry_spec.rb | 32 +++++--- .../security/project/internal_access_spec.rb | 3 + .../security/project/private_access_spec.rb | 3 + .../security/project/public_access_spec.rb | 3 + spec/lib/container_registry/blob_spec.rb | 15 +++- spec/lib/container_registry/registry_spec.rb | 2 +- .../lib/container_registry/repository_spec.rb | 65 ----------------- spec/lib/container_registry/tag_spec.rb | 11 ++- spec/lib/gitlab/import_export/all_models.yml | 3 + spec/models/ci/build_spec.rb | 2 +- spec/models/container_image_spec.rb | 73 +++++++++++++++++++ spec/models/namespace_spec.rb | 8 +- spec/models/project_spec.rb | 41 ++--------- .../services/projects/destroy_service_spec.rb | 15 ++-- .../projects/transfer_service_spec.rb | 3 + 32 files changed, 211 insertions(+), 210 deletions(-) delete mode 100644 lib/container_registry/repository.rb create mode 100644 spec/factories/container_images.rb delete mode 100644 spec/lib/container_registry/repository_spec.rb create mode 100644 spec/models/container_image_spec.rb diff --git a/app/assets/stylesheets/pages/container_registry.scss b/app/assets/stylesheets/pages/container_registry.scss index 7d68eae3c97..92543d7d714 100644 --- a/app/assets/stylesheets/pages/container_registry.scss +++ b/app/assets/stylesheets/pages/container_registry.scss @@ -3,14 +3,14 @@ */ .container-image { - border-bottom: 1px solid #f0f0f0; + border-bottom: 1px solid $white-normal; } .container-image-head { - padding: 0px 16px; + padding: 0 16px; line-height: 4; } .table.tags { - margin-bottom: 0px; + margin-bottom: 0; } diff --git a/app/controllers/projects/container_registry_controller.rb b/app/controllers/projects/container_registry_controller.rb index 54bcb5f504a..f656f86fcdb 100644 --- a/app/controllers/projects/container_registry_controller.rb +++ b/app/controllers/projects/container_registry_controller.rb @@ -20,7 +20,6 @@ class Projects::ContainerRegistryController < Projects::ApplicationController redirect_to url, alert: 'Failed to remove image' end end - end private diff --git a/app/models/container_image.rb b/app/models/container_image.rb index 7721c53a6fc..583cb977910 100644 --- a/app/models/container_image.rb +++ b/app/models/container_image.rb @@ -22,7 +22,7 @@ class ContainerImage < ActiveRecord::Base end def name_with_namespace - [container_registry_path_with_namespace, name].compact.join('/') + [container_registry_path_with_namespace, name].reject(&:blank?).join('/') end def tag(tag) @@ -55,6 +55,8 @@ class ContainerImage < ActiveRecord::Base end end + # rubocop:disable RedundantReturn + def self.split_namespace(full_path) image_name = full_path.split('/').last namespace = full_path.gsub(/(.*)(#{Regexp.escape('/' + image_name)})/, '\1') diff --git a/app/models/namespace.rb b/app/models/namespace.rb index bd0336c984a..c8e329044e0 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -118,8 +118,8 @@ class Namespace < ActiveRecord::Base end def move_dir - if any_project_has_container_registry_tags? - raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry') + if any_project_has_container_registry_images? + raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has images in container registry') end # Move the namespace directory in all storages paths used by member projects @@ -154,8 +154,8 @@ class Namespace < ActiveRecord::Base end end - def any_project_has_container_registry_tags? - projects.any?(&:has_container_registry_tags?) + def any_project_has_container_registry_images? + projects.any? { |project| project.container_images.present? } end def send_update_instructions diff --git a/app/models/project.rb b/app/models/project.rb index afaf2095a4c..d4f5584f53d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -421,18 +421,12 @@ class Project < ActiveRecord::Base end end - def container_registry_repository_url + def container_registry_url if Gitlab.config.registry.enabled "#{Gitlab.config.registry.host_port}/#{container_registry_path_with_namespace}" end end - def has_container_registry_tags? - return unless container_images - - container_images.first.tags.any? - end - def commit(ref = 'HEAD') repository.commit(ref) end @@ -913,11 +907,11 @@ class Project < ActiveRecord::Base expire_caches_before_rename(old_path_with_namespace) - if has_container_registry_tags? - Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present" + if container_images.present? + Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry images are present" - # we currently doesn't support renaming repository if it contains tags in container registry - raise StandardError.new('Project cannot be renamed, because tags are present in its container registry') + # we currently doesn't support renaming repository if it contains images in container registry + raise StandardError.new('Project cannot be renamed, because images are present in its container registry') end if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace) @@ -1264,7 +1258,7 @@ class Project < ActiveRecord::Base ] if container_registry_enabled? - variables << { key: 'CI_REGISTRY_IMAGE', value: container_registry_repository_url, public: true } + variables << { key: 'CI_REGISTRY_IMAGE', value: container_registry_url, public: true } end variables diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 6b83b38fa4d..5b2fcdf3b16 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -16,7 +16,8 @@ module Auth { token: authorized_token(scope).encoded } end - def self.full_access_token(names) + def self.full_access_token(*names) + names = names.flatten registry = Gitlab.config.registry token = JSONWebToken::RSAToken.new(registry.key) token.issuer = registry.issuer diff --git a/app/services/container_images/destroy_service.rb b/app/services/container_images/destroy_service.rb index bc5b53fd055..c73b6cfefba 100644 --- a/app/services/container_images/destroy_service.rb +++ b/app/services/container_images/destroy_service.rb @@ -1,6 +1,5 @@ module ContainerImages class DestroyService < BaseService - class DestroyError < StandardError; end def execute(container_image) diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 20dfbddc823..3e241b9e7c0 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -36,9 +36,9 @@ module Projects raise TransferError.new("Project with same path in target namespace already exists") end - if project.has_container_registry_tags? - # we currently doesn't support renaming repository if it contains tags in container registry - raise TransferError.new('Project cannot be transferred, because tags are present in its container registry') + unless project.container_images.empty? + # we currently doesn't support renaming repository if it contains images in container registry + raise TransferError.new('Project cannot be transferred, because images are present in its container registry') end project.expire_caches_before_rename(old_path) diff --git a/app/views/projects/container_registry/_image.html.haml b/app/views/projects/container_registry/_image.html.haml index b1d62e34a97..5845efd345a 100644 --- a/app/views/projects/container_registry/_image.html.haml +++ b/app/views/projects/container_registry/_image.html.haml @@ -10,7 +10,7 @@ = escape_once(image.name) = clipboard_button(clipboard_text: "docker pull #{image.path}") .controls.hidden-xs.pull-right - = link_to namespace_project_container_registry_path(@project.namespace, @project, image.id), class: 'btn btn-remove has-tooltip', title: "Remove", data: { confirm: "Are you sure?" }, method: :delete do + = link_to namespace_project_container_registry_path(@project.namespace, @project, image.id), class: 'btn btn-remove has-tooltip', title: "Remove image", data: { confirm: "Are you sure?" }, method: :delete do = icon("trash cred") diff --git a/app/views/projects/container_registry/_tag.html.haml b/app/views/projects/container_registry/_tag.html.haml index 00345ec26de..b35a9cb621f 100644 --- a/app/views/projects/container_registry/_tag.html.haml +++ b/app/views/projects/container_registry/_tag.html.haml @@ -25,5 +25,5 @@ - if can?(current_user, :update_container_image, @project) %td.content .controls.hidden-xs.pull-right - = link_to namespace_project_container_registry_path(@project.namespace, @project, { id: tag.repository.id, tag: tag.name} ), class: 'btn btn-remove has-tooltip', title: "Remove", data: { confirm: "Are you sure?" }, method: :delete do + = link_to namespace_project_container_registry_path(@project.namespace, @project, { id: tag.repository.id, tag: tag.name} ), class: 'btn btn-remove has-tooltip', title: "Remove tag", data: { confirm: "Are you sure?" }, method: :delete do = icon("trash cred") diff --git a/app/views/projects/container_registry/index.html.haml b/app/views/projects/container_registry/index.html.haml index f074ce6be6d..ab6213f03d8 100644 --- a/app/views/projects/container_registry/index.html.haml +++ b/app/views/projects/container_registry/index.html.haml @@ -15,9 +15,9 @@ %br Then you are free to create and upload a container image with build and push commands: %pre - docker build -t #{escape_once(@project.container_registry_repository_url)} . + docker build -t #{escape_once(@project.container_registry_url)} . %br - docker push #{escape_once(@project.container_registry_repository_url)} + docker push #{escape_once(@project.container_registry_url)} - if @images.blank? .nothing-here-block No container images in Container Registry for this project. diff --git a/db/schema.rb b/db/schema.rb index 88aaa6c3c55..36df20fc8f2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -109,6 +109,7 @@ ActiveRecord::Schema.define(version: 20170215200045) do t.boolean "html_emails_enabled", default: true t.string "plantuml_url" t.boolean "plantuml_enabled" + t.string "container_registry_access_token" t.integer "max_pages_size", default: 100, null: false t.integer "terminal_max_session_time", default: 0, null: false end @@ -392,6 +393,11 @@ ActiveRecord::Schema.define(version: 20170215200045) do add_index "ci_variables", ["gl_project_id"], name: "index_ci_variables_on_gl_project_id", using: :btree + create_table "container_images", force: :cascade do |t| + t.integer "project_id" + t.string "name" + end + create_table "deploy_keys_projects", force: :cascade do |t| t.integer "deploy_key_id", null: false t.integer "project_id", null: false diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb index e52433339eb..12305a49f0f 100644 --- a/lib/api/registry_events.rb +++ b/lib/api/registry_events.rb @@ -46,9 +46,7 @@ module API if project container_image = project.container_images.find_or_create_by(name: container_image_name) - if container_image.valid? - puts('Valid!') - else + unless container_image.valid? render_api_error!({ error: "Failed to create container image!" }, 400) end else diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb index eb5a2596177..8db8e483b1d 100644 --- a/lib/container_registry/blob.rb +++ b/lib/container_registry/blob.rb @@ -38,11 +38,11 @@ module ContainerRegistry end def delete - client.delete_blob(repository.name, digest) + client.delete_blob(repository.name_with_namespace, digest) end def data - @data ||= client.blob(repository.name, digest, type) + @data ||= client.blob(repository.name_with_namespace, digest, type) end end end diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb index 0e634f6b6ef..63bce655f57 100644 --- a/lib/container_registry/registry.rb +++ b/lib/container_registry/registry.rb @@ -8,10 +8,6 @@ module ContainerRegistry @client = ContainerRegistry::Client.new(uri, options) end - def repository(name) - ContainerRegistry::Repository.new(self, name) - end - private def default_path diff --git a/lib/container_registry/repository.rb b/lib/container_registry/repository.rb deleted file mode 100644 index 0e4a7cb3cc9..00000000000 --- a/lib/container_registry/repository.rb +++ /dev/null @@ -1,48 +0,0 @@ -module ContainerRegistry - class Repository - attr_reader :registry, :name - - delegate :client, to: :registry - - def initialize(registry, name) - @registry, @name = registry, name - end - - def path - [registry.path, name].compact.join('/') - end - - def tag(tag) - ContainerRegistry::Tag.new(self, tag) - end - - def manifest - return @manifest if defined?(@manifest) - - @manifest = client.repository_tags(name) - end - - def valid? - manifest.present? - end - - def tags - return @tags if defined?(@tags) - return [] unless manifest && manifest['tags'] - - @tags = manifest['tags'].map do |tag| - ContainerRegistry::Tag.new(self, tag) - end - end - - def blob(config) - ContainerRegistry::Blob.new(self, config) - end - - def delete_tags - return unless tags - - tags.all?(&:delete) - end - end -end diff --git a/spec/factories/container_images.rb b/spec/factories/container_images.rb new file mode 100644 index 00000000000..6141a519a75 --- /dev/null +++ b/spec/factories/container_images.rb @@ -0,0 +1,21 @@ +FactoryGirl.define do + factory :container_image do + name "test_container_image" + project + + transient do + tags ['tag'] + stubbed true + end + + after(:build) do |image, evaluator| + if evaluator.stubbed + allow(Gitlab.config.registry).to receive(:enabled).and_return(true) + allow(image.client).to receive(:repository_tags).and_return({ + name: image.name_with_namespace, + tags: evaluator.tags + }) + end + end + end +end diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index 203e55a36f2..862c9fbf6c0 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -2,15 +2,18 @@ require 'spec_helper' describe "Container Registry" do let(:project) { create(:empty_project) } - let(:repository) { project.container_registry_repository } + let(:registry) { project.container_registry } let(:tag_name) { 'latest' } let(:tags) { [tag_name] } + let(:container_image) { create(:container_image) } + let(:image_name) { container_image.name } before do login_as(:user) project.team << [@user, :developer] - stub_container_registry_tags(*tags) stub_container_registry_config(enabled: true) + stub_container_registry_tags(*tags) + project.container_images << container_image unless container_image.nil? allow(Auth::ContainerRegistryAuthenticationService).to receive(:full_access_token).and_return('token') end @@ -19,15 +22,26 @@ describe "Container Registry" do visit namespace_project_container_registry_index_path(project.namespace, project) end - context 'when no tags' do - let(:tags) { [] } + context 'when no images' do + let(:container_image) { } - it { expect(page).to have_content('No images in Container Registry for this project') } + it { expect(page).to have_content('No container images in Container Registry for this project') } end - context 'when there are tags' do - it { expect(page).to have_content(tag_name) } - it { expect(page).to have_content('d7a513a66') } + context 'when there are images' do + it { expect(page).to have_content(image_name) } + end + end + + describe 'DELETE /:project/container_registry/:image_id' do + before do + visit namespace_project_container_registry_index_path(project.namespace, project) + end + + it do + expect_any_instance_of(ContainerImage).to receive(:delete_tags).and_return(true) + + click_on 'Remove image' end end @@ -39,7 +53,7 @@ describe "Container Registry" do it do expect_any_instance_of(::ContainerRegistry::Tag).to receive(:delete).and_return(true) - click_on 'Remove' + click_on 'Remove tag' end end end diff --git a/spec/features/security/project/internal_access_spec.rb b/spec/features/security/project/internal_access_spec.rb index 24af062d763..4e7a2c0ecc0 100644 --- a/spec/features/security/project/internal_access_spec.rb +++ b/spec/features/security/project/internal_access_spec.rb @@ -429,9 +429,12 @@ describe "Internal Project Access", feature: true do end describe "GET /:project_path/container_registry" do + let(:container_image) { create(:container_image) } + before do stub_container_registry_tags('latest') stub_container_registry_config(enabled: true) + project.container_images << container_image end subject { namespace_project_container_registry_index_path(project.namespace, project) } diff --git a/spec/features/security/project/private_access_spec.rb b/spec/features/security/project/private_access_spec.rb index c511dcfa18e..c74cdc05593 100644 --- a/spec/features/security/project/private_access_spec.rb +++ b/spec/features/security/project/private_access_spec.rb @@ -418,9 +418,12 @@ describe "Private Project Access", feature: true do end describe "GET /:project_path/container_registry" do + let(:container_image) { create(:container_image) } + before do stub_container_registry_tags('latest') stub_container_registry_config(enabled: true) + project.container_images << container_image end subject { namespace_project_container_registry_index_path(project.namespace, project) } diff --git a/spec/features/security/project/public_access_spec.rb b/spec/features/security/project/public_access_spec.rb index d8cc012c27e..485ef335b78 100644 --- a/spec/features/security/project/public_access_spec.rb +++ b/spec/features/security/project/public_access_spec.rb @@ -429,9 +429,12 @@ describe "Public Project Access", feature: true do end describe "GET /:project_path/container_registry" do + let(:container_image) { create(:container_image) } + before do stub_container_registry_tags('latest') stub_container_registry_config(enabled: true) + project.container_images << container_image end subject { namespace_project_container_registry_index_path(project.namespace, project) } diff --git a/spec/lib/container_registry/blob_spec.rb b/spec/lib/container_registry/blob_spec.rb index bbacdc67ebd..f092449c4bd 100644 --- a/spec/lib/container_registry/blob_spec.rb +++ b/spec/lib/container_registry/blob_spec.rb @@ -9,12 +9,19 @@ describe ContainerRegistry::Blob do 'size' => 1000 } end - let(:token) { 'authorization-token' } - - let(:registry) { ContainerRegistry::Registry.new('http://example.com', token: token) } - let(:repository) { registry.repository('group/test') } + let(:token) { 'token' } + + let(:group) { create(:group, name: 'group') } + let(:project) { create(:project, path: 'test', group: group) } + let(:example_host) { 'example.com' } + let(:registry_url) { 'http://' + example_host } + let(:repository) { create(:container_image, name: '', project: project) } let(:blob) { repository.blob(config) } + before do + stub_container_registry_config(enabled: true, api_url: registry_url, host_port: example_host) + end + it { expect(blob).to respond_to(:repository) } it { expect(blob).to delegate_method(:registry).to(:repository) } it { expect(blob).to delegate_method(:client).to(:repository) } diff --git a/spec/lib/container_registry/registry_spec.rb b/spec/lib/container_registry/registry_spec.rb index 4f3f8b24fc4..4d6eea94bf0 100644 --- a/spec/lib/container_registry/registry_spec.rb +++ b/spec/lib/container_registry/registry_spec.rb @@ -10,7 +10,7 @@ describe ContainerRegistry::Registry do it { is_expected.to respond_to(:uri) } it { is_expected.to respond_to(:path) } - it { expect(subject.repository('test')).not_to be_nil } + it { expect(subject).not_to be_nil } context '#path' do subject { registry.path } diff --git a/spec/lib/container_registry/repository_spec.rb b/spec/lib/container_registry/repository_spec.rb deleted file mode 100644 index c364e759108..00000000000 --- a/spec/lib/container_registry/repository_spec.rb +++ /dev/null @@ -1,65 +0,0 @@ -require 'spec_helper' - -describe ContainerRegistry::Repository do - let(:registry) { ContainerRegistry::Registry.new('http://example.com') } - let(:repository) { registry.repository('group/test') } - - it { expect(repository).to respond_to(:registry) } - it { expect(repository).to delegate_method(:client).to(:registry) } - it { expect(repository.tag('test')).not_to be_nil } - - context '#path' do - subject { repository.path } - - it { is_expected.to eq('example.com/group/test') } - end - - context 'manifest processing' do - before do - stub_request(:get, 'http://example.com/v2/group/test/tags/list'). - with(headers: { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' }). - to_return( - status: 200, - body: JSON.dump(tags: ['test']), - headers: { 'Content-Type' => 'application/json' }) - end - - context '#manifest' do - subject { repository.manifest } - - it { is_expected.not_to be_nil } - end - - context '#valid?' do - subject { repository.valid? } - - it { is_expected.to be_truthy } - end - - context '#tags' do - subject { repository.tags } - - it { is_expected.not_to be_empty } - end - end - - context '#delete_tags' do - let(:tag) { ContainerRegistry::Tag.new(repository, 'tag') } - - before { expect(repository).to receive(:tags).twice.and_return([tag]) } - - subject { repository.delete_tags } - - context 'succeeds' do - before { expect(tag).to receive(:delete).and_return(true) } - - it { is_expected.to be_truthy } - end - - context 'any fails' do - before { expect(tag).to receive(:delete).and_return(false) } - - it { is_expected.to be_falsey } - end - end -end diff --git a/spec/lib/container_registry/tag_spec.rb b/spec/lib/container_registry/tag_spec.rb index c5e31ae82b6..cdd0fe66bc3 100644 --- a/spec/lib/container_registry/tag_spec.rb +++ b/spec/lib/container_registry/tag_spec.rb @@ -1,11 +1,18 @@ require 'spec_helper' describe ContainerRegistry::Tag do - let(:registry) { ContainerRegistry::Registry.new('http://example.com') } - let(:repository) { registry.repository('group/test') } + let(:group) { create(:group, name: 'group') } + let(:project) { create(:project, path: 'test', group: group) } + let(:example_host) { 'example.com' } + let(:registry_url) { 'http://' + example_host } + let(:repository) { create(:container_image, name: '', project: project) } let(:tag) { repository.tag('tag') } let(:headers) { { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' } } + before do + stub_container_registry_config(enabled: true, api_url: registry_url, host_port: example_host) + end + it { expect(tag).to respond_to(:repository) } it { expect(tag).to delegate_method(:registry).to(:repository) } it { expect(tag).to delegate_method(:client).to(:repository) } diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 06617f3b007..9c08f41fe82 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -114,6 +114,8 @@ merge_access_levels: - protected_branch push_access_levels: - protected_branch +container_images: +- name project: - taggings - base_tags @@ -197,6 +199,7 @@ project: - project_authorizations - route - statistics +- container_images award_emoji: - awardable - user diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 2dfca8bcfce..83a2efb55b9 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1397,7 +1397,7 @@ describe Ci::Build, :models do { key: 'CI_REGISTRY', value: 'registry.example.com', public: true } end let(:ci_registry_image) do - { key: 'CI_REGISTRY_IMAGE', value: project.container_registry_repository_url, public: true } + { key: 'CI_REGISTRY_IMAGE', value: project.container_registry_url, public: true } end context 'and is disabled for project' do diff --git a/spec/models/container_image_spec.rb b/spec/models/container_image_spec.rb new file mode 100644 index 00000000000..e0bea737f59 --- /dev/null +++ b/spec/models/container_image_spec.rb @@ -0,0 +1,73 @@ +require 'spec_helper' + +describe ContainerImage do + let(:group) { create(:group, name: 'group') } + let(:project) { create(:project, path: 'test', group: group) } + let(:example_host) { 'example.com' } + let(:registry_url) { 'http://' + example_host } + let(:container_image) { create(:container_image, name: '', project: project, stubbed: false) } + + before do + stub_container_registry_config(enabled: true, api_url: registry_url, host_port: example_host) + stub_request(:get, 'http://example.com/v2/group/test/tags/list'). + with(headers: { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' }). + to_return( + status: 200, + body: JSON.dump(tags: ['test']), + headers: { 'Content-Type' => 'application/json' }) + end + + it { expect(container_image).to respond_to(:project) } + it { expect(container_image).to delegate_method(:container_registry).to(:project) } + it { expect(container_image).to delegate_method(:client).to(:container_registry) } + it { expect(container_image.tag('test')).not_to be_nil } + + context '#path' do + subject { container_image.path } + + it { is_expected.to eq('example.com/group/test') } + end + + context 'manifest processing' do + context '#manifest' do + subject { container_image.manifest } + + it { is_expected.not_to be_nil } + end + + context '#valid?' do + subject { container_image.valid? } + + it { is_expected.to be_truthy } + end + + context '#tags' do + subject { container_image.tags } + + it { is_expected.not_to be_empty } + end + end + + context '#delete_tags' do + let(:tag) { ContainerRegistry::Tag.new(container_image, 'tag') } + + before do + expect(container_image).to receive(:tags).twice.and_return([tag]) + expect(tag).to receive(:digest).and_return('sha256:4c8e63ca4cb663ce6c688cb06f1c3672a172b088dac5b6d7ad7d49cd620d85cf') + end + + subject { container_image.delete_tags } + + context 'succeeds' do + before { expect(container_image.client).to receive(:delete_repository_tag).and_return(true) } + + it { is_expected.to be_truthy } + end + + context 'any fails' do + before { expect(container_image.client).to receive(:delete_repository_tag).and_return(false) } + + it { is_expected.to be_falsey } + end + end +end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 35d932f1c64..aeb4eeb0b55 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -134,18 +134,20 @@ describe Namespace, models: true do expect(@namespace.move_dir).to be_truthy end - context "when any project has container tags" do + context "when any project has container images" do + let(:container_image) { create(:container_image) } + before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') - create(:empty_project, namespace: @namespace) + create(:empty_project, namespace: @namespace, container_images: [container_image]) allow(@namespace).to receive(:path_was).and_return(@namespace.path) allow(@namespace).to receive(:path).and_return('new_path') end - it { expect { @namespace.move_dir }.to raise_error('Namespace cannot be moved, because at least one project has tags in container registry') } + it { expect { @namespace.move_dir }.to raise_error('Namespace cannot be moved, because at least one project has images in container registry') } end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index b0087a9e15d..77f2ff3d17b 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1173,10 +1173,13 @@ describe Project, models: true do project.rename_repo end - context 'container registry with tags' do + context 'container registry with images' do + let(:container_image) { create(:container_image) } + before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') + project.container_images << container_image end subject { project.rename_repo } @@ -1383,20 +1386,20 @@ describe Project, models: true do it { is_expected.to eq(project.path_with_namespace.downcase) } end - describe '#container_registry_repository' do + describe '#container_registry' do let(:project) { create(:empty_project) } before { stub_container_registry_config(enabled: true) } - subject { project.container_registry_repository } + subject { project.container_registry } it { is_expected.not_to be_nil } end - describe '#container_registry_repository_url' do + describe '#container_registry_url' do let(:project) { create(:empty_project) } - subject { project.container_registry_repository_url } + subject { project.container_registry_url } before { stub_container_registry_config(**registry_settings) } @@ -1422,34 +1425,6 @@ describe Project, models: true do end end - describe '#has_container_registry_tags?' do - let(:project) { create(:empty_project) } - - subject { project.has_container_registry_tags? } - - context 'for enabled registry' do - before { stub_container_registry_config(enabled: true) } - - context 'with tags' do - before { stub_container_registry_tags('test', 'test2') } - - it { is_expected.to be_truthy } - end - - context 'when no tags' do - before { stub_container_registry_tags } - - it { is_expected.to be_falsey } - end - end - - context 'for disabled registry' do - before { stub_container_registry_config(enabled: false) } - - it { is_expected.to be_falsey } - end - end - describe '#latest_successful_builds_for' do def create_pipeline(status = 'success') create(:ci_pipeline, project: project, diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index 74bfba44dfd..270e630e70e 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -90,25 +90,30 @@ describe Projects::DestroyService, services: true do end context 'container registry' do + let(:container_image) { create(:container_image) } + before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') + project.container_images << container_image end - context 'tags deletion succeeds' do + context 'images deletion succeeds' do it do - expect_any_instance_of(ContainerRegistry::Tag).to receive(:delete).and_return(true) + expect_any_instance_of(ContainerImage).to receive(:delete_tags).and_return(true) destroy_project(project, user, {}) end end - context 'tags deletion fails' do - before { expect_any_instance_of(ContainerRegistry::Tag).to receive(:delete).and_return(false) } + context 'images deletion fails' do + before do + expect_any_instance_of(ContainerImage).to receive(:delete_tags).and_return(false) + end subject { destroy_project(project, user, {}) } - it { expect{subject}.to raise_error(Projects::DestroyService::DestroyError) } + it { expect{subject}.to raise_error(ActiveRecord::RecordNotDestroyed) } end end diff --git a/spec/services/projects/transfer_service_spec.rb b/spec/services/projects/transfer_service_spec.rb index 5c6fbea8d0e..5e56226ff91 100644 --- a/spec/services/projects/transfer_service_spec.rb +++ b/spec/services/projects/transfer_service_spec.rb @@ -29,9 +29,12 @@ describe Projects::TransferService, services: true do end context 'disallow transfering of project with tags' do + let(:container_image) { create(:container_image) } + before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') + project.container_images << container_image end subject { transfer_project(project, user, group) } From 164ef8a348cac86097313bc453493ccf739adffe Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Fri, 16 Dec 2016 11:12:37 -0200 Subject: [PATCH 06/97] Fixing typos in docs --- doc/administration/container_registry.md | 4 ++-- doc/user/project/container_registry.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/administration/container_registry.md b/doc/administration/container_registry.md index 14795601246..4d1cb391e69 100644 --- a/doc/administration/container_registry.md +++ b/doc/administration/container_registry.md @@ -76,7 +76,7 @@ you modify its settings. Read the upstream documentation on how to achieve that. At the absolute minimum, make sure your [Registry configuration][registry-auth] has `container_registry` as the service and `https://gitlab.example.com/jwt/auth` -as the realm. +as the realm: ``` auth: @@ -494,7 +494,7 @@ configurable in future releases. **GitLab 8.8 ([source docs][8-8-docs])** - GitLab Container Registry feature was introduced. -i + [reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure [restart gitlab]: restart_gitlab.md#installations-from-source [wildcard certificate]: https://en.wikipedia.org/wiki/Wildcard_certificate diff --git a/doc/user/project/container_registry.md b/doc/user/project/container_registry.md index eada8e04227..c5b2266ff19 100644 --- a/doc/user/project/container_registry.md +++ b/doc/user/project/container_registry.md @@ -10,7 +10,7 @@ - Starting from GitLab 8.12, if you have 2FA enabled in your account, you need to pass a personal access token instead of your password in order to login to GitLab's Container Registry. -- Multiple level image names support was added in GitLab ?8.15? +- Multiple level image names support was added in GitLab 8.15 With the Docker Container Registry integrated into GitLab, every project can have its own space to store its Docker images. From b408a192e0fbf630d4f9a4112f6835be50a681d8 Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Fri, 16 Dec 2016 12:07:20 -0200 Subject: [PATCH 07/97] Adding mock for full_access_token --- spec/factories/container_images.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/factories/container_images.rb b/spec/factories/container_images.rb index 6141a519a75..3693865101d 100644 --- a/spec/factories/container_images.rb +++ b/spec/factories/container_images.rb @@ -11,6 +11,7 @@ FactoryGirl.define do after(:build) do |image, evaluator| if evaluator.stubbed allow(Gitlab.config.registry).to receive(:enabled).and_return(true) + allow(Auth::ContainerRegistryAuthenticationService).to receive(:full_access_token).and_return('token') allow(image.client).to receive(:repository_tags).and_return({ name: image.name_with_namespace, tags: evaluator.tags From ea17df5c4c23890c48cd51af17e2517f04f7c88b Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Wed, 25 Jan 2017 10:02:09 -0200 Subject: [PATCH 08/97] Fixing minor view issues --- app/views/projects/container_registry/_image.html.haml | 6 +++--- app/views/projects/container_registry/_tag.html.haml | 2 +- app/views/projects/container_registry/index.html.haml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/views/projects/container_registry/_image.html.haml b/app/views/projects/container_registry/_image.html.haml index 5845efd345a..72f2103b862 100644 --- a/app/views/projects/container_registry/_image.html.haml +++ b/app/views/projects/container_registry/_image.html.haml @@ -8,7 +8,7 @@ = icon("chevron-down") = escape_once(image.name) - = clipboard_button(clipboard_text: "docker pull #{image.path}") + = clipboard_button(clipboard_text: "docker pull #{image.path}") .controls.hidden-xs.pull-right = link_to namespace_project_container_registry_path(@project.namespace, @project, image.id), class: 'btn btn-remove has-tooltip', title: "Remove image", data: { confirm: "Are you sure?" }, method: :delete do = icon("trash cred") @@ -24,8 +24,8 @@ %table.table.tags %thead %tr - %th Name - %th Image ID + %th Tag + %th Tag ID %th Size %th Created - if can?(current_user, :update_container_image, @project) diff --git a/app/views/projects/container_registry/_tag.html.haml b/app/views/projects/container_registry/_tag.html.haml index b35a9cb621f..f7161e85428 100644 --- a/app/views/projects/container_registry/_tag.html.haml +++ b/app/views/projects/container_registry/_tag.html.haml @@ -25,5 +25,5 @@ - if can?(current_user, :update_container_image, @project) %td.content .controls.hidden-xs.pull-right - = link_to namespace_project_container_registry_path(@project.namespace, @project, { id: tag.repository.id, tag: tag.name} ), class: 'btn btn-remove has-tooltip', title: "Remove tag", data: { confirm: "Are you sure?" }, method: :delete do + = link_to namespace_project_container_registry_path(@project.namespace, @project, { id: tag.repository.id, tag: tag.name} ), class: 'btn btn-remove has-tooltip', title: "Remove tag", data: { confirm: "Due to a Docker limitation, all tags with the same ID will also be deleted. Are you sure?" }, method: :delete do = icon("trash cred") diff --git a/app/views/projects/container_registry/index.html.haml b/app/views/projects/container_registry/index.html.haml index ab6213f03d8..5508a3de396 100644 --- a/app/views/projects/container_registry/index.html.haml +++ b/app/views/projects/container_registry/index.html.haml @@ -15,9 +15,9 @@ %br Then you are free to create and upload a container image with build and push commands: %pre - docker build -t #{escape_once(@project.container_registry_url)} . + docker build -t #{escape_once(@project.container_registry_url)}/image . %br - docker push #{escape_once(@project.container_registry_url)} + docker push #{escape_once(@project.container_registry_url)}/image - if @images.blank? .nothing-here-block No container images in Container Registry for this project. From 8294756fc110fdb84036e4ae097940410a8ad6de Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Wed, 25 Jan 2017 10:24:50 -0200 Subject: [PATCH 09/97] Improved readability in tag/image delete condition --- .../projects/container_registry_controller.rb | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/app/controllers/projects/container_registry_controller.rb b/app/controllers/projects/container_registry_controller.rb index f656f86fcdb..4981e57ed22 100644 --- a/app/controllers/projects/container_registry_controller.rb +++ b/app/controllers/projects/container_registry_controller.rb @@ -9,31 +9,37 @@ class Projects::ContainerRegistryController < Projects::ApplicationController end def destroy - url = namespace_project_container_registry_index_path(project.namespace, project) - if tag - delete_tag(url) + delete_tag else - if image.destroy - redirect_to url - else - redirect_to url, alert: 'Failed to remove image' - end + delete_image end end private + def registry_url + @registry_url ||= namespace_project_container_registry_index_path(project.namespace, project) + end + def verify_registry_enabled render_404 unless Gitlab.config.registry.enabled end - def delete_tag(url) + def delete_image + if image.destroy + redirect_to registry_url + else + redirect_to registry_url, alert: 'Failed to remove image' + end + end + + def delete_tag if tag.delete image.destroy if image.tags.empty? - redirect_to url + redirect_to registry_url else - redirect_to url, alert: 'Failed to remove tag' + redirect_to registry_url, alert: 'Failed to remove tag' end end From db5b4b8b1a9b8aa07c8310dde53b7c3ed391bafd Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Wed, 22 Feb 2017 11:19:23 -0300 Subject: [PATCH 10/97] Creates specs for destroy service and improves namespace container image query performance --- app/models/namespace.rb | 2 +- .../container_images/destroy_service.rb | 26 ++------------ .../admin/container_registry/show.html.haml | 2 +- .../20161031013926_create_container_image.rb | 16 --------- ...ry_access_token_to_application_settings.rb | 16 --------- db/schema.rb | 10 +++--- lib/api/registry_events.rb | 4 +-- .../container_images/destroy_service_spec.rb | 34 +++++++++++++++++++ 8 files changed, 45 insertions(+), 65 deletions(-) create mode 100644 spec/services/container_images/destroy_service_spec.rb diff --git a/app/models/namespace.rb b/app/models/namespace.rb index c8e329044e0..a803be2e780 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -155,7 +155,7 @@ class Namespace < ActiveRecord::Base end def any_project_has_container_registry_images? - projects.any? { |project| project.container_images.present? } + projects.joins(:container_images).any? end def send_update_instructions diff --git a/app/services/container_images/destroy_service.rb b/app/services/container_images/destroy_service.rb index c73b6cfefba..15dca227291 100644 --- a/app/services/container_images/destroy_service.rb +++ b/app/services/container_images/destroy_service.rb @@ -1,31 +1,9 @@ module ContainerImages class DestroyService < BaseService - class DestroyError < StandardError; end - def execute(container_image) - @container_image = container_image + return false unless can?(current_user, :update_container_image, project) - return false unless can?(current_user, :remove_project, project) - - ContainerImage.transaction do - container_image.destroy! - - unless remove_container_image_tags - raise_error('Failed to remove container image tags. Please try again or contact administrator') - end - end - - true - end - - private - - def raise_error(message) - raise DestroyError.new(message) - end - - def remove_container_image_tags - container_image.delete_tags + container_image.destroy! end end end diff --git a/app/views/admin/container_registry/show.html.haml b/app/views/admin/container_registry/show.html.haml index 8803eddda69..ffaa7736d65 100644 --- a/app/views/admin/container_registry/show.html.haml +++ b/app/views/admin/container_registry/show.html.haml @@ -17,7 +17,7 @@ X-Registry-Token: [#{@access_token}] %br Access token is - %code{ id: 'registry-token' } #{@access_token} + %code{ id: 'registry-token' }= @access_token .bs-callout.clearfix .pull-left diff --git a/db/migrate/20161031013926_create_container_image.rb b/db/migrate/20161031013926_create_container_image.rb index 85c0913b8f3..884c78880eb 100644 --- a/db/migrate/20161031013926_create_container_image.rb +++ b/db/migrate/20161031013926_create_container_image.rb @@ -7,22 +7,6 @@ class CreateContainerImage < ActiveRecord::Migration # Set this constant to true if this migration requires downtime. DOWNTIME = false - # When a migration requires downtime you **must** uncomment the following - # constant and define a short and easy to understand explanation as to why the - # migration requires downtime. - # DOWNTIME_REASON = '' - - # When using the methods "add_concurrent_index" or "add_column_with_default" - # you must disable the use of transactions as these methods can not run in an - # existing transaction. When using "add_concurrent_index" make sure that this - # method is the _only_ method called in the migration, any other changes - # should go in a separate migration. This ensures that upon failure _only_ the - # index creation fails and can be retried or reverted easily. - # - # To disable transactions uncomment the following line and remove these - # comments: - # disable_ddl_transaction! - def change create_table :container_images do |t| t.integer :project_id diff --git a/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb b/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb index f89f9b00a5f..23d87cc6d0a 100644 --- a/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb +++ b/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb @@ -7,22 +7,6 @@ class AddContainerRegistryAccessTokenToApplicationSettings < ActiveRecord::Migra # Set this constant to true if this migration requires downtime. DOWNTIME = false - # When a migration requires downtime you **must** uncomment the following - # constant and define a short and easy to understand explanation as to why the - # migration requires downtime. - # DOWNTIME_REASON = '' - - # When using the methods "add_concurrent_index" or "add_column_with_default" - # you must disable the use of transactions as these methods can not run in an - # existing transaction. When using "add_concurrent_index" make sure that this - # method is the _only_ method called in the migration, any other changes - # should go in a separate migration. This ensures that upon failure _only_ the - # index creation fails and can be retried or reverted easily. - # - # To disable transactions uncomment the following line and remove these - # comments: - # disable_ddl_transaction! - def change add_column :application_settings, :container_registry_access_token, :string end diff --git a/db/schema.rb b/db/schema.rb index 36df20fc8f2..08d11546800 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -61,6 +61,7 @@ ActiveRecord::Schema.define(version: 20170215200045) do t.boolean "shared_runners_enabled", default: true, null: false t.integer "max_artifacts_size", default: 100, null: false t.string "runners_registration_token" + t.integer "max_pages_size", default: 100, null: false t.boolean "require_two_factor_authentication", default: false t.integer "two_factor_grace_period", default: 48 t.boolean "metrics_enabled", default: false @@ -107,10 +108,9 @@ ActiveRecord::Schema.define(version: 20170215200045) do t.string "sidekiq_throttling_queues" t.decimal "sidekiq_throttling_factor" t.boolean "html_emails_enabled", default: true + t.string "container_registry_access_token" t.string "plantuml_url" t.boolean "plantuml_enabled" - t.string "container_registry_access_token" - t.integer "max_pages_size", default: 100, null: false t.integer "terminal_max_session_time", default: 0, null: false end @@ -586,9 +586,9 @@ ActiveRecord::Schema.define(version: 20170215200045) do end add_index "labels", ["group_id", "project_id", "title"], name: "index_labels_on_group_id_and_project_id_and_title", unique: true, using: :btree - add_index "labels", ["type", "project_id"], name: "index_labels_on_type_and_project_id", using: :btree add_index "labels", ["project_id"], name: "index_labels_on_project_id", using: :btree add_index "labels", ["title"], name: "index_labels_on_title", using: :btree + add_index "labels", ["type", "project_id"], name: "index_labels_on_type_and_project_id", using: :btree create_table "lfs_objects", force: :cascade do |t| t.string "oid", null: false @@ -761,8 +761,8 @@ ActiveRecord::Schema.define(version: 20170215200045) do t.integer "visibility_level", default: 20, null: false t.boolean "request_access_enabled", default: false, null: false t.datetime "deleted_at" - t.boolean "lfs_enabled" t.text "description_html" + t.boolean "lfs_enabled" t.integer "parent_id" end @@ -1283,8 +1283,8 @@ ActiveRecord::Schema.define(version: 20170215200045) do t.datetime "otp_grace_period_started_at" t.boolean "ldap_email", default: false, null: false t.boolean "external", default: false - t.string "organization" t.string "incoming_email_token" + t.string "organization" t.boolean "authorized_projects_populated" t.boolean "notified_of_own_activity", default: false, null: false end diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb index 12305a49f0f..fc6fc0b97e0 100644 --- a/lib/api/registry_events.rb +++ b/lib/api/registry_events.rb @@ -39,9 +39,9 @@ module API params['events'].each do |event| repository = event['target']['repository'] - if event['action'] == 'push' and !!event['target']['tag'] + if event['action'] == 'push' && !!event['target']['tag'] namespace, container_image_name = ContainerImage::split_namespace(repository) - project = Project::find_with_namespace(namespace) + project = Project::find_by_full_path(namespace) if project container_image = project.container_images.find_or_create_by(name: container_image_name) diff --git a/spec/services/container_images/destroy_service_spec.rb b/spec/services/container_images/destroy_service_spec.rb new file mode 100644 index 00000000000..5b4dbaa7934 --- /dev/null +++ b/spec/services/container_images/destroy_service_spec.rb @@ -0,0 +1,34 @@ +require 'spec_helper' + +describe ContainerImages::DestroyService, services: true do + describe '#execute' do + let(:user) { create(:user) } + let(:container_image) { create(:container_image, name: '') } + let(:project) { create(:project, path: 'test', namespace: user.namespace, container_images: [container_image]) } + let(:example_host) { 'example.com' } + let(:registry_url) { 'http://' + example_host } + + it { expect(container_image).to be_valid } + it { expect(project.container_images).not_to be_empty } + + context 'when container image has tags' do + before do + project.team << [user, :master] + end + + it 'removes all tags before destroy' do + service = described_class.new(project, user) + + expect(container_image).to receive(:delete_tags).and_return(true) + expect { service.execute(container_image) }.to change(project.container_images, :count).by(-1) + end + + it 'fails when tags are not removed' do + service = described_class.new(project, user) + + expect(container_image).to receive(:delete_tags).and_return(false) + expect { service.execute(container_image) }.to raise_error(ActiveRecord::RecordNotDestroyed) + end + end + end +end From 53d332d3c73f8a883fa54d8eaaf91f92da73c33f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 21 Mar 2017 13:39:57 +0100 Subject: [PATCH 11/97] Add configured container registry key to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0b602d613c7..5e9f19d8319 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ eslint-report.html /config/unicorn.rb /config/secrets.yml /config/sidekiq.yml +/config/registry.key /coverage/* /coverage-javascript/ /db/*.sqlite3 From c64d36306cafac463f20d49e750f397a9b32960b Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Tue, 21 Mar 2017 10:35:02 -0300 Subject: [PATCH 12/97] Makes ContainerImages Routable Conflicts: db/schema.rb --- app/models/container_image.rb | 14 ++++++++++++-- .../container_registry_authentication_service.rb | 2 +- .../projects/container_registry/_image.html.haml | 3 +-- .../projects/container_registry/index.html.haml | 3 +-- .../20161031013926_create_container_image.rb | 1 + db/schema.rb | 3 ++- lib/api/registry_events.rb | 2 +- 7 files changed, 19 insertions(+), 9 deletions(-) diff --git a/app/models/container_image.rb b/app/models/container_image.rb index 583cb977910..a362ea3adbc 100644 --- a/app/models/container_image.rb +++ b/app/models/container_image.rb @@ -1,4 +1,6 @@ class ContainerImage < ActiveRecord::Base + include Routable + belongs_to :project delegate :container_registry, :container_registry_allowed_paths, @@ -17,10 +19,18 @@ class ContainerImage < ActiveRecord::Base client.update_token(token) end - def path - [container_registry.path, name_with_namespace].compact.join('/') + def parent + project end + def parent_changed? + project_id_changed? + end + + # def path + # [container_registry.path, name_with_namespace].compact.join('/') + # end + def name_with_namespace [container_registry_path_with_namespace, name].reject(&:blank?).join('/') end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 08fe6e3293e..a3c8d77bf09 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -66,7 +66,7 @@ module Auth # per image authentication. # Removes only last occurence in light # of future nested groups - namespace, _ = ContainerImage::split_namespace(name) + namespace, a = ContainerImage::split_namespace(name) requested_project = Project.find_by_full_path(namespace) return unless requested_project diff --git a/app/views/projects/container_registry/_image.html.haml b/app/views/projects/container_registry/_image.html.haml index 72f2103b862..4fd642a56c9 100644 --- a/app/views/projects/container_registry/_image.html.haml +++ b/app/views/projects/container_registry/_image.html.haml @@ -31,5 +31,4 @@ - if can?(current_user, :update_container_image, @project) %th - - image.tags.each do |tag| - = render 'tag', tag: tag + = render partial: 'tag', collection: image.tags diff --git a/app/views/projects/container_registry/index.html.haml b/app/views/projects/container_registry/index.html.haml index 5508a3de396..1b5d000e801 100644 --- a/app/views/projects/container_registry/index.html.haml +++ b/app/views/projects/container_registry/index.html.haml @@ -23,5 +23,4 @@ .nothing-here-block No container images in Container Registry for this project. - else - - @images.each do |image| - = render 'image', image: image + = render partial: 'image', collection: @images diff --git a/db/migrate/20161031013926_create_container_image.rb b/db/migrate/20161031013926_create_container_image.rb index 884c78880eb..06c409857da 100644 --- a/db/migrate/20161031013926_create_container_image.rb +++ b/db/migrate/20161031013926_create_container_image.rb @@ -11,6 +11,7 @@ class CreateContainerImage < ActiveRecord::Migration create_table :container_images do |t| t.integer :project_id t.string :name + t.string :path end end end diff --git a/db/schema.rb b/db/schema.rb index db57fb0a548..a1fc5dc1f58 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -108,7 +108,6 @@ ActiveRecord::Schema.define(version: 20170315194013) do t.string "sidekiq_throttling_queues" t.decimal "sidekiq_throttling_factor" t.boolean "html_emails_enabled", default: true - t.string "container_registry_access_token" t.string "plantuml_url" t.boolean "plantuml_enabled" t.integer "terminal_max_session_time", default: 0, null: false @@ -117,6 +116,7 @@ ActiveRecord::Schema.define(version: 20170315194013) do t.integer "unique_ips_limit_per_user" t.integer "unique_ips_limit_time_window" t.boolean "unique_ips_limit_enabled", default: false, null: false + t.string "container_registry_access_token" end create_table "audit_events", force: :cascade do |t| @@ -327,6 +327,7 @@ ActiveRecord::Schema.define(version: 20170315194013) do create_table "container_images", force: :cascade do |t| t.integer "project_id" t.string "name" + t.string "path" end create_table "deploy_keys_projects", force: :cascade do |t| diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb index fc6fc0b97e0..8c53e0fcfc0 100644 --- a/lib/api/registry_events.rb +++ b/lib/api/registry_events.rb @@ -44,7 +44,7 @@ module API project = Project::find_by_full_path(namespace) if project - container_image = project.container_images.find_or_create_by(name: container_image_name) + container_image = project.container_images.find_or_create_by(name: container_image_name, path: container_image_name) unless container_image.valid? render_api_error!({ error: "Failed to create container image!" }, 400) From 68a2fa54dedcdbe893ec811413d1703e5f6ac2dc Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 22 Mar 2017 11:08:23 +0100 Subject: [PATCH 13/97] Remove out-of-scope changes for multi-level images --- .../admin/application_settings_controller.rb | 6 -- .../admin/container_registry_controller.rb | 11 ---- app/models/application_setting.rb | 6 -- .../admin/container_registry/show.html.haml | 31 ---------- app/views/admin/dashboard/_head.html.haml | 4 -- config/routes/admin.rb | 2 - ...ry_access_token_to_application_settings.rb | 13 ---- doc/administration/container_registry.md | 18 ------ lib/api/api.rb | 1 - lib/api/helpers.rb | 10 ---- lib/api/registry_events.rb | 60 ------------------- lib/container_registry/ROADMAP.md | 7 --- 12 files changed, 169 deletions(-) delete mode 100644 app/controllers/admin/container_registry_controller.rb delete mode 100644 app/views/admin/container_registry/show.html.haml delete mode 100644 db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb delete mode 100644 lib/api/registry_events.rb delete mode 100644 lib/container_registry/ROADMAP.md diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb index 1d0bd6e0b81..8d831ffdd70 100644 --- a/app/controllers/admin/application_settings_controller.rb +++ b/app/controllers/admin/application_settings_controller.rb @@ -29,12 +29,6 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController redirect_to :back end - def reset_container_registry_token - @application_setting.reset_container_registry_access_token! - flash[:notice] = 'New container registry access token has been generated!' - redirect_to :back - end - def clear_repository_check_states RepositoryCheck::ClearWorker.perform_async diff --git a/app/controllers/admin/container_registry_controller.rb b/app/controllers/admin/container_registry_controller.rb deleted file mode 100644 index 265c032c67d..00000000000 --- a/app/controllers/admin/container_registry_controller.rb +++ /dev/null @@ -1,11 +0,0 @@ -class Admin::ContainerRegistryController < Admin::ApplicationController - def show - @access_token = container_registry_access_token - end - - private - - def container_registry_access_token - current_application_settings.container_registry_access_token - end -end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 9d01a70c77d..671a0fe98cc 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -4,7 +4,6 @@ class ApplicationSetting < ActiveRecord::Base add_authentication_token_field :runners_registration_token add_authentication_token_field :health_check_access_token - add_authentication_token_field :container_registry_access_token CACHE_KEY = 'application_setting.last'.freeze DOMAIN_LIST_SEPARATOR = %r{\s*[,;]\s* # comma or semicolon, optionally surrounded by whitespace @@ -158,7 +157,6 @@ class ApplicationSetting < ActiveRecord::Base before_save :ensure_runners_registration_token before_save :ensure_health_check_access_token - before_save :ensure_container_registry_access_token after_commit do Rails.cache.write(CACHE_KEY, self) @@ -332,10 +330,6 @@ class ApplicationSetting < ActiveRecord::Base ensure_health_check_access_token! end - def container_registry_access_token - ensure_container_registry_access_token! - end - def sidekiq_throttling_enabled? return false unless sidekiq_throttling_column_exists? diff --git a/app/views/admin/container_registry/show.html.haml b/app/views/admin/container_registry/show.html.haml deleted file mode 100644 index ffaa7736d65..00000000000 --- a/app/views/admin/container_registry/show.html.haml +++ /dev/null @@ -1,31 +0,0 @@ -- @no_container = true -= render "admin/dashboard/head" - -%div{ class: container_class } - - %p.prepend-top-default - %span - To properly configure the Container Registry you should add the following - access token to the Docker Registry config.yml as follows: - %pre - %code - :plain - notifications: - endpoints: - - ... - headers: - X-Registry-Token: [#{@access_token}] - %br - Access token is - %code{ id: 'registry-token' }= @access_token - - .bs-callout.clearfix - .pull-left - %p - You can reset container registry access token by pressing the button below. - %p - = button_to reset_container_registry_token_admin_application_settings_path, - method: :put, class: 'btn btn-default', - data: { confirm: 'Are you sure you want to reset container registry token?' } do - = icon('refresh') - Reset container registry access token diff --git a/app/views/admin/dashboard/_head.html.haml b/app/views/admin/dashboard/_head.html.haml index dbd039547fa..7893c1dee97 100644 --- a/app/views/admin/dashboard/_head.html.haml +++ b/app/views/admin/dashboard/_head.html.haml @@ -27,7 +27,3 @@ = link_to admin_runners_path, title: 'Runners' do %span Runners - = nav_link path: 'container_registry#show' do - = link_to admin_container_registry_path, title: 'Registry' do - %span - Registry diff --git a/config/routes/admin.rb b/config/routes/admin.rb index fcbe2e2c435..486ce3c5c87 100644 --- a/config/routes/admin.rb +++ b/config/routes/admin.rb @@ -63,7 +63,6 @@ namespace :admin do resource :background_jobs, controller: 'background_jobs', only: [:show] resource :system_info, controller: 'system_info', only: [:show] resources :requests_profiles, only: [:index, :show], param: :name, constraints: { name: /.+\.html/ } - resource :container_registry, controller: 'container_registry', only: [:show] resources :projects, only: [:index] @@ -94,7 +93,6 @@ namespace :admin do resources :services, only: [:index, :edit, :update] put :reset_runners_token put :reset_health_check_token - put :reset_container_registry_token put :clear_repository_check_states end diff --git a/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb b/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb deleted file mode 100644 index 23d87cc6d0a..00000000000 --- a/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb +++ /dev/null @@ -1,13 +0,0 @@ -# See http://doc.gitlab.com/ce/development/migration_style_guide.html -# for more information on how to write migrations for GitLab. - -class AddContainerRegistryAccessTokenToApplicationSettings < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - # Set this constant to true if this migration requires downtime. - DOWNTIME = false - - def change - add_column :application_settings, :container_registry_access_token, :string - end -end diff --git a/doc/administration/container_registry.md b/doc/administration/container_registry.md index dc4e57f25fb..f707039827b 100644 --- a/doc/administration/container_registry.md +++ b/doc/administration/container_registry.md @@ -87,23 +87,6 @@ auth: rootcertbundle: /root/certs/certbundle ``` -Also a notification endpoint must be configured with the token from -Admin Area -> Overview -> Registry (`/admin/container_registry`) like in the following sample: - -``` -notifications: - endpoints: - - name: listener - url: https://gitlab.example.com/api/v3/registry_events - headers: - X-Registry-Token: [57Cx95fc2zHFh93VTiGD] - timeout: 500ms - threshold: 5 - backoff: 1s -``` - -Check the [Registry endpoint configuration][registry-endpoint] for details. - ## Container Registry domain configuration There are two ways you can configure the Registry's external domain. @@ -600,7 +583,6 @@ notifications: [storage-config]: https://docs.docker.com/registry/configuration/#storage [registry-http-config]: https://docs.docker.com/registry/configuration/#http [registry-auth]: https://docs.docker.com/registry/configuration/#auth -[registry-endpoint]: https://docs.docker.com/registry/notifications/#/configuration [token-config]: https://docs.docker.com/registry/configuration/#token [8-8-docs]: https://gitlab.com/gitlab-org/gitlab-ce/blob/8-8-stable/doc/administration/container_registry.md [registry-ssl]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/registry-ssl diff --git a/lib/api/api.rb b/lib/api/api.rb index 7c7bfada7d0..1bf20f76ad6 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -104,7 +104,6 @@ module API mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings - mount ::API::RegistryEvents mount ::API::Pipelines mount ::API::ProjectHooks mount ::API::Projects diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 3c173b544aa..bd22b82476b 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -111,16 +111,6 @@ module API end end - def authenticate_container_registry_access_token! - token = request.headers['X-Registry-Token'] - unless token.present? && ActiveSupport::SecurityUtils.variable_size_secure_compare( - token, - current_application_settings.container_registry_access_token - ) - unauthorized! - end - end - def authenticated_as_admin! authenticate! forbidden! unless current_user.is_admin? diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb deleted file mode 100644 index 8c53e0fcfc0..00000000000 --- a/lib/api/registry_events.rb +++ /dev/null @@ -1,60 +0,0 @@ -module API - # RegistryEvents API - class RegistryEvents < Grape::API - before { authenticate_container_registry_access_token! } - - content_type :json, 'application/vnd.docker.distribution.events.v1+json' - - params do - requires :events, type: Array, desc: 'The ID of a project' do - requires :id, type: String, desc: 'The ID of the event' - requires :timestamp, type: String, desc: 'Timestamp of the event' - requires :action, type: String, desc: 'Action performed by event' - requires :target, type: Hash, desc: 'Target of the event' do - optional :mediaType, type: String, desc: 'Media type of the target' - optional :size, type: Integer, desc: 'Size in bytes of the target' - requires :digest, type: String, desc: 'Digest of the target' - requires :repository, type: String, desc: 'Repository of target' - optional :url, type: String, desc: 'Url of the target' - optional :tag, type: String, desc: 'Tag of the target' - end - requires :request, type: Hash, desc: 'Request of the event' do - requires :id, type: String, desc: 'The ID of the request' - optional :addr, type: String, desc: 'IP Address of the request client' - optional :host, type: String, desc: 'Hostname of the registry instance' - requires :method, type: String, desc: 'Request method' - requires :useragent, type: String, desc: 'UserAgent header of the request' - end - requires :actor, type: Hash, desc: 'Actor that initiated the event' do - optional :name, type: String, desc: 'Actor name' - end - requires :source, type: Hash, desc: 'Source of the event' do - optional :addr, type: String, desc: 'Hostname of source registry node' - optional :instanceID, type: String, desc: 'Source registry node instanceID' - end - end - end - resource :registry_events do - post do - params['events'].each do |event| - repository = event['target']['repository'] - - if event['action'] == 'push' && !!event['target']['tag'] - namespace, container_image_name = ContainerImage::split_namespace(repository) - project = Project::find_by_full_path(namespace) - - if project - container_image = project.container_images.find_or_create_by(name: container_image_name, path: container_image_name) - - unless container_image.valid? - render_api_error!({ error: "Failed to create container image!" }, 400) - end - else - not_found!('Project') - end - end - end - end - end - end -end diff --git a/lib/container_registry/ROADMAP.md b/lib/container_registry/ROADMAP.md deleted file mode 100644 index e0a20776404..00000000000 --- a/lib/container_registry/ROADMAP.md +++ /dev/null @@ -1,7 +0,0 @@ -## Road map - -### Initial thoughts - -- Determine if image names will be persisted or fetched from API -- If persisted, how to update the stored names upon modification -- If fetched, how to fetch only images of a given project From 29c34267556198ee3dbbe2f13bc81708f5e60f10 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 22 Mar 2017 11:41:05 +0100 Subject: [PATCH 14/97] Move container images migration to the present time --- ...iner_image.rb => 20170322013926_create_container_image.rb} | 4 ---- 1 file changed, 4 deletions(-) rename db/migrate/{20161031013926_create_container_image.rb => 20170322013926_create_container_image.rb} (56%) diff --git a/db/migrate/20161031013926_create_container_image.rb b/db/migrate/20170322013926_create_container_image.rb similarity index 56% rename from db/migrate/20161031013926_create_container_image.rb rename to db/migrate/20170322013926_create_container_image.rb index 06c409857da..c494f2a56c7 100644 --- a/db/migrate/20161031013926_create_container_image.rb +++ b/db/migrate/20170322013926_create_container_image.rb @@ -1,10 +1,6 @@ -# See http://doc.gitlab.com/ce/development/migration_style_guide.html -# for more information on how to write migrations for GitLab. - class CreateContainerImage < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers - # Set this constant to true if this migration requires downtime. DOWNTIME = false def change From 95e2c0196b7e492f8c03c6cfeb6b37e97f75813e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 22 Mar 2017 12:28:23 +0100 Subject: [PATCH 15/97] Clean code related to accessing registry from project [ci skip] --- app/models/container_image.rb | 28 ++++------------------------ app/models/project.rb | 16 +++++----------- 2 files changed, 9 insertions(+), 35 deletions(-) diff --git a/app/models/container_image.rb b/app/models/container_image.rb index a362ea3adbc..411617ccd71 100644 --- a/app/models/container_image.rb +++ b/app/models/container_image.rb @@ -3,36 +3,16 @@ class ContainerImage < ActiveRecord::Base belongs_to :project - delegate :container_registry, :container_registry_allowed_paths, - :container_registry_path_with_namespace, to: :project - + delegate :container_registry, to: :project delegate :client, to: :container_registry validates :manifest, presence: true before_destroy :delete_tags - before_validation :update_token, on: :create - def update_token - paths = container_registry_allowed_paths << name_with_namespace - token = Auth::ContainerRegistryAuthenticationService.full_access_token(paths) - client.update_token(token) - end - - def parent - project - end - - def parent_changed? - project_id_changed? - end - - # def path - # [container_registry.path, name_with_namespace].compact.join('/') - # end - - def name_with_namespace - [container_registry_path_with_namespace, name].reject(&:blank?).join('/') + def registry + # TODO, container registry with image access level + token = Auth::ContainerRegistryAuthenticationService.image_token(self) end def tag(tag) diff --git a/app/models/project.rb b/app/models/project.rb index 928965643a0..4aa9c6bb2f2 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -405,29 +405,23 @@ class Project < ActiveRecord::Base @repository ||= Repository.new(path_with_namespace, self) end - def container_registry_path_with_namespace - path_with_namespace.downcase - end - - def container_registry_allowed_paths - @container_registry_allowed_paths ||= [container_registry_path_with_namespace] + - container_images.map { |i| i.name_with_namespace } - end - def container_registry return unless Gitlab.config.registry.enabled @container_registry ||= begin - token = Auth::ContainerRegistryAuthenticationService.full_access_token(container_registry_allowed_paths) + token = Auth::ContainerRegistryAuthenticationService.full_access_token(project) + url = Gitlab.config.registry.api_url host_port = Gitlab.config.registry.host_port + # TODO, move configuration vars into ContainerRegistry::Registry, clean + # this method up afterwards ContainerRegistry::Registry.new(url, token: token, path: host_port) end end def container_registry_url if Gitlab.config.registry.enabled - "#{Gitlab.config.registry.host_port}/#{container_registry_path_with_namespace}" + "#{Gitlab.config.registry.host_port}/#{path_with_namespace.downcase}" end end From 896b13b929369c02f72fa881eda24ca4a6a0d900 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 22 Mar 2017 16:07:27 +0100 Subject: [PATCH 16/97] Refactor splitting container image full path [ci skip] --- app/models/container_image.rb | 17 +++++++---------- ...container_registry_authentication_service.rb | 7 +------ 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/app/models/container_image.rb b/app/models/container_image.rb index 411617ccd71..6e9a060d7a8 100644 --- a/app/models/container_image.rb +++ b/app/models/container_image.rb @@ -1,6 +1,4 @@ class ContainerImage < ActiveRecord::Base - include Routable - belongs_to :project delegate :container_registry, to: :project @@ -45,14 +43,13 @@ class ContainerImage < ActiveRecord::Base end end - # rubocop:disable RedundantReturn + def self.from_path(full_path) + return unless full_path.include?('/') - def self.split_namespace(full_path) - image_name = full_path.split('/').last - namespace = full_path.gsub(/(.*)(#{Regexp.escape('/' + image_name)})/, '\1') - if namespace.count('/') < 1 - namespace, image_name = full_path, "" - end - return namespace, image_name + path = full_path[0...full_path.rindex('/')] + name = full_path[full_path.rindex('/')+1..-1] + project = Project.find_by_full_path(path) + + self.new(name: name, path: path, project: project) end end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index a3c8d77bf09..7e412040c7c 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -62,12 +62,7 @@ module Auth end def process_repository_access(type, name, actions) - # Strips image name due to lack of - # per image authentication. - # Removes only last occurence in light - # of future nested groups - namespace, a = ContainerImage::split_namespace(name) - requested_project = Project.find_by_full_path(namespace) + requested_project = ContainerImage.from_path(name).project return unless requested_project actions = actions.select do |action| From 4005eb643657e5ee8b1f328e36a3204253e3acf4 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 11:41:16 +0100 Subject: [PATCH 17/97] Fix communication between GitLab and Container Registry --- app/models/container_image.rb | 19 +++++++++++++------ ...ntainer_registry_authentication_service.rb | 17 +++++++++-------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/app/models/container_image.rb b/app/models/container_image.rb index 6e9a060d7a8..434302159b0 100644 --- a/app/models/container_image.rb +++ b/app/models/container_image.rb @@ -43,13 +43,20 @@ class ContainerImage < ActiveRecord::Base end end - def self.from_path(full_path) - return unless full_path.include?('/') + def self.project_from_path(image_path) + return unless image_path.include?('/') - path = full_path[0...full_path.rindex('/')] - name = full_path[full_path.rindex('/')+1..-1] - project = Project.find_by_full_path(path) + ## + # Projects are always located inside a namespace, so we can remove + # the last node, and see if project with that path exists. + # + truncated_path = image_path.slice(0...image_path.rindex('/')) - self.new(name: name, path: path, project: project) + ## + # We still make it possible to search projects by a full image path + # in order to maintain backwards compatibility. + # + Project.find_by_full_path(truncated_path) || + Project.find_by_full_path(image_path) end end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 7e412040c7c..2205b0897e2 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -38,13 +38,13 @@ module Auth private def authorized_token(*accesses) - token = JSONWebToken::RSAToken.new(registry.key) - token.issuer = registry.issuer - token.audience = params[:service] - token.subject = current_user.try(:username) - token.expire_time = self.class.token_expire_at - token[:access] = accesses.compact - token + JSONWebToken::RSAToken.new(registry.key).tap do |token| + token.issuer = registry.issuer + token.audience = params[:service] + token.subject = current_user.try(:username) + token.expire_time = self.class.token_expire_at + token[:access] = accesses.compact + end end def scope @@ -62,7 +62,8 @@ module Auth end def process_repository_access(type, name, actions) - requested_project = ContainerImage.from_path(name).project + requested_project = ContainerImage.project_from_path(name) + return unless requested_project actions = actions.select do |action| From bd8c8df6ed8dd7321608ce652e23d86ef3bd6899 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 12:01:12 +0100 Subject: [PATCH 18/97] Fix database schema --- db/schema.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index a1fc5dc1f58..b5bd6fc3121 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170315194013) do +ActiveRecord::Schema.define(version: 20170322013926) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -111,12 +111,10 @@ ActiveRecord::Schema.define(version: 20170315194013) do t.string "plantuml_url" t.boolean "plantuml_enabled" t.integer "terminal_max_session_time", default: 0, null: false - t.integer "max_pages_size", default: 100, null: false t.string "default_artifacts_expire_in", default: "0", null: false t.integer "unique_ips_limit_per_user" t.integer "unique_ips_limit_time_window" t.boolean "unique_ips_limit_enabled", default: false, null: false - t.string "container_registry_access_token" end create_table "audit_events", force: :cascade do |t| @@ -993,6 +991,7 @@ ActiveRecord::Schema.define(version: 20170315194013) do end add_index "routes", ["path"], name: "index_routes_on_path", unique: true, using: :btree + add_index "routes", ["path"], name: "index_routes_on_path_text_pattern_ops", using: :btree, opclasses: {"path"=>"varchar_pattern_ops"} add_index "routes", ["source_type", "source_id"], name: "index_routes_on_source_type_and_source_id", unique: true, using: :btree create_table "sent_notifications", force: :cascade do |t| From 01d159b409d8b24d36204979a73de249843d71bf Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 14:00:41 +0100 Subject: [PATCH 19/97] Rename container image model to container repository --- ...{container_image.rb => container_repository.rb} | 14 +++++++------- .../container_registry_authentication_service.rb | 2 +- ... 20170322013926_create_container_repository.rb} | 5 ++--- db/schema.rb | 10 ++++------ 4 files changed, 14 insertions(+), 17 deletions(-) rename app/models/{container_image.rb => container_repository.rb} (75%) rename db/migrate/{20170322013926_create_container_image.rb => 20170322013926_create_container_repository.rb} (55%) diff --git a/app/models/container_image.rb b/app/models/container_repository.rb similarity index 75% rename from app/models/container_image.rb rename to app/models/container_repository.rb index 434302159b0..2e78fc148b4 100644 --- a/app/models/container_image.rb +++ b/app/models/container_repository.rb @@ -1,4 +1,4 @@ -class ContainerImage < ActiveRecord::Base +class ContainerRepository < ActiveRecord::Base belongs_to :project delegate :container_registry, to: :project @@ -18,7 +18,7 @@ class ContainerImage < ActiveRecord::Base end def manifest - @manifest ||= client.repository_tags(name_with_namespace) + @manifest ||= client.repository_tags(self.path) end def tags @@ -39,24 +39,24 @@ class ContainerImage < ActiveRecord::Base digests = tags.map {|tag| tag.digest }.to_set digests.all? do |digest| - client.delete_repository_tag(name_with_namespace, digest) + client.delete_repository_tag(self.path, digest) end end - def self.project_from_path(image_path) - return unless image_path.include?('/') + def self.project_from_path(repository_path) + return unless repository_path.include?('/') ## # Projects are always located inside a namespace, so we can remove # the last node, and see if project with that path exists. # - truncated_path = image_path.slice(0...image_path.rindex('/')) + truncated_path = repository_path.slice(0...repository_path.rindex('/')) ## # We still make it possible to search projects by a full image path # in order to maintain backwards compatibility. # Project.find_by_full_path(truncated_path) || - Project.find_by_full_path(image_path) + Project.find_by_full_path(repository_path) end end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 2205b0897e2..3d151c6a357 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -62,7 +62,7 @@ module Auth end def process_repository_access(type, name, actions) - requested_project = ContainerImage.project_from_path(name) + requested_project = ContainerRepository.project_from_path(name) return unless requested_project diff --git a/db/migrate/20170322013926_create_container_image.rb b/db/migrate/20170322013926_create_container_repository.rb similarity index 55% rename from db/migrate/20170322013926_create_container_image.rb rename to db/migrate/20170322013926_create_container_repository.rb index c494f2a56c7..0235fd7e096 100644 --- a/db/migrate/20170322013926_create_container_image.rb +++ b/db/migrate/20170322013926_create_container_repository.rb @@ -1,12 +1,11 @@ -class CreateContainerImage < ActiveRecord::Migration +class CreateContainerRepository < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers DOWNTIME = false def change - create_table :container_images do |t| + create_table :container_repositories do |t| t.integer :project_id - t.string :name t.string :path end end diff --git a/db/schema.rb b/db/schema.rb index b5bd6fc3121..be7604a14d5 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -61,7 +61,6 @@ ActiveRecord::Schema.define(version: 20170322013926) do t.boolean "shared_runners_enabled", default: true, null: false t.integer "max_artifacts_size", default: 100, null: false t.string "runners_registration_token" - t.integer "max_pages_size", default: 100, null: false t.boolean "require_two_factor_authentication", default: false t.integer "two_factor_grace_period", default: 48 t.boolean "metrics_enabled", default: false @@ -111,6 +110,7 @@ ActiveRecord::Schema.define(version: 20170322013926) do t.string "plantuml_url" t.boolean "plantuml_enabled" t.integer "terminal_max_session_time", default: 0, null: false + t.integer "max_pages_size", default: 100, null: false t.string "default_artifacts_expire_in", default: "0", null: false t.integer "unique_ips_limit_per_user" t.integer "unique_ips_limit_time_window" @@ -322,9 +322,8 @@ ActiveRecord::Schema.define(version: 20170322013926) do add_index "ci_variables", ["project_id"], name: "index_ci_variables_on_project_id", using: :btree - create_table "container_images", force: :cascade do |t| + create_table "container_repositories", force: :cascade do |t| t.integer "project_id" - t.string "name" t.string "path" end @@ -694,8 +693,8 @@ ActiveRecord::Schema.define(version: 20170322013926) do t.integer "visibility_level", default: 20, null: false t.boolean "request_access_enabled", default: false, null: false t.datetime "deleted_at" - t.text "description_html" t.boolean "lfs_enabled" + t.text "description_html" t.integer "parent_id" end @@ -991,7 +990,6 @@ ActiveRecord::Schema.define(version: 20170322013926) do end add_index "routes", ["path"], name: "index_routes_on_path", unique: true, using: :btree - add_index "routes", ["path"], name: "index_routes_on_path_text_pattern_ops", using: :btree, opclasses: {"path"=>"varchar_pattern_ops"} add_index "routes", ["source_type", "source_id"], name: "index_routes_on_source_type_and_source_id", unique: true, using: :btree create_table "sent_notifications", force: :cascade do |t| @@ -1238,8 +1236,8 @@ ActiveRecord::Schema.define(version: 20170322013926) do t.datetime "otp_grace_period_started_at" t.boolean "ldap_email", default: false, null: false t.boolean "external", default: false - t.string "incoming_email_token" t.string "organization" + t.string "incoming_email_token" t.boolean "authorized_projects_populated" t.boolean "ghost" end From ea16ea5bfcb78f66c6bb37e470d387bf1ac26c9f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 14:09:01 +0100 Subject: [PATCH 20/97] Identify container repository by a single name --- db/migrate/20170322013926_create_container_repository.rb | 2 +- db/schema.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db/migrate/20170322013926_create_container_repository.rb b/db/migrate/20170322013926_create_container_repository.rb index 0235fd7e096..87a1523724c 100644 --- a/db/migrate/20170322013926_create_container_repository.rb +++ b/db/migrate/20170322013926_create_container_repository.rb @@ -6,7 +6,7 @@ class CreateContainerRepository < ActiveRecord::Migration def change create_table :container_repositories do |t| t.integer :project_id - t.string :path + t.string :name end end end diff --git a/db/schema.rb b/db/schema.rb index be7604a14d5..5d9b219ebea 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -324,7 +324,7 @@ ActiveRecord::Schema.define(version: 20170322013926) do create_table "container_repositories", force: :cascade do |t| t.integer "project_id" - t.string "path" + t.string "name" end create_table "deploy_keys_projects", force: :cascade do |t| From f451173a191474be681d208eceb6a0148ba2c0d0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 14:37:17 +0100 Subject: [PATCH 21/97] Fix specs for container repository model class --- app/models/container_repository.rb | 21 ++++-- spec/factories/container_images.rb | 22 ------ spec/factories/container_repositories.rb | 22 ++++++ spec/models/container_image_spec.rb | 73 ------------------ spec/models/container_repository_spec.rb | 96 ++++++++++++++++++++++++ 5 files changed, 132 insertions(+), 102 deletions(-) delete mode 100644 spec/factories/container_images.rb create mode 100644 spec/factories/container_repositories.rb delete mode 100644 spec/models/container_image_spec.rb create mode 100644 spec/models/container_repository_spec.rb diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 2e78fc148b4..b3a8ec691de 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -1,16 +1,23 @@ class ContainerRepository < ActiveRecord::Base belongs_to :project - - delegate :container_registry, to: :project - delegate :client, to: :container_registry - + delegate :client, to: :registry validates :manifest, presence: true - + validates :name, presence: true before_destroy :delete_tags def registry - # TODO, container registry with image access level - token = Auth::ContainerRegistryAuthenticationService.image_token(self) + @registry ||= begin + token = Auth::ContainerRegistryAuthenticationService.full_access_token(path) + + url = Gitlab.config.registry.api_url + host_port = Gitlab.config.registry.host_port + + ContainerRegistry::Registry.new(url, token: token, path: host_port) + end + end + + def path + @path ||= "#{project.full_path}/#{name}" end def tag(tag) diff --git a/spec/factories/container_images.rb b/spec/factories/container_images.rb deleted file mode 100644 index 3693865101d..00000000000 --- a/spec/factories/container_images.rb +++ /dev/null @@ -1,22 +0,0 @@ -FactoryGirl.define do - factory :container_image do - name "test_container_image" - project - - transient do - tags ['tag'] - stubbed true - end - - after(:build) do |image, evaluator| - if evaluator.stubbed - allow(Gitlab.config.registry).to receive(:enabled).and_return(true) - allow(Auth::ContainerRegistryAuthenticationService).to receive(:full_access_token).and_return('token') - allow(image.client).to receive(:repository_tags).and_return({ - name: image.name_with_namespace, - tags: evaluator.tags - }) - end - end - end -end diff --git a/spec/factories/container_repositories.rb b/spec/factories/container_repositories.rb new file mode 100644 index 00000000000..fbf6bf62dfd --- /dev/null +++ b/spec/factories/container_repositories.rb @@ -0,0 +1,22 @@ +FactoryGirl.define do + factory :container_repository do + name "test_container_image" + project + + transient do + tags ['tag'] + end + + after(:build) do |image, evaluator| + # if evaluator.tags.to_a.any? + # allow(Gitlab.config.registry).to receive(:enabled).and_return(true) + # allow(Auth::ContainerRegistryAuthenticationService) + # .to receive(:full_access_token).and_return('token') + # allow(image.client).to receive(:repository_tags).and_return({ + # name: image.name_with_namespace, + # tags: evaluator.tags + # }) + # end + end + end +end diff --git a/spec/models/container_image_spec.rb b/spec/models/container_image_spec.rb deleted file mode 100644 index e0bea737f59..00000000000 --- a/spec/models/container_image_spec.rb +++ /dev/null @@ -1,73 +0,0 @@ -require 'spec_helper' - -describe ContainerImage do - let(:group) { create(:group, name: 'group') } - let(:project) { create(:project, path: 'test', group: group) } - let(:example_host) { 'example.com' } - let(:registry_url) { 'http://' + example_host } - let(:container_image) { create(:container_image, name: '', project: project, stubbed: false) } - - before do - stub_container_registry_config(enabled: true, api_url: registry_url, host_port: example_host) - stub_request(:get, 'http://example.com/v2/group/test/tags/list'). - with(headers: { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' }). - to_return( - status: 200, - body: JSON.dump(tags: ['test']), - headers: { 'Content-Type' => 'application/json' }) - end - - it { expect(container_image).to respond_to(:project) } - it { expect(container_image).to delegate_method(:container_registry).to(:project) } - it { expect(container_image).to delegate_method(:client).to(:container_registry) } - it { expect(container_image.tag('test')).not_to be_nil } - - context '#path' do - subject { container_image.path } - - it { is_expected.to eq('example.com/group/test') } - end - - context 'manifest processing' do - context '#manifest' do - subject { container_image.manifest } - - it { is_expected.not_to be_nil } - end - - context '#valid?' do - subject { container_image.valid? } - - it { is_expected.to be_truthy } - end - - context '#tags' do - subject { container_image.tags } - - it { is_expected.not_to be_empty } - end - end - - context '#delete_tags' do - let(:tag) { ContainerRegistry::Tag.new(container_image, 'tag') } - - before do - expect(container_image).to receive(:tags).twice.and_return([tag]) - expect(tag).to receive(:digest).and_return('sha256:4c8e63ca4cb663ce6c688cb06f1c3672a172b088dac5b6d7ad7d49cd620d85cf') - end - - subject { container_image.delete_tags } - - context 'succeeds' do - before { expect(container_image.client).to receive(:delete_repository_tag).and_return(true) } - - it { is_expected.to be_truthy } - end - - context 'any fails' do - before { expect(container_image.client).to receive(:delete_repository_tag).and_return(false) } - - it { is_expected.to be_falsey } - end - end -end diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb new file mode 100644 index 00000000000..3997c4ca682 --- /dev/null +++ b/spec/models/container_repository_spec.rb @@ -0,0 +1,96 @@ +require 'spec_helper' + +describe ContainerRepository do + let(:group) { create(:group, name: 'group') } + let(:project) { create(:project, path: 'test', group: group) } + + let(:container_repository) do + create(:container_repository, name: 'my_image', project: project) + end + + before do + stub_container_registry_config(enabled: true, + api_url: 'http://registry.gitlab', + host_port: 'registry.gitlab') + + stub_request(:get, 'http://registry.gitlab/v2/group/test/my_image/tags/list') + .with(headers: { + 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' }) + .to_return( + status: 200, + body: JSON.dump(tags: ['test_tag']), + headers: { 'Content-Type' => 'application/json' }) + end + + describe 'associations' do + it 'belongs to the project' do + expect(container_repository).to belong_to(:project) + end + end + + describe '#tag' do + it 'has a test tag' do + expect(container_repository.tag('test')).not_to be_nil + end + end + + describe '#path' do + it 'returns a full path to the repository' do + expect(container_repository.path).to eq('group/test/my_image') + end + end + + describe '#manifest' do + subject { container_repository.manifest } + + it { is_expected.not_to be_nil } + end + + describe '#valid?' do + subject { container_repository.valid? } + + it { is_expected.to be_truthy } + end + + describe '#tags' do + subject { container_repository.tags } + + it { is_expected.not_to be_empty } + end + + # TODO, improve these specs + # + describe '#delete_tags' do + let(:tag) { ContainerRegistry::Tag.new(container_repository, 'tag') } + + before do + allow(container_repository).to receive(:tags).twice.and_return([tag]) + allow(tag).to receive(:digest) + .and_return('sha256:4c8e63ca4cb663ce6c688cb06f1c3672a172b088dac5b6d7ad7d49cd620d85cf') + end + + context 'when action succeeds' do + before do + allow(container_repository.client) + .to receive(:delete_repository_tag) + .and_return(true) + end + + it 'returns status that indicates success' do + expect(container_repository.delete_tags).to be_truthy + end + end + + context 'when action fails' do + before do + allow(container_repository.client) + .to receive(:delete_repository_tag) + .and_return(false) + end + + it 'returns status that indicates failure' do + expect(container_repository.delete_tags).to be_falsey + end + end + end +end From 249084b48a86a99c02eefe45b507ebcdf811c20f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 14:48:24 +0100 Subject: [PATCH 22/97] Fix some specs using the old ContainerImage const --- spec/features/container_registry_spec.rb | 2 +- spec/services/projects/destroy_service_spec.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index 862c9fbf6c0..0199d08ab63 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -39,7 +39,7 @@ describe "Container Registry" do end it do - expect_any_instance_of(ContainerImage).to receive(:delete_tags).and_return(true) + expect_any_instance_of(ContainerRepository).to receive(:delete_tags).and_return(true) click_on 'Remove image' end diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index 270e630e70e..f91d62ebdaf 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -100,7 +100,7 @@ describe Projects::DestroyService, services: true do context 'images deletion succeeds' do it do - expect_any_instance_of(ContainerImage).to receive(:delete_tags).and_return(true) + expect_any_instance_of(ContainerRepository).to receive(:delete_tags).and_return(true) destroy_project(project, user, {}) end @@ -108,7 +108,7 @@ describe Projects::DestroyService, services: true do context 'images deletion fails' do before do - expect_any_instance_of(ContainerImage).to receive(:delete_tags).and_return(false) + expect_any_instance_of(ContainerRepository).to receive(:delete_tags).and_return(false) end subject { destroy_project(project, user, {}) } From 3e01fed5cb36962065f5d19ab6a0cef1dfc14b48 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 14:57:33 +0100 Subject: [PATCH 23/97] Fix Rubocop offenses in container repository code --- app/models/container_repository.rb | 2 +- spec/models/container_repository_spec.rb | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index b3a8ec691de..2f0fd3014a8 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -64,6 +64,6 @@ class ContainerRepository < ActiveRecord::Base # in order to maintain backwards compatibility. # Project.find_by_full_path(truncated_path) || - Project.find_by_full_path(repository_path) + Project.find_by_full_path(repository_path) end end diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index 3997c4ca682..e3180e01758 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -14,12 +14,11 @@ describe ContainerRepository do host_port: 'registry.gitlab') stub_request(:get, 'http://registry.gitlab/v2/group/test/my_image/tags/list') - .with(headers: { - 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' }) + .with(headers: { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' }) .to_return( - status: 200, - body: JSON.dump(tags: ['test_tag']), - headers: { 'Content-Type' => 'application/json' }) + status: 200, + body: JSON.dump(tags: ['test_tag']), + headers: { 'Content-Type' => 'application/json' }) end describe 'associations' do From dcd2eeb1cfb633f4a28ddd9bc79deac0e3171d3f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 15:54:59 +0100 Subject: [PATCH 24/97] Rename container image to repository in specs --- app/models/namespace.rb | 2 +- app/models/project.rb | 4 ++-- app/services/projects/transfer_service.rb | 2 +- spec/features/container_registry_spec.rb | 8 ++++---- .../security/project/internal_access_spec.rb | 4 ++-- .../security/project/private_access_spec.rb | 4 ++-- .../security/project/public_access_spec.rb | 4 ++-- spec/lib/container_registry/blob_spec.rb | 2 +- spec/lib/container_registry/tag_spec.rb | 2 +- spec/lib/gitlab/import_export/all_models.yml | 2 +- spec/models/namespace_spec.rb | 4 ++-- spec/models/project_spec.rb | 4 ++-- .../container_images/destroy_service_spec.rb | 16 ++++++++-------- spec/services/projects/destroy_service_spec.rb | 4 ++-- spec/services/projects/transfer_service_spec.rb | 4 ++-- 15 files changed, 33 insertions(+), 33 deletions(-) diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 4ae9d0122f2..ac03f098908 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -150,7 +150,7 @@ class Namespace < ActiveRecord::Base end def any_project_has_container_registry_images? - projects.joins(:container_images).any? + projects.joins(:container_repositories).any? end def send_update_instructions diff --git a/app/models/project.rb b/app/models/project.rb index 4aa9c6bb2f2..5c6672c95b3 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -157,7 +157,7 @@ class Project < ActiveRecord::Base has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" has_one :project_feature, dependent: :destroy has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete - has_many :container_images, dependent: :destroy + has_many :container_repositories, dependent: :destroy has_many :commit_statuses, dependent: :destroy has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline' @@ -908,7 +908,7 @@ class Project < ActiveRecord::Base expire_caches_before_rename(old_path_with_namespace) - if container_images.present? + if container_repositories.present? Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry images are present" # we currently doesn't support renaming repository if it contains images in container registry diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 6d9e7de4f24..f46bf884e37 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -36,7 +36,7 @@ module Projects raise TransferError.new("Project with same path in target namespace already exists") end - unless project.container_images.empty? + unless project.container_repositories.empty? # we currently doesn't support renaming repository if it contains images in container registry raise TransferError.new('Project cannot be transferred, because images are present in its container registry') end diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index 0199d08ab63..88642f72772 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -5,15 +5,15 @@ describe "Container Registry" do let(:registry) { project.container_registry } let(:tag_name) { 'latest' } let(:tags) { [tag_name] } - let(:container_image) { create(:container_image) } - let(:image_name) { container_image.name } + let(:container_repository) { create(:container_repository) } + let(:image_name) { container_repository.name } before do login_as(:user) project.team << [@user, :developer] stub_container_registry_config(enabled: true) stub_container_registry_tags(*tags) - project.container_images << container_image unless container_image.nil? + project.container_repositories << container_repository unless container_repository.nil? allow(Auth::ContainerRegistryAuthenticationService).to receive(:full_access_token).and_return('token') end @@ -23,7 +23,7 @@ describe "Container Registry" do end context 'when no images' do - let(:container_image) { } + let(:container_repository) { } it { expect(page).to have_content('No container images in Container Registry for this project') } end diff --git a/spec/features/security/project/internal_access_spec.rb b/spec/features/security/project/internal_access_spec.rb index 350de2e5b6b..a961d8b4f69 100644 --- a/spec/features/security/project/internal_access_spec.rb +++ b/spec/features/security/project/internal_access_spec.rb @@ -443,12 +443,12 @@ describe "Internal Project Access", feature: true do end describe "GET /:project_path/container_registry" do - let(:container_image) { create(:container_image) } + let(:container_repository) { create(:container_repository) } before do stub_container_registry_tags('latest') stub_container_registry_config(enabled: true) - project.container_images << container_image + project.container_repositories << container_repository end subject { namespace_project_container_registry_index_path(project.namespace, project) } diff --git a/spec/features/security/project/private_access_spec.rb b/spec/features/security/project/private_access_spec.rb index 62364206440..b7e42e67d82 100644 --- a/spec/features/security/project/private_access_spec.rb +++ b/spec/features/security/project/private_access_spec.rb @@ -432,12 +432,12 @@ describe "Private Project Access", feature: true do end describe "GET /:project_path/container_registry" do - let(:container_image) { create(:container_image) } + let(:container_repository) { create(:container_repository) } before do stub_container_registry_tags('latest') stub_container_registry_config(enabled: true) - project.container_images << container_image + project.container_repositories << container_repository end subject { namespace_project_container_registry_index_path(project.namespace, project) } diff --git a/spec/features/security/project/public_access_spec.rb b/spec/features/security/project/public_access_spec.rb index 0e0c3140fd0..02660984b29 100644 --- a/spec/features/security/project/public_access_spec.rb +++ b/spec/features/security/project/public_access_spec.rb @@ -443,12 +443,12 @@ describe "Public Project Access", feature: true do end describe "GET /:project_path/container_registry" do - let(:container_image) { create(:container_image) } + let(:container_repository) { create(:container_repository) } before do stub_container_registry_tags('latest') stub_container_registry_config(enabled: true) - project.container_images << container_image + project.container_repositories << container_repository end subject { namespace_project_container_registry_index_path(project.namespace, project) } diff --git a/spec/lib/container_registry/blob_spec.rb b/spec/lib/container_registry/blob_spec.rb index f092449c4bd..718a61ba291 100644 --- a/spec/lib/container_registry/blob_spec.rb +++ b/spec/lib/container_registry/blob_spec.rb @@ -15,7 +15,7 @@ describe ContainerRegistry::Blob do let(:project) { create(:project, path: 'test', group: group) } let(:example_host) { 'example.com' } let(:registry_url) { 'http://' + example_host } - let(:repository) { create(:container_image, name: '', project: project) } + let(:repository) { create(:container_repository, name: '', project: project) } let(:blob) { repository.blob(config) } before do diff --git a/spec/lib/container_registry/tag_spec.rb b/spec/lib/container_registry/tag_spec.rb index cdd0fe66bc3..01153a6eca9 100644 --- a/spec/lib/container_registry/tag_spec.rb +++ b/spec/lib/container_registry/tag_spec.rb @@ -5,7 +5,7 @@ describe ContainerRegistry::Tag do let(:project) { create(:project, path: 'test', group: group) } let(:example_host) { 'example.com' } let(:registry_url) { 'http://' + example_host } - let(:repository) { create(:container_image, name: '', project: project) } + let(:repository) { create(:container_repository, name: '', project: project) } let(:tag) { repository.tag('tag') } let(:headers) { { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' } } diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index c3ee743035a..0429636486c 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -115,7 +115,7 @@ merge_access_levels: - protected_branch push_access_levels: - protected_branch -container_images: +container_repositories: - name project: - taggings diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index d22447c602f..bc70c6f4aa2 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -149,13 +149,13 @@ describe Namespace, models: true do end context "when any project has container images" do - let(:container_image) { create(:container_image) } + let(:container_repository) { create(:container_repository) } before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') - create(:empty_project, namespace: @namespace, container_images: [container_image]) + create(:empty_project, namespace: @namespace, container_repositories: [container_repository]) allow(@namespace).to receive(:path_was).and_return(@namespace.path) allow(@namespace).to receive(:path).and_return('new_path') diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index aefbedf0b93..69b7906bb4e 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1186,12 +1186,12 @@ describe Project, models: true do end context 'container registry with images' do - let(:container_image) { create(:container_image) } + let(:container_repository) { create(:container_repository) } before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') - project.container_images << container_image + project.container_repositories << container_repository end subject { project.rename_repo } diff --git a/spec/services/container_images/destroy_service_spec.rb b/spec/services/container_images/destroy_service_spec.rb index 5b4dbaa7934..ebc15598cce 100644 --- a/spec/services/container_images/destroy_service_spec.rb +++ b/spec/services/container_images/destroy_service_spec.rb @@ -3,13 +3,13 @@ require 'spec_helper' describe ContainerImages::DestroyService, services: true do describe '#execute' do let(:user) { create(:user) } - let(:container_image) { create(:container_image, name: '') } - let(:project) { create(:project, path: 'test', namespace: user.namespace, container_images: [container_image]) } + let(:container_repository) { create(:container_repository, name: '') } + let(:project) { create(:project, path: 'test', namespace: user.namespace, container_repositorys: [container_repository]) } let(:example_host) { 'example.com' } let(:registry_url) { 'http://' + example_host } - it { expect(container_image).to be_valid } - it { expect(project.container_images).not_to be_empty } + it { expect(container_repository).to be_valid } + it { expect(project.container_repositorys).not_to be_empty } context 'when container image has tags' do before do @@ -19,15 +19,15 @@ describe ContainerImages::DestroyService, services: true do it 'removes all tags before destroy' do service = described_class.new(project, user) - expect(container_image).to receive(:delete_tags).and_return(true) - expect { service.execute(container_image) }.to change(project.container_images, :count).by(-1) + expect(container_repository).to receive(:delete_tags).and_return(true) + expect { service.execute(container_repository) }.to change(project.container_repositorys, :count).by(-1) end it 'fails when tags are not removed' do service = described_class.new(project, user) - expect(container_image).to receive(:delete_tags).and_return(false) - expect { service.execute(container_image) }.to raise_error(ActiveRecord::RecordNotDestroyed) + expect(container_repository).to receive(:delete_tags).and_return(false) + expect { service.execute(container_repository) }.to raise_error(ActiveRecord::RecordNotDestroyed) end end end diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index f91d62ebdaf..daad65d478a 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -90,12 +90,12 @@ describe Projects::DestroyService, services: true do end context 'container registry' do - let(:container_image) { create(:container_image) } + let(:container_repository) { create(:container_repository) } before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') - project.container_images << container_image + project.container_repositorys << container_repository end context 'images deletion succeeds' do diff --git a/spec/services/projects/transfer_service_spec.rb b/spec/services/projects/transfer_service_spec.rb index 5e56226ff91..adf8ede5086 100644 --- a/spec/services/projects/transfer_service_spec.rb +++ b/spec/services/projects/transfer_service_spec.rb @@ -29,12 +29,12 @@ describe Projects::TransferService, services: true do end context 'disallow transfering of project with tags' do - let(:container_image) { create(:container_image) } + let(:container_repository) { create(:container_repository) } before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') - project.container_images << container_image + project.container_repositorys << container_repository end subject { transfer_project(project, user, group) } From af42dd29a0e81d524731f4ce3ced2ed17bac9903 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 24 Mar 2017 12:31:34 +0100 Subject: [PATCH 25/97] Fix specs for container repository tags --- app/models/container_repository.rb | 8 ++-- lib/container_registry/blob.rb | 4 +- lib/container_registry/tag.rb | 6 +-- spec/factories/container_repositories.rb | 23 +++++----- spec/lib/container_registry/tag_spec.rb | 58 ++++++++++++++++++------ 5 files changed, 64 insertions(+), 35 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 2f0fd3014a8..e5076f30c8e 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -1,8 +1,10 @@ class ContainerRepository < ActiveRecord::Base belongs_to :project - delegate :client, to: :registry + validates :manifest, presence: true - validates :name, presence: true + validates :name, length: { minimum: 0, allow_nil: false } + + delegate :client, to: :registry before_destroy :delete_tags def registry @@ -17,7 +19,7 @@ class ContainerRepository < ActiveRecord::Base end def path - @path ||= "#{project.full_path}/#{name}" + @path ||= [project.full_path, name].select(&:present?).join('/') end def tag(tag) diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb index 8db8e483b1d..d5f85f9fcad 100644 --- a/lib/container_registry/blob.rb +++ b/lib/container_registry/blob.rb @@ -38,11 +38,11 @@ module ContainerRegistry end def delete - client.delete_blob(repository.name_with_namespace, digest) + client.delete_blob(repository.path, digest) end def data - @data ||= client.blob(repository.name_with_namespace, digest, type) + @data ||= client.blob(repository.path, digest, type) end end end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 68dd87c979d..d653deb3bf1 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -22,7 +22,7 @@ module ContainerRegistry end def manifest - @manifest ||= client.repository_manifest(repository.name_with_namespace, name) + @manifest ||= client.repository_manifest(repository.path, name) end def path @@ -38,7 +38,7 @@ module ContainerRegistry def digest return @digest if defined?(@digest) - @digest = client.repository_tag_digest(repository.name_with_namespace, name) + @digest = client.repository_tag_digest(repository.path, name) end def config_blob @@ -80,7 +80,7 @@ module ContainerRegistry def delete return unless digest - client.delete_repository_tag(repository.name_with_namespace, digest) + client.delete_repository_tag(repository.path, digest) end end end diff --git a/spec/factories/container_repositories.rb b/spec/factories/container_repositories.rb index fbf6bf62dfd..295b3596ee9 100644 --- a/spec/factories/container_repositories.rb +++ b/spec/factories/container_repositories.rb @@ -1,22 +1,21 @@ FactoryGirl.define do factory :container_repository do - name "test_container_image" + name 'test_container_image' project transient do - tags ['tag'] + tags [] end - after(:build) do |image, evaluator| - # if evaluator.tags.to_a.any? - # allow(Gitlab.config.registry).to receive(:enabled).and_return(true) - # allow(Auth::ContainerRegistryAuthenticationService) - # .to receive(:full_access_token).and_return('token') - # allow(image.client).to receive(:repository_tags).and_return({ - # name: image.name_with_namespace, - # tags: evaluator.tags - # }) - # end + after(:build) do |repository, evaluator| + if evaluator.tags.any? + allow(repository.client) + .to receive(:repository_tags) + .and_return({ + name: repository.path, + tags: evaluator.tags + }) + end end end end diff --git a/spec/lib/container_registry/tag_spec.rb b/spec/lib/container_registry/tag_spec.rb index 01153a6eca9..37eaa10f4a4 100644 --- a/spec/lib/container_registry/tag_spec.rb +++ b/spec/lib/container_registry/tag_spec.rb @@ -3,30 +3,58 @@ require 'spec_helper' describe ContainerRegistry::Tag do let(:group) { create(:group, name: 'group') } let(:project) { create(:project, path: 'test', group: group) } - let(:example_host) { 'example.com' } - let(:registry_url) { 'http://' + example_host } - let(:repository) { create(:container_repository, name: '', project: project) } - let(:tag) { repository.tag('tag') } - let(:headers) { { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' } } + + let(:repository) do + create(:container_repository, name: '', tags: %w[latest], project: project) + end + + # TODO, move stubs to helper with this header + let(:headers) do + { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' } + end + + let(:tag) { described_class.new(repository, 'tag') } before do - stub_container_registry_config(enabled: true, api_url: registry_url, host_port: example_host) + stub_container_registry_config(enabled: true, + api_url: 'http://registry.gitlab', + host_port: 'registry.gitlab') end it { expect(tag).to respond_to(:repository) } it { expect(tag).to delegate_method(:registry).to(:repository) } it { expect(tag).to delegate_method(:client).to(:repository) } - context '#path' do - subject { tag.path } + describe '#path' do + context 'when tag belongs to zero-level repository' do + let(:repository) do + create(:container_repository, name: '', + tags: %w[rc1], + project: project) + end - it { is_expected.to eq('example.com/group/test:tag') } + it 'returns path to the image' do + expect(tag.path).to eq('group/test:tag') + end + end + + context 'when tag belongs to first-level repository' do + let(:repository) do + create(:container_repository, name: 'my_image', + tags: %w[latest], + project: project) + end + + it 'returns path to the image' do + expect(tag.path).to eq('group/test/my_image:tag') + end + end end context 'manifest processing' do context 'schema v1' do before do - stub_request(:get, 'http://example.com/v2/group/test/manifests/tag'). + stub_request(:get, 'http://registry.gitlab/v2/group/test/manifests/tag'). with(headers: headers). to_return( status: 200, @@ -63,7 +91,7 @@ describe ContainerRegistry::Tag do context 'schema v2' do before do - stub_request(:get, 'http://example.com/v2/group/test/manifests/tag'). + stub_request(:get, 'http://registry.gitlab/v2/group/test/manifests/tag'). with(headers: headers). to_return( status: 200, @@ -100,7 +128,7 @@ describe ContainerRegistry::Tag do context 'when locally stored' do before do - stub_request(:get, 'http://example.com/v2/group/test/blobs/sha256:d7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac'). + stub_request(:get, 'http://registry.gitlab/v2/group/test/blobs/sha256:d7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac'). with(headers: { 'Accept' => 'application/octet-stream' }). to_return( status: 200, @@ -112,7 +140,7 @@ describe ContainerRegistry::Tag do context 'when externally stored' do before do - stub_request(:get, 'http://example.com/v2/group/test/blobs/sha256:d7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac'). + stub_request(:get, 'http://registry.gitlab/v2/group/test/blobs/sha256:d7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac'). with(headers: { 'Accept' => 'application/octet-stream' }). to_return( status: 307, @@ -132,7 +160,7 @@ describe ContainerRegistry::Tag do context 'manifest digest' do before do - stub_request(:head, 'http://example.com/v2/group/test/manifests/tag'). + stub_request(:head, 'http://registry.gitlab/v2/group/test/manifests/tag'). with(headers: headers). to_return(status: 200, headers: { 'Docker-Content-Digest' => 'sha256:digest' }) end @@ -145,7 +173,7 @@ describe ContainerRegistry::Tag do context '#delete' do before do - stub_request(:delete, 'http://example.com/v2/group/test/manifests/sha256:digest'). + stub_request(:delete, 'http://registry.gitlab/v2/group/test/manifests/sha256:digest'). with(headers: headers). to_return(status: 200) end From 7db1f22673f1b6890b6ce4b9db9b367eae3988f0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 24 Mar 2017 12:41:42 +0100 Subject: [PATCH 26/97] Fix specs for services related to container registry --- .../container_images/destroy_service_spec.rb | 54 +++++++++++-------- .../services/projects/destroy_service_spec.rb | 2 +- .../projects/transfer_service_spec.rb | 2 +- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/spec/services/container_images/destroy_service_spec.rb b/spec/services/container_images/destroy_service_spec.rb index ebc15598cce..ab5ebe5eac8 100644 --- a/spec/services/container_images/destroy_service_spec.rb +++ b/spec/services/container_images/destroy_service_spec.rb @@ -1,34 +1,42 @@ require 'spec_helper' -describe ContainerImages::DestroyService, services: true do - describe '#execute' do - let(:user) { create(:user) } - let(:container_repository) { create(:container_repository, name: '') } - let(:project) { create(:project, path: 'test', namespace: user.namespace, container_repositorys: [container_repository]) } - let(:example_host) { 'example.com' } - let(:registry_url) { 'http://' + example_host } +describe ContainerImages::DestroyService, '#execute', :services do + let(:user) { create(:user) } - it { expect(container_repository).to be_valid } - it { expect(project.container_repositorys).not_to be_empty } + let(:container_repository) do + create(:container_repository, name: 'myimage', tags: %w[latest]) + end - context 'when container image has tags' do - before do - project.team << [user, :master] - end + let(:project) do + create(:project, path: 'test', + namespace: user.namespace, + container_repositories: [container_repository]) + end - it 'removes all tags before destroy' do - service = described_class.new(project, user) + it { expect(container_repository).to be_valid } + it { expect(project.container_repositories).not_to be_empty } - expect(container_repository).to receive(:delete_tags).and_return(true) - expect { service.execute(container_repository) }.to change(project.container_repositorys, :count).by(-1) - end + context 'when container image has tags' do + before do + project.add_master(user) + end - it 'fails when tags are not removed' do - service = described_class.new(project, user) + it 'removes all tags before destroy' do + service = described_class.new(project, user) - expect(container_repository).to receive(:delete_tags).and_return(false) - expect { service.execute(container_repository) }.to raise_error(ActiveRecord::RecordNotDestroyed) - end + expect(container_repository) + .to receive(:delete_tags).and_return(true) + expect { service.execute(container_repository) } + .to change(project.container_repositories, :count).by(-1) + end + + it 'fails when tags are not removed' do + service = described_class.new(project, user) + + expect(container_repository) + .to receive(:delete_tags).and_return(false) + expect { service.execute(container_repository) } + .to raise_error(ActiveRecord::RecordNotDestroyed) end end end diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index daad65d478a..44e0286350b 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -95,7 +95,7 @@ describe Projects::DestroyService, services: true do before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') - project.container_repositorys << container_repository + project.container_repositories << container_repository end context 'images deletion succeeds' do diff --git a/spec/services/projects/transfer_service_spec.rb b/spec/services/projects/transfer_service_spec.rb index adf8ede5086..a3babaf1e0b 100644 --- a/spec/services/projects/transfer_service_spec.rb +++ b/spec/services/projects/transfer_service_spec.rb @@ -34,7 +34,7 @@ describe Projects::TransferService, services: true do before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') - project.container_repositorys << container_repository + project.container_repositories << container_repository end subject { transfer_project(project, user, group) } From 4fdcaca6a148475a18aa5931b62c4173301ded8c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 24 Mar 2017 13:08:21 +0100 Subject: [PATCH 27/97] Fix container registry blob specs --- spec/lib/container_registry/blob_spec.rb | 118 ++++++++++++----------- 1 file changed, 61 insertions(+), 57 deletions(-) diff --git a/spec/lib/container_registry/blob_spec.rb b/spec/lib/container_registry/blob_spec.rb index 718a61ba291..76ea29666ea 100644 --- a/spec/lib/container_registry/blob_spec.rb +++ b/spec/lib/container_registry/blob_spec.rb @@ -1,117 +1,121 @@ require 'spec_helper' describe ContainerRegistry::Blob do - let(:digest) { 'sha256:0123456789012345' } - let(:config) do - { - 'digest' => digest, - 'mediaType' => 'binary', - 'size' => 1000 - } - end - let(:token) { 'token' } - let(:group) { create(:group, name: 'group') } let(:project) { create(:project, path: 'test', group: group) } - let(:example_host) { 'example.com' } - let(:registry_url) { 'http://' + example_host } - let(:repository) { create(:container_repository, name: '', project: project) } - let(:blob) { repository.blob(config) } + + let(:repository) do + create(:container_repository, name: 'image', + tags: %w[latest rc1], + project: project) + end + + let(:config) do + { 'digest' => 'sha256:0123456789012345', + 'mediaType' => 'binary', + 'size' => 1000 } + end + + let(:blob) { described_class.new(repository, config) } before do - stub_container_registry_config(enabled: true, api_url: registry_url, host_port: example_host) + stub_container_registry_config(enabled: true, + api_url: 'http://registry.gitlab', + host_port: 'registry.gitlab') end it { expect(blob).to respond_to(:repository) } it { expect(blob).to delegate_method(:registry).to(:repository) } it { expect(blob).to delegate_method(:client).to(:repository) } - context '#path' do - subject { blob.path } - - it { is_expected.to eq('example.com/group/test@sha256:0123456789012345') } + describe '#path' do + it 'returns a valid path to the blob' do + expect(blob.path).to eq('group/test/image@sha256:0123456789012345') + end end - context '#digest' do - subject { blob.digest } - - it { is_expected.to eq(digest) } + describe '#digest' do + it 'return correct digest value' do + expect(blob.digest).to eq 'sha256:0123456789012345' + end end - context '#type' do - subject { blob.type } - - it { is_expected.to eq('binary') } + describe '#type' do + it 'returns a correct type' do + expect(blob.type).to eq 'binary' + end end - context '#revision' do - subject { blob.revision } - - it { is_expected.to eq('0123456789012345') } + describe '#revision' do + it 'returns a correct blob SHA' do + expect(blob.revision).to eq '0123456789012345' + end end - context '#short_revision' do - subject { blob.short_revision } - - it { is_expected.to eq('012345678') } + describe '#short_revision' do + it 'return a short SHA' do + expect(blob.short_revision).to eq '012345678' + end end - context '#delete' do + describe '#delete' do before do - stub_request(:delete, 'http://example.com/v2/group/test/blobs/sha256:0123456789012345'). - to_return(status: 200) + stub_request(:delete, 'http://registry.gitlab/v2/group/test/image/blobs/sha256:0123456789012345') + .to_return(status: 200) end - subject { blob.delete } - - it { is_expected.to be_truthy } + it 'returns true when blob has been successfuly deleted' do + expect(blob.delete).to be_truthy + end end - context '#data' do - let(:data) { '{"key":"value"}' } - - subject { blob.data } - + describe '#data' do context 'when locally stored' do before do - stub_request(:get, 'http://example.com/v2/group/test/blobs/sha256:0123456789012345'). + stub_request(:get, 'http://registry.gitlab/v2/group/test/image/blobs/sha256:0123456789012345'). to_return( status: 200, headers: { 'Content-Type' => 'application/json' }, - body: data) + body: '{"key":"value"}') end - it { is_expected.to eq(data) } + it 'returns a correct blob data' do + expect(blob.data).to eq '{"key":"value"}' + end end context 'when externally stored' do + let(:location) { 'http://external.com/blob/file' } + before do - stub_request(:get, 'http://example.com/v2/group/test/blobs/sha256:0123456789012345'). - with(headers: { 'Authorization' => "bearer #{token}" }). - to_return( + stub_request(:get, 'http://registry.gitlab/v2/group/test/image/blobs/sha256:0123456789012345') + .with(headers: { 'Authorization' => 'bearer token' }) + .to_return( status: 307, headers: { 'Location' => location }) end context 'for a valid address' do - let(:location) { 'http://external.com/blob/file' } - before do stub_request(:get, location). with(headers: { 'Authorization' => nil }). to_return( status: 200, headers: { 'Content-Type' => 'application/json' }, - body: data) + body: '{"key":"value"}') end - it { is_expected.to eq(data) } + it 'returns correct data' do + expect(blob.data).to eq '{"key":"value"}' + end end context 'for invalid file' do let(:location) { 'file:///etc/passwd' } - it { expect{ subject }.to raise_error(ArgumentError, 'invalid address') } + it 'raises an error' do + expect { blob.data }.to raise_error(ArgumentError, 'invalid address') + end end end end From 16e645c6563b81ce03e481b094abc2d331d05f36 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 24 Mar 2017 13:27:05 +0100 Subject: [PATCH 28/97] Remove container_registry method from project class --- app/models/project.rb | 14 -------------- spec/models/project_spec.rb | 29 +++-------------------------- 2 files changed, 3 insertions(+), 40 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index 5c6672c95b3..a579ac7dc64 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -405,20 +405,6 @@ class Project < ActiveRecord::Base @repository ||= Repository.new(path_with_namespace, self) end - def container_registry - return unless Gitlab.config.registry.enabled - - @container_registry ||= begin - token = Auth::ContainerRegistryAuthenticationService.full_access_token(project) - - url = Gitlab.config.registry.api_url - host_port = Gitlab.config.registry.host_port - # TODO, move configuration vars into ContainerRegistry::Registry, clean - # this method up afterwards - ContainerRegistry::Registry.new(url, token: token, path: host_port) - end - end - def container_registry_url if Gitlab.config.registry.enabled "#{Gitlab.config.registry.host_port}/#{path_with_namespace.downcase}" diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 69b7906bb4e..e4e75cc6a09 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1389,25 +1389,6 @@ describe Project, models: true do end end - describe '#container_registry_path_with_namespace' do - let(:project) { create(:empty_project, path: 'PROJECT') } - - subject { project.container_registry_path_with_namespace } - - it { is_expected.not_to eq(project.path_with_namespace) } - it { is_expected.to eq(project.path_with_namespace.downcase) } - end - - describe '#container_registry' do - let(:project) { create(:empty_project) } - - before { stub_container_registry_config(enabled: true) } - - subject { project.container_registry } - - it { is_expected.not_to be_nil } - end - describe '#container_registry_url' do let(:project) { create(:empty_project) } @@ -1417,10 +1398,8 @@ describe Project, models: true do context 'for enabled registry' do let(:registry_settings) do - { - enabled: true, - host_port: 'example.com', - } + { enabled: true, + host_port: 'example.com' } end it { is_expected.not_to be_nil } @@ -1428,9 +1407,7 @@ describe Project, models: true do context 'for disabled registry' do let(:registry_settings) do - { - enabled: false - } + { enabled: false } end it { is_expected.to be_nil } From d6f37a34c1c262f49a92f26dd187819419d56c2f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 24 Mar 2017 13:27:18 +0100 Subject: [PATCH 29/97] Fix feature specs related to container registry --- app/controllers/projects/container_registry_controller.rb | 4 ++-- spec/lib/gitlab/import_export/all_models.yml | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/controllers/projects/container_registry_controller.rb b/app/controllers/projects/container_registry_controller.rb index 4981e57ed22..8929bd0aa55 100644 --- a/app/controllers/projects/container_registry_controller.rb +++ b/app/controllers/projects/container_registry_controller.rb @@ -5,7 +5,7 @@ class Projects::ContainerRegistryController < Projects::ApplicationController layout 'project' def index - @images = project.container_images + @images = project.container_repositories end def destroy @@ -44,7 +44,7 @@ class Projects::ContainerRegistryController < Projects::ApplicationController end def image - @image ||= project.container_images.find_by(id: params[:id]) + @image ||= project.container_repositories.find_by(id: params[:id]) end def tag diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 0429636486c..e5bba29d85b 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -116,6 +116,7 @@ merge_access_levels: push_access_levels: - protected_branch container_repositories: +- project - name project: - taggings @@ -201,7 +202,7 @@ project: - project_authorizations - route - statistics -- container_images +- container_repositories - uploads award_emoji: - awardable From dce706bfcbddf7952e2c42c0c42825044cbb43a2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 24 Mar 2017 13:47:29 +0100 Subject: [PATCH 30/97] Do not require a manifest for container repository Container repository can be empty - no tags or blogs is OK. --- app/models/container_repository.rb | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index e5076f30c8e..c8c56e69269 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -1,7 +1,6 @@ class ContainerRepository < ActiveRecord::Base belongs_to :project - validates :manifest, presence: true validates :name, length: { minimum: 0, allow_nil: false } delegate :client, to: :registry @@ -43,6 +42,8 @@ class ContainerRepository < ActiveRecord::Base ContainerRegistry::Blob.new(self, config) end + # TODO, add bang to this method + # def delete_tags return unless tags @@ -52,6 +53,14 @@ class ContainerRepository < ActiveRecord::Base end end + # TODO, specs needed + # + def empty? + tags.none? + end + + # TODO, we will return a new ContainerRepository object here + # def self.project_from_path(repository_path) return unless repository_path.include?('/') From 7ada193e0fd28b4a6eca1fda7dda6f0ebe6b2d72 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 24 Mar 2017 14:58:25 +0100 Subject: [PATCH 31/97] Fix specs for container repository destroy service --- spec/features/container_registry_spec.rb | 1 - spec/services/container_images/destroy_service_spec.rb | 4 ++++ spec/support/stub_gitlab_calls.rb | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index 88642f72772..73b22fa1b7d 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -14,7 +14,6 @@ describe "Container Registry" do stub_container_registry_config(enabled: true) stub_container_registry_tags(*tags) project.container_repositories << container_repository unless container_repository.nil? - allow(Auth::ContainerRegistryAuthenticationService).to receive(:full_access_token).and_return('token') end describe 'GET /:project/container_registry' do diff --git a/spec/services/container_images/destroy_service_spec.rb b/spec/services/container_images/destroy_service_spec.rb index ab5ebe5eac8..6931b503e6d 100644 --- a/spec/services/container_images/destroy_service_spec.rb +++ b/spec/services/container_images/destroy_service_spec.rb @@ -13,6 +13,10 @@ describe ContainerImages::DestroyService, '#execute', :services do container_repositories: [container_repository]) end + before do + stub_container_registry_config(enabled: true) + end + it { expect(container_repository).to be_valid } it { expect(project.container_repositories).not_to be_empty } diff --git a/spec/support/stub_gitlab_calls.rb b/spec/support/stub_gitlab_calls.rb index a01ef576234..3949784aabb 100644 --- a/spec/support/stub_gitlab_calls.rb +++ b/spec/support/stub_gitlab_calls.rb @@ -27,7 +27,8 @@ module StubGitlabCalls def stub_container_registry_config(registry_settings) allow(Gitlab.config.registry).to receive_messages(registry_settings) - allow(Auth::ContainerRegistryAuthenticationService).to receive(:full_access_token).and_return('token') + allow(Auth::ContainerRegistryAuthenticationService) + .to receive(:full_access_token).and_return('token') end def stub_container_registry_tags(*tags) From f09e3fe85116743c0cb537fb958a8b0ab1ad19fb Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 27 Mar 2017 16:04:38 +0200 Subject: [PATCH 32/97] Add a few pending specs for container repository --- spec/models/container_repository_spec.rb | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index e3180e01758..296b9e713a8 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -92,4 +92,29 @@ describe ContainerRepository do end end end + + describe '#from_repository_path' do + context 'when received multi-level repository path' do + let(:repository) do + described_class.from_repository_path('group/test/some/image/name') + end + + pending 'fabricates object within a correct project' do + expect(repository.project).to eq project + end + + pending 'it fabricates project with a correct name' do + expect(repository.name).to eq 'some/image/name' + end + end + + context 'when path contains too many nodes' do + end + + context 'when received multi-level repository with nested groups' do + end + + context 'when received root repository path' do + end + end end From b15d9042e2688f29b002f90e0154e793ff1544ff Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 28 Mar 2017 14:34:56 +0200 Subject: [PATCH 33/97] Implement container repository path class --- lib/container_registry/path.rb | 29 ++++++ spec/lib/container_registry/path_spec.rb | 119 +++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 lib/container_registry/path.rb create mode 100644 spec/lib/container_registry/path_spec.rb diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb new file mode 100644 index 00000000000..f32df1bc0d1 --- /dev/null +++ b/lib/container_registry/path.rb @@ -0,0 +1,29 @@ +module ContainerRegistry + class Path + InvalidRegistryPathError = Class.new(StandardError) + + def initialize(name) + @nodes = name.to_s.split('/') + end + + def valid? + @nodes.size > 1 && + @nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED + end + + def components + raise InvalidRegistryPathError unless valid? + + @components ||= @nodes.size.downto(2).map do |length| + @nodes.take(length).join('/') + end + end + + def repository_project + @project ||= Project.where_full_path_in(components.first(3))&.first + end + + def repository_name + end + end +end diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb new file mode 100644 index 00000000000..32f25f5e527 --- /dev/null +++ b/spec/lib/container_registry/path_spec.rb @@ -0,0 +1,119 @@ +require 'spec_helper' + +describe ContainerRegistry::Path do + let(:path) { described_class.new(name) } + + describe '#components' do + context 'when repository path is valid' do + let(:name) { 'path/to/some/project' } + + it 'return all project-like components in reverse order' do + expect(path.components).to eq %w[path/to/some/project + path/to/some + path/to] + end + end + + context 'when repository path is invalid' do + let(:name) { '' } + + it 'rasises en error' do + expect { path.components } + .to raise_error described_class::InvalidRegistryPathError + end + end + end + + describe '#valid?' do + context 'when path has less than two components' do + let(:name) { 'something/' } + + it 'is not valid' do + expect(path).not_to be_valid + end + end + + context 'when path has more than allowed number of components' do + let(:name) { 'a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/r/s/t/u/w/y/z' } + + it 'is not valid' do + expect(path).not_to be_valid + end + end + + context 'when path has two or more components' do + let(:name) { 'some/path' } + + it 'is valid' do + expect(path).to be_valid + end + end + end + + describe '#repository_project' do + let(:group) { create(:group, path: 'some_group') } + + context 'when project for given path exists' do + let(:name) { 'some_group/some_project' } + + before do + create(:empty_project, group: group, name: 'some_project') + create(:empty_project, name: 'some_project') + end + + it 'returns a correct project' do + expect(path.repository_project.group).to eq group + end + end + + context 'when project for given path does not exist' do + let(:name) { 'not/matching' } + + it 'returns nil' do + expect(path.repository_project).to be_nil + end + end + + context 'when matching multi-level path' do + let(:project) do + create(:empty_project, group: group, name: 'some_project') + end + + context 'when using the zero-level path' do + let(:name) { project.full_path } + + it 'supports zero-level path' do + expect(path.repository_project).to eq project + end + end + + context 'when using first-level path' do + let(:name) { "#{project.full_path}/repository" } + + it 'supports first-level path' do + expect(path.repository_project).to eq project + end + end + + context 'when using second-level path' do + let(:name) { "#{project.full_path}/repository/name" } + + it 'supports second-level path' do + expect(path.repository_project).to eq project + end + end + + context 'when using too deep nesting in the path' do + let(:name) { "#{project.full_path}/repository/name/invalid" } + + it 'does not support three-levels of nesting' do + expect(path.repository_project).to be_nil + end + end + end + end + + describe '#repository_name' do + pending 'returns a correct name' + end +end From bdc1e1b9e005eeaf564b79698d61a801c3c6360f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 28 Mar 2017 14:57:22 +0200 Subject: [PATCH 34/97] Implement method matching container repository names --- lib/container_registry/path.rb | 8 +++++ spec/lib/container_registry/path_spec.rb | 45 +++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index f32df1bc0d1..89ef396f374 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -3,6 +3,7 @@ module ContainerRegistry InvalidRegistryPathError = Class.new(StandardError) def initialize(name) + @name = name @nodes = name.to_s.split('/') end @@ -19,11 +20,18 @@ module ContainerRegistry end end + def has_repository? + # ContainerRepository.find_by_full_path(@name).present? + end + def repository_project @project ||= Project.where_full_path_in(components.first(3))&.first end def repository_name + return unless repository_project + + @name.remove(%r(^?#{Regexp.escape(repository_project.full_path)}/?)) end end end diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index 32f25f5e527..278b1fc1b55 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -114,6 +114,49 @@ describe ContainerRegistry::Path do end describe '#repository_name' do - pending 'returns a correct name' + context 'when project does not exist' do + let(:name) { 'some/name' } + + it 'returns nil' do + expect(path.repository_name).to be_nil + end + end + + context 'when project exists' do + let(:group) { create(:group, path: 'some_group') } + + let(:project) do + create(:empty_project, group: group, name: 'some_project') + end + + before do + allow(path).to receive(:repository_project) + .and_return(project) + end + + context 'when project path equal repository path' do + let(:name) { 'some_group/some_project' } + + it 'returns an empty string' do + expect(path.repository_name).to eq '' + end + end + + context 'when repository path has one additional level' do + let(:name) { 'some_group/some_project/repository' } + + it 'returns a correct repository name' do + expect(path.repository_name).to eq 'repository' + end + end + + context 'when repository path has two additional levels' do + let(:name) { 'some_group/some_project/repository/image' } + + it 'returns a correct repository name' do + expect(path.repository_name).to eq 'repository/image' + end + end + end end end From 8584798886a8c7d0077157c29e0dc05087656aaf Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 28 Mar 2017 16:20:16 +0200 Subject: [PATCH 35/97] Fix rubocop offense in container registry path class --- lib/container_registry/path.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 89ef396f374..0ca51ab3766 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -8,8 +8,7 @@ module ContainerRegistry end def valid? - @nodes.size > 1 && - @nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED + @nodes.size > 1 && @nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED end def components From 95faf5f5b7268ea1750f3a764cd0537b3e0d1e25 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 29 Mar 2017 12:14:06 +0200 Subject: [PATCH 36/97] Use new registry path class to match repository project --- app/models/container_repository.rb | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index c8c56e69269..149d65ddbff 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -62,19 +62,7 @@ class ContainerRepository < ActiveRecord::Base # TODO, we will return a new ContainerRepository object here # def self.project_from_path(repository_path) - return unless repository_path.include?('/') - - ## - # Projects are always located inside a namespace, so we can remove - # the last node, and see if project with that path exists. - # - truncated_path = repository_path.slice(0...repository_path.rindex('/')) - - ## - # We still make it possible to search projects by a full image path - # in order to maintain backwards compatibility. - # - Project.find_by_full_path(truncated_path) || - Project.find_by_full_path(repository_path) + ContainerRegistry::Path.new(repository_path) + .repository_project end end From a222486c3e06d02e93704aa8440299cb4a677cef Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 29 Mar 2017 12:14:29 +0200 Subject: [PATCH 37/97] Rename method for checking tags in container repository This is important because method `empty?` is triggered when validation happens, and we don't want to make API request to registry when record is validated. --- app/models/container_repository.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 149d65ddbff..b128069ca0e 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -42,6 +42,12 @@ class ContainerRepository < ActiveRecord::Base ContainerRegistry::Blob.new(self, config) end + # TODO, specs needed + # + def has_tags? + tags.any? + end + # TODO, add bang to this method # def delete_tags @@ -53,12 +59,6 @@ class ContainerRepository < ActiveRecord::Base end end - # TODO, specs needed - # - def empty? - tags.none? - end - # TODO, we will return a new ContainerRepository object here # def self.project_from_path(repository_path) From 5a7f8cb5d2f9ab7bc7172cedeb220b7d78530f78 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 29 Mar 2017 12:30:38 +0200 Subject: [PATCH 38/97] Add readability improvements to registry auth specs --- ...er_registry_authentication_service_spec.rb | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/spec/services/auth/container_registry_authentication_service_spec.rb b/spec/services/auth/container_registry_authentication_service_spec.rb index b91234ddb1e..fac7f1b1235 100644 --- a/spec/services/auth/container_registry_authentication_service_spec.rb +++ b/spec/services/auth/container_registry_authentication_service_spec.rb @@ -6,14 +6,15 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do let(:current_params) { {} } let(:rsa_key) { OpenSSL::PKey::RSA.generate(512) } let(:payload) { JWT.decode(subject[:token], rsa_key).first } + let(:authentication_abilities) do - [ - :read_container_image, - :create_container_image - ] + [:read_container_image, :create_container_image] end - subject { described_class.new(current_project, current_user, current_params).execute(authentication_abilities: authentication_abilities) } + subject do + described_class.new(current_project, current_user, current_params) + .execute(authentication_abilities: authentication_abilities) + end before do allow(Gitlab.config.registry).to receive_messages(enabled: true, issuer: 'rspec', key: nil) @@ -40,13 +41,11 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end end - shared_examples 'a accessible' do + shared_examples 'an accessible' do let(:access) do - [{ - 'type' => 'repository', + [{ 'type' => 'repository', 'name' => project.path_with_namespace, - 'actions' => actions, - }] + 'actions' => actions }] end it_behaves_like 'a valid token' @@ -59,19 +58,19 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end shared_examples 'a pullable' do - it_behaves_like 'a accessible' do + it_behaves_like 'an accessible' do let(:actions) { ['pull'] } end end shared_examples 'a pushable' do - it_behaves_like 'a accessible' do + it_behaves_like 'an accessible' do let(:actions) { ['push'] } end end shared_examples 'a pullable and pushable' do - it_behaves_like 'a accessible' do + it_behaves_like 'an accessible' do let(:actions) { %w(pull push) } end end @@ -87,7 +86,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do subject { { token: token } } - it_behaves_like 'a accessible' do + it_behaves_like 'an accessible' do let(:actions) { ['*'] } end end @@ -198,11 +197,9 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do context 'build authorized as user' do let(:current_project) { create(:empty_project) } let(:current_user) { create(:user) } + let(:authentication_abilities) do - [ - :build_read_container_image, - :build_create_container_image - ] + [:build_read_container_image, :build_create_container_image] end before do From 06bae00365cda6930063b98fde1a9b804f2ae3bc Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 29 Mar 2017 12:53:02 +0200 Subject: [PATCH 39/97] Make container repository path code more readable --- lib/container_registry/path.rb | 24 ++++++-- spec/lib/container_registry/path_spec.rb | 70 +++++++++++++----------- 2 files changed, 58 insertions(+), 36 deletions(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 0ca51ab3766..d3f8bf74e14 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -1,10 +1,24 @@ module ContainerRegistry + ## + # Class reponsible for extracting project and repository name from + # image repository path provided by a containers registry API response. + # + # Example: + # + # some/group/my_project/my/image -> + # project: some/group/my_project + # repository: my/image + # class Path InvalidRegistryPathError = Class.new(StandardError) - def initialize(name) - @name = name - @nodes = name.to_s.split('/') + def initialize(path) + @path = path + @nodes = path.to_s.split('/') + end + + def to_s + @path end def valid? @@ -20,7 +34,7 @@ module ContainerRegistry end def has_repository? - # ContainerRepository.find_by_full_path(@name).present? + # ContainerRepository.find_by_full_path(@path).present? end def repository_project @@ -30,7 +44,7 @@ module ContainerRegistry def repository_name return unless repository_project - @name.remove(%r(^?#{Regexp.escape(repository_project.full_path)}/?)) + @path.remove(%r(^?#{Regexp.escape(repository_project.full_path)}/?)) end end end diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index 278b1fc1b55..a680a0adcb2 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -1,51 +1,59 @@ require 'spec_helper' describe ContainerRegistry::Path do - let(:path) { described_class.new(name) } + subject { described_class.new(path) } describe '#components' do context 'when repository path is valid' do - let(:name) { 'path/to/some/project' } + let(:path) { 'path/to/some/project' } it 'return all project-like components in reverse order' do - expect(path.components).to eq %w[path/to/some/project + expect(subject.components).to eq %w[path/to/some/project path/to/some path/to] end end context 'when repository path is invalid' do - let(:name) { '' } + let(:path) { '' } it 'rasises en error' do - expect { path.components } + expect { subject.components } .to raise_error described_class::InvalidRegistryPathError end end end + describe '#to_s' do + let(:path) { 'some/image' } + + it 'return a string with a repository path' do + expect(subject.to_s).to eq path + end + end + describe '#valid?' do context 'when path has less than two components' do - let(:name) { 'something/' } + let(:path) { 'something/' } it 'is not valid' do - expect(path).not_to be_valid + expect(subject).not_to be_valid end end context 'when path has more than allowed number of components' do - let(:name) { 'a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/r/s/t/u/w/y/z' } + let(:path) { 'a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/r/s/t/u/w/y/z' } it 'is not valid' do - expect(path).not_to be_valid + expect(subject).not_to be_valid end end context 'when path has two or more components' do - let(:name) { 'some/path' } + let(:path) { 'some/path' } it 'is valid' do - expect(path).to be_valid + expect(subject).to be_valid end end end @@ -54,7 +62,7 @@ describe ContainerRegistry::Path do let(:group) { create(:group, path: 'some_group') } context 'when project for given path exists' do - let(:name) { 'some_group/some_project' } + let(:path) { 'some_group/some_project' } before do create(:empty_project, group: group, name: 'some_project') @@ -62,15 +70,15 @@ describe ContainerRegistry::Path do end it 'returns a correct project' do - expect(path.repository_project.group).to eq group + expect(subject.repository_project.group).to eq group end end context 'when project for given path does not exist' do - let(:name) { 'not/matching' } + let(:path) { 'not/matching' } it 'returns nil' do - expect(path.repository_project).to be_nil + expect(subject.repository_project).to be_nil end end @@ -80,34 +88,34 @@ describe ContainerRegistry::Path do end context 'when using the zero-level path' do - let(:name) { project.full_path } + let(:path) { project.full_path } it 'supports zero-level path' do - expect(path.repository_project).to eq project + expect(subject.repository_project).to eq project end end context 'when using first-level path' do - let(:name) { "#{project.full_path}/repository" } + let(:path) { "#{project.full_path}/repository" } it 'supports first-level path' do - expect(path.repository_project).to eq project + expect(subject.repository_project).to eq project end end context 'when using second-level path' do - let(:name) { "#{project.full_path}/repository/name" } + let(:path) { "#{project.full_path}/repository/name" } it 'supports second-level path' do - expect(path.repository_project).to eq project + expect(subject.repository_project).to eq project end end context 'when using too deep nesting in the path' do - let(:name) { "#{project.full_path}/repository/name/invalid" } + let(:path) { "#{project.full_path}/repository/name/invalid" } it 'does not support three-levels of nesting' do - expect(path.repository_project).to be_nil + expect(subject.repository_project).to be_nil end end end @@ -115,10 +123,10 @@ describe ContainerRegistry::Path do describe '#repository_name' do context 'when project does not exist' do - let(:name) { 'some/name' } + let(:path) { 'some/name' } it 'returns nil' do - expect(path.repository_name).to be_nil + expect(subject.repository_name).to be_nil end end @@ -135,26 +143,26 @@ describe ContainerRegistry::Path do end context 'when project path equal repository path' do - let(:name) { 'some_group/some_project' } + let(:path) { 'some_group/some_project' } it 'returns an empty string' do - expect(path.repository_name).to eq '' + expect(subject.repository_name).to eq '' end end context 'when repository path has one additional level' do - let(:name) { 'some_group/some_project/repository' } + let(:path) { 'some_group/some_project/repository' } it 'returns a correct repository name' do - expect(path.repository_name).to eq 'repository' + expect(subject.repository_name).to eq 'repository' end end context 'when repository path has two additional levels' do - let(:name) { 'some_group/some_project/repository/image' } + let(:path) { 'some_group/some_project/repository/image' } it 'returns a correct repository name' do - expect(path.repository_name).to eq 'repository/image' + expect(subject.repository_name).to eq 'repository/image' end end end From 3bfc05be5ee5b0262857febf90fc7e1f17895d4e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 29 Mar 2017 13:01:48 +0200 Subject: [PATCH 40/97] Use container repository path inside auth service --- app/models/container_repository.rb | 7 ------- .../container_registry_authentication_service.rb | 12 ++++++++---- spec/lib/container_registry/path_spec.rb | 4 ++-- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index b128069ca0e..98acf49c939 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -58,11 +58,4 @@ class ContainerRepository < ActiveRecord::Base client.delete_repository_tag(self.path, digest) end end - - # TODO, we will return a new ContainerRepository object here - # - def self.project_from_path(repository_path) - ContainerRegistry::Path.new(repository_path) - .repository_project - end end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 3d151c6a357..7a2ec9664c1 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -56,13 +56,15 @@ module Auth def process_scope(scope) type, name, actions = scope.split(':', 3) actions = actions.split(',') + path = ContainerRegistry::Path.new(name) + return unless type == 'repository' - process_repository_access(type, name, actions) + process_repository_access(type, path, actions) end - def process_repository_access(type, name, actions) - requested_project = ContainerRepository.project_from_path(name) + def process_repository_access(type, path, actions) + requested_project = path.repository_project return unless requested_project @@ -70,7 +72,9 @@ module Auth can_access?(requested_project, action) end - { type: type, name: name, actions: actions } if actions.present? + return unless actions.present? + + { type: type, name: path.to_s, actions: actions } end def can_access?(requested_project, requested_action) diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index a680a0adcb2..6384850eb19 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -9,8 +9,8 @@ describe ContainerRegistry::Path do it 'return all project-like components in reverse order' do expect(subject.components).to eq %w[path/to/some/project - path/to/some - path/to] + path/to/some + path/to] end end From 313e35e817445271b68d40ab06f192f1d3e0ccf0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 29 Mar 2017 13:14:06 +0200 Subject: [PATCH 41/97] Remove container images destroy service class --- .../container_images/destroy_service.rb | 9 ---- .../container_images/destroy_service_spec.rb | 46 ------------------- 2 files changed, 55 deletions(-) delete mode 100644 app/services/container_images/destroy_service.rb delete mode 100644 spec/services/container_images/destroy_service_spec.rb diff --git a/app/services/container_images/destroy_service.rb b/app/services/container_images/destroy_service.rb deleted file mode 100644 index 15dca227291..00000000000 --- a/app/services/container_images/destroy_service.rb +++ /dev/null @@ -1,9 +0,0 @@ -module ContainerImages - class DestroyService < BaseService - def execute(container_image) - return false unless can?(current_user, :update_container_image, project) - - container_image.destroy! - end - end -end diff --git a/spec/services/container_images/destroy_service_spec.rb b/spec/services/container_images/destroy_service_spec.rb deleted file mode 100644 index 6931b503e6d..00000000000 --- a/spec/services/container_images/destroy_service_spec.rb +++ /dev/null @@ -1,46 +0,0 @@ -require 'spec_helper' - -describe ContainerImages::DestroyService, '#execute', :services do - let(:user) { create(:user) } - - let(:container_repository) do - create(:container_repository, name: 'myimage', tags: %w[latest]) - end - - let(:project) do - create(:project, path: 'test', - namespace: user.namespace, - container_repositories: [container_repository]) - end - - before do - stub_container_registry_config(enabled: true) - end - - it { expect(container_repository).to be_valid } - it { expect(project.container_repositories).not_to be_empty } - - context 'when container image has tags' do - before do - project.add_master(user) - end - - it 'removes all tags before destroy' do - service = described_class.new(project, user) - - expect(container_repository) - .to receive(:delete_tags).and_return(true) - expect { service.execute(container_repository) } - .to change(project.container_repositories, :count).by(-1) - end - - it 'fails when tags are not removed' do - service = described_class.new(project, user) - - expect(container_repository) - .to receive(:delete_tags).and_return(false) - expect { service.execute(container_repository) } - .to raise_error(ActiveRecord::RecordNotDestroyed) - end - end -end From 4407d3cf19bc815142ea3a7908003e85efde76ed Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 29 Mar 2017 14:01:05 +0200 Subject: [PATCH 42/97] Add comment to container registry auth service Comment explains why we still have authentication without user object there. The legacy authentication mechanism should be removed in 10.0. --- .../container_registry_authentication_service.rb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 7a2ec9664c1..b050f1dd51b 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -107,6 +107,11 @@ module Auth can?(current_user, :read_container_image, requested_project) end + ## + # We still support legacy pipeline triggers which do not have associated + # actor. New permissions model and new triggers are always associated with + # an actor, so this should be improved in 10.0 version of GitLab. + # def build_can_push?(requested_project) # Build can push only to the project from which it originates has_authentication_ability?(:build_create_container_image) && @@ -119,14 +124,11 @@ module Auth end def error(code, status:, message: '') - { - errors: [{ code: code, message: message }], - http_status: status - } + { errors: [{ code: code, message: message }], http_status: status } end def has_authentication_ability?(capability) - (@authentication_abilities || []).include?(capability) + @authentication_abilities.to_a.include?(capability) end end end From 031122eb54390b4ed792289237ecfb156ec69002 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 30 Mar 2017 13:31:33 +0200 Subject: [PATCH 43/97] Add container repository create service with specs --- app/models/container_repository.rb | 4 + .../create_repository_service.rb | 33 +++++++++ .../create_repository_service_spec.rb | 74 +++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 app/services/container_registry/create_repository_service.rb create mode 100644 spec/services/container_registry/create_repository_service_spec.rb diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 98acf49c939..33e2574d389 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -58,4 +58,8 @@ class ContainerRepository < ActiveRecord::Base client.delete_repository_tag(self.path, digest) end end + + def self.create_from_path(path) + self.create(project: path.repository_project, name: path.repository_name) + end end diff --git a/app/services/container_registry/create_repository_service.rb b/app/services/container_registry/create_repository_service.rb new file mode 100644 index 00000000000..84218702a4f --- /dev/null +++ b/app/services/container_registry/create_repository_service.rb @@ -0,0 +1,33 @@ +module ContainerRegistry + ## + # Service for creating a container repository. + # + # It is usually executed before registry authenticator returns + # a token for given request. + # + class CreateRepositoryService < BaseService + def execute(path) + @path = path + + return if path.has_repository? + + unless user_can_create? || legacy_trigger_can_create? + raise Gitlab::Access::AccessDeniedError + end + + ContainerRepository.create_from_path(path) + end + + private + + def user_can_create? + can?(@current_user, :create_container_image, @path.repository_project) + end + + ## TODO, remove it after removing legacy triggers. + # + def legacy_trigger_can_create? + @current_user.nil? && @project == @path.repository_project + end + end +end diff --git a/spec/services/container_registry/create_repository_service_spec.rb b/spec/services/container_registry/create_repository_service_spec.rb new file mode 100644 index 00000000000..dfd07c8cc02 --- /dev/null +++ b/spec/services/container_registry/create_repository_service_spec.rb @@ -0,0 +1,74 @@ +require 'spec_helper' + +describe ContainerRegistry::CreateRepositoryService, '#execute' do + let(:project) { create(:empty_project) } + let(:user) { create(:user) } + + let(:path) do + ContainerRegistry::Path.new("#{project.full_path}/my/image") + end + + let(:service) { described_class.new(project, user) } + + before do + stub_container_registry_config(enabled: true) + end + + context 'when container repository already exists' do + before do + create(:container_repository, project: project, name: 'my/image') + end + + it 'does not create container repository again' do + expect { service.execute(path) } + .to raise_error(Gitlab::Access::AccessDeniedError) + .and change { ContainerRepository.count }.by(0) + end + end + + context 'when repository is created by an user' do + context 'when user has no ability to create a repository' do + it 'does not create a new container repository' do + expect { service.execute(path) } + .to raise_error(Gitlab::Access::AccessDeniedError) + .and change { ContainerRepository.count }.by(0) + end + end + + context 'when user has ability do create a repository' do + before do + project.add_developer(user) + end + + it 'creates a new container repository' do + expect { service.execute(path) } + .to change { project.container_repositories.count }.by(1) + end + end + end + + context 'when repository is created by a legacy pipeline trigger' do + let(:user) { nil } + + context 'when repository path matches authenticated project' do + it 'creates a new container repository' do + expect { service.execute(path) } + .to change { project.container_repositories.count }.by(1) + end + end + + context 'when repository path does not match authenticated project' do + let(:private_project) { create(:empty_project, :private) } + + let(:path) do + ContainerRegistry::Path.new("#{private_project.full_path}/my/image") + end + + it 'does not create a new container repository' do + expect { service.execute(path) } + .to raise_error(Gitlab::Access::AccessDeniedError) + .and change { ContainerRepository.count }.by(0) + end + end + end +end From 003a51d17aea6af92953f7207e2b39a5cb6db8de Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 30 Mar 2017 13:45:54 +0200 Subject: [PATCH 44/97] Check container repository exists for a given path --- lib/container_registry/path.rb | 11 +++++-- spec/lib/container_registry/path_spec.rb | 30 +++++++++++++++++- .../create_repository_service_spec.rb | 31 ++++++++++--------- 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index d3f8bf74e14..3a6fde08e8f 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -33,8 +33,15 @@ module ContainerRegistry end end + def has_project? + repository_project.present? + end + def has_repository? - # ContainerRepository.find_by_full_path(@path).present? + return false unless has_project? + + repository_project.container_repositories + .where(name: repository_name).any? end def repository_project @@ -42,7 +49,7 @@ module ContainerRegistry end def repository_name - return unless repository_project + return unless has_project? @path.remove(%r(^?#{Regexp.escape(repository_project.full_path)}/?)) end diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index 6384850eb19..906dd920031 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -39,7 +39,7 @@ describe ContainerRegistry::Path do it 'is not valid' do expect(subject).not_to be_valid end - end + end context 'when path has more than allowed number of components' do let(:path) { 'a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/r/s/t/u/w/y/z' } @@ -58,6 +58,34 @@ describe ContainerRegistry::Path do end end + describe '#has_repository?' do + context 'when project exists' do + let(:project) { create(:empty_project) } + let(:path) { "#{project.full_path}/my/image" } + + context 'when path already has matching repository' do + before do + create(:container_repository, project: project, name: 'my/image') + end + + it { is_expected.to have_repository } + it { is_expected.to have_project } + end + + context 'when path does not have matching repository' do + it { is_expected.not_to have_repository } + it { is_expected.to have_project } + end + end + + context 'when project does not exist' do + let(:path) { 'some/project/my/image' } + + it { is_expected.not_to have_repository } + it { is_expected.not_to have_project } + end + end + describe '#repository_project' do let(:group) { create(:group, path: 'some_group') } diff --git a/spec/services/container_registry/create_repository_service_spec.rb b/spec/services/container_registry/create_repository_service_spec.rb index dfd07c8cc02..ac7b147f92e 100644 --- a/spec/services/container_registry/create_repository_service_spec.rb +++ b/spec/services/container_registry/create_repository_service_spec.rb @@ -14,18 +14,6 @@ describe ContainerRegistry::CreateRepositoryService, '#execute' do stub_container_registry_config(enabled: true) end - context 'when container repository already exists' do - before do - create(:container_repository, project: project, name: 'my/image') - end - - it 'does not create container repository again' do - expect { service.execute(path) } - .to raise_error(Gitlab::Access::AccessDeniedError) - .and change { ContainerRepository.count }.by(0) - end - end - context 'when repository is created by an user' do context 'when user has no ability to create a repository' do it 'does not create a new container repository' do @@ -40,9 +28,22 @@ describe ContainerRegistry::CreateRepositoryService, '#execute' do project.add_developer(user) end - it 'creates a new container repository' do - expect { service.execute(path) } - .to change { project.container_repositories.count }.by(1) + context 'when repository already exists' do + before do + create(:container_repository, project: project, name: 'my/image') + end + + it 'does not create container repository again' do + expect { service.execute(path) } + .to_not change { ContainerRepository.count } + end + end + + context 'when repository does not exist yet' do + it 'creates a new container repository' do + expect { service.execute(path) } + .to change { project.container_repositories.count }.by(1) + end end end end From 236c9c170374190bcb6dc54a1ca3dddda2d3dcb2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 30 Mar 2017 15:24:08 +0200 Subject: [PATCH 45/97] Simplify how we create container registry resources --- .../create_repository_service.rb | 33 -------- .../create_repository_service_spec.rb | 75 ------------------- 2 files changed, 108 deletions(-) delete mode 100644 app/services/container_registry/create_repository_service.rb delete mode 100644 spec/services/container_registry/create_repository_service_spec.rb diff --git a/app/services/container_registry/create_repository_service.rb b/app/services/container_registry/create_repository_service.rb deleted file mode 100644 index 84218702a4f..00000000000 --- a/app/services/container_registry/create_repository_service.rb +++ /dev/null @@ -1,33 +0,0 @@ -module ContainerRegistry - ## - # Service for creating a container repository. - # - # It is usually executed before registry authenticator returns - # a token for given request. - # - class CreateRepositoryService < BaseService - def execute(path) - @path = path - - return if path.has_repository? - - unless user_can_create? || legacy_trigger_can_create? - raise Gitlab::Access::AccessDeniedError - end - - ContainerRepository.create_from_path(path) - end - - private - - def user_can_create? - can?(@current_user, :create_container_image, @path.repository_project) - end - - ## TODO, remove it after removing legacy triggers. - # - def legacy_trigger_can_create? - @current_user.nil? && @project == @path.repository_project - end - end -end diff --git a/spec/services/container_registry/create_repository_service_spec.rb b/spec/services/container_registry/create_repository_service_spec.rb deleted file mode 100644 index ac7b147f92e..00000000000 --- a/spec/services/container_registry/create_repository_service_spec.rb +++ /dev/null @@ -1,75 +0,0 @@ -require 'spec_helper' - -describe ContainerRegistry::CreateRepositoryService, '#execute' do - let(:project) { create(:empty_project) } - let(:user) { create(:user) } - - let(:path) do - ContainerRegistry::Path.new("#{project.full_path}/my/image") - end - - let(:service) { described_class.new(project, user) } - - before do - stub_container_registry_config(enabled: true) - end - - context 'when repository is created by an user' do - context 'when user has no ability to create a repository' do - it 'does not create a new container repository' do - expect { service.execute(path) } - .to raise_error(Gitlab::Access::AccessDeniedError) - .and change { ContainerRepository.count }.by(0) - end - end - - context 'when user has ability do create a repository' do - before do - project.add_developer(user) - end - - context 'when repository already exists' do - before do - create(:container_repository, project: project, name: 'my/image') - end - - it 'does not create container repository again' do - expect { service.execute(path) } - .to_not change { ContainerRepository.count } - end - end - - context 'when repository does not exist yet' do - it 'creates a new container repository' do - expect { service.execute(path) } - .to change { project.container_repositories.count }.by(1) - end - end - end - end - - context 'when repository is created by a legacy pipeline trigger' do - let(:user) { nil } - - context 'when repository path matches authenticated project' do - it 'creates a new container repository' do - expect { service.execute(path) } - .to change { project.container_repositories.count }.by(1) - end - end - - context 'when repository path does not match authenticated project' do - let(:private_project) { create(:empty_project, :private) } - - let(:path) do - ContainerRegistry::Path.new("#{private_project.full_path}/my/image") - end - - it 'does not create a new container repository' do - expect { service.execute(path) } - .to raise_error(Gitlab::Access::AccessDeniedError) - .and change { ContainerRepository.count }.by(0) - end - end - end -end From 7d3d1ec5a7b3ee13397af587c6ecb3a3fc297006 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 30 Mar 2017 15:24:46 +0200 Subject: [PATCH 46/97] Create container repository on successful push auth Because we do not have yet two way communication between container registry and GitLab, we need to eagerly create a new container repository objects in database. We now do that after user/build successfully authenticates a push action using auth service. --- app/models/container_repository.rb | 3 +- ...ntainer_registry_authentication_service.rb | 16 ++++++ lib/container_registry/path.rb | 4 ++ ...er_registry_authentication_service_spec.rb | 52 +++++++++++++++++-- 4 files changed, 70 insertions(+), 5 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 33e2574d389..5663b3db92f 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -60,6 +60,7 @@ class ContainerRepository < ActiveRecord::Base end def self.create_from_path(path) - self.create(project: path.repository_project, name: path.repository_name) + self.create(project: path.repository_project, + name: path.repository_name) end end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index b050f1dd51b..839f514ad58 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -74,9 +74,25 @@ module Auth return unless actions.present? + # At this point user/build is already authenticated. + # + ensure_container_repository!(path, actions) + { type: type, name: path.to_s, actions: actions } end + ## + # Because we do not have two way communication with registry yet, + # we create a container repository image resource when push to the + # registry is successfuly authorized. + # + def ensure_container_repository!(path, actions) + return if path.has_repository? + return unless actions.include?('push') + + ContainerRepository.create_from_path(path) + end + def can_access?(requested_project, requested_action) return false unless requested_project.container_registry_enabled? diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 3a6fde08e8f..27e0e7897ff 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -44,6 +44,10 @@ module ContainerRegistry .where(name: repository_name).any? end + def root_repository? + @path == repository_project.full_path + end + def repository_project @project ||= Project.where_full_path_in(components.first(3))&.first end diff --git a/spec/services/auth/container_registry_authentication_service_spec.rb b/spec/services/auth/container_registry_authentication_service_spec.rb index fac7f1b1235..a4a6430011e 100644 --- a/spec/services/auth/container_registry_authentication_service_spec.rb +++ b/spec/services/auth/container_registry_authentication_service_spec.rb @@ -80,6 +80,19 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do it { is_expected.not_to include(:token) } end + shared_examples 'container repository factory' do + it 'creates a new containe repository resource' do + expect { subject } + .to change { project.container_repositories.count }.by(1) + end + end + + shared_examples 'container repository factory' do + it 'does not create a new container repository resource' do + expect { subject }.not_to change { ContainerRepository.count } + end + end + describe '#full_access_token' do let(:project) { create(:empty_project) } let(:token) { described_class.full_access_token(project.path_with_namespace) } @@ -89,6 +102,8 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do it_behaves_like 'an accessible' do let(:actions) { ['*'] } end + + it_behaves_like 'not a container repository factory' end context 'user authorization' do @@ -109,16 +124,20 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a pushable' + it_behaves_like 'container repository factory' end context 'allow reporter to pull images' do before { project.team << [current_user, :reporter] } - let(:current_params) do - { scope: "repository:#{project.path_with_namespace}:pull" } - end + context 'when pulling from root level repository' do + let(:current_params) do + { scope: "repository:#{project.path_with_namespace}:pull" } + end - it_behaves_like 'a pullable' + it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' + end end context 'return a least of privileges' do @@ -129,6 +148,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end context 'disallow guest to pull or push images' do @@ -139,6 +159,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end end @@ -151,6 +172,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end context 'disallow anyone to push images' do @@ -159,6 +181,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end end @@ -172,6 +195,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end context 'disallow anyone to push images' do @@ -180,6 +204,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end end @@ -190,6 +215,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end end end @@ -216,6 +242,10 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do it_behaves_like 'a pullable and pushable' do let(:project) { current_project } end + + it_behaves_like 'container repository factory' do + let(:project) { current_project } + end end context 'for other projects' do @@ -228,11 +258,13 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do let(:project) { create(:empty_project, :public) } it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end shared_examples 'pullable for being team member' do context 'when you are not member' do it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end context 'when you are member' do @@ -241,12 +273,14 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end context 'when you are owner' do let(:project) { create(:empty_project, namespace: current_user.namespace) } it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end end @@ -260,6 +294,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do context 'when you are not member' do it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end context 'when you are member' do @@ -268,12 +303,14 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end context 'when you are owner' do let(:project) { create(:empty_project, namespace: current_user.namespace) } it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end end end @@ -293,12 +330,14 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end context 'when you are owner' do let(:project) { create(:empty_project, :public, namespace: current_user.namespace) } it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end end end @@ -315,6 +354,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end end end @@ -322,6 +362,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do context 'unauthorized' do context 'disallow to use scope-less authentication' do it_behaves_like 'a forbidden' + it_behaves_like 'not a container repository factory' end context 'for invalid scope' do @@ -330,6 +371,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a forbidden' + it_behaves_like 'not a container repository factory' end context 'for private project' do @@ -351,6 +393,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end context 'when pushing' do @@ -359,6 +402,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a forbidden' + it_behaves_like 'not a container repository factory' end end end From fffc8a59d7f193e451daace7c69f33603c906e8f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 30 Mar 2017 15:41:35 +0200 Subject: [PATCH 47/97] Show full container repository path in the UI --- app/views/projects/container_registry/_image.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/projects/container_registry/_image.html.haml b/app/views/projects/container_registry/_image.html.haml index 4fd642a56c9..64b11e375c6 100644 --- a/app/views/projects/container_registry/_image.html.haml +++ b/app/views/projects/container_registry/_image.html.haml @@ -7,7 +7,7 @@ - else = icon("chevron-down") - = escape_once(image.name) + = escape_once(image.path) = clipboard_button(clipboard_text: "docker pull #{image.path}") .controls.hidden-xs.pull-right = link_to namespace_project_container_registry_path(@project.namespace, @project, image.id), class: 'btn btn-remove has-tooltip', title: "Remove image", data: { confirm: "Are you sure?" }, method: :delete do From cf042068b5f69b416640f9a4fcb21fbec1082268 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 30 Mar 2017 15:41:51 +0200 Subject: [PATCH 48/97] Do not allow registry requests for invalid repositories --- .../auth/container_registry_authentication_service.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 839f514ad58..dcb728b6151 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -64,6 +64,10 @@ module Auth end def process_repository_access(type, path, actions) + # TODO, add specs for invalid paths + # + return unless path.valid? + requested_project = path.repository_project return unless requested_project From 600bbe15a103b63e14daa295abaffdf1aeafaef3 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 11:12:38 +0200 Subject: [PATCH 49/97] Fix rubocop offense in registry path specs --- spec/lib/container_registry/path_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index 906dd920031..68732b12542 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -39,7 +39,7 @@ describe ContainerRegistry::Path do it 'is not valid' do expect(subject).not_to be_valid end - end + end context 'when path has more than allowed number of components' do let(:path) { 'a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/r/s/t/u/w/y/z' } From a7466af3a6f31311d64654631a2ea2740c42b88e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 11:54:09 +0200 Subject: [PATCH 50/97] Improve code related to removing container image tags --- app/models/container_repository.rb | 11 ++++--- spec/factories/container_repositories.rb | 20 +++++++++---- spec/features/container_registry_spec.rb | 3 +- spec/models/container_repository_spec.rb | 30 +++++++------------ .../services/projects/destroy_service_spec.rb | 6 ++-- 5 files changed, 36 insertions(+), 34 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 5663b3db92f..052d93c3bdc 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -45,15 +45,14 @@ class ContainerRepository < ActiveRecord::Base # TODO, specs needed # def has_tags? - tags.any? + tags.to_a.any? end - # TODO, add bang to this method - # - def delete_tags - return unless tags + def delete_tags! + return unless has_tags? + + digests = tags.map { |tag| tag.digest }.to_set - digests = tags.map {|tag| tag.digest }.to_set digests.all? do |digest| client.delete_repository_tag(self.path, digest) end diff --git a/spec/factories/container_repositories.rb b/spec/factories/container_repositories.rb index 295b3596ee9..4919a03cdf2 100644 --- a/spec/factories/container_repositories.rb +++ b/spec/factories/container_repositories.rb @@ -8,13 +8,21 @@ FactoryGirl.define do end after(:build) do |repository, evaluator| - if evaluator.tags.any? + next if evaluator.tags.to_a.none? + + allow(repository.client) + .to receive(:repository_tags) + .and_return({ + 'name' => repository.path, + 'tags' => evaluator.tags + }) + + evaluator.tags.each do |tag| allow(repository.client) - .to receive(:repository_tags) - .and_return({ - name: repository.path, - tags: evaluator.tags - }) + .to receive(:repository_tag_digest) + .with(repository.path, tag) + .and_return('sha256:4c8e63ca4cb663ce6c688cb06f1c3' \ + '72b088dac5b6d7ad7d49cd620d85cf72a15') end end end diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index 73b22fa1b7d..530e6af92d3 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -38,7 +38,8 @@ describe "Container Registry" do end it do - expect_any_instance_of(ContainerRepository).to receive(:delete_tags).and_return(true) + expect_any_instance_of(ContainerRepository) + .to receive(:delete_tags!).and_return(true) click_on 'Remove image' end diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index 296b9e713a8..92dccf76d71 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -57,38 +57,30 @@ describe ContainerRepository do it { is_expected.not_to be_empty } end - # TODO, improve these specs - # - describe '#delete_tags' do - let(:tag) { ContainerRegistry::Tag.new(container_repository, 'tag') } - - before do - allow(container_repository).to receive(:tags).twice.and_return([tag]) - allow(tag).to receive(:digest) - .and_return('sha256:4c8e63ca4cb663ce6c688cb06f1c3672a172b088dac5b6d7ad7d49cd620d85cf') + describe '#delete_tags!' do + let(:container_repository) do + create(:container_repository, name: 'my_image', + tags: %w[latest rc1], + project: project) end context 'when action succeeds' do - before do - allow(container_repository.client) + it 'returns status that indicates success' do + expect(container_repository.client) .to receive(:delete_repository_tag) .and_return(true) - end - it 'returns status that indicates success' do - expect(container_repository.delete_tags).to be_truthy + expect(container_repository.delete_tags!).to be_truthy end end context 'when action fails' do - before do - allow(container_repository.client) + it 'returns status that indicates failure' do + expect(container_repository.client) .to receive(:delete_repository_tag) .and_return(false) - end - it 'returns status that indicates failure' do - expect(container_repository.delete_tags).to be_falsey + expect(container_repository.delete_tags!).to be_falsey end end end diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index 44e0286350b..193ccd17282 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -100,7 +100,8 @@ describe Projects::DestroyService, services: true do context 'images deletion succeeds' do it do - expect_any_instance_of(ContainerRepository).to receive(:delete_tags).and_return(true) + expect_any_instance_of(ContainerRepository) + .to receive(:delete_tags!).and_return(true) destroy_project(project, user, {}) end @@ -108,7 +109,8 @@ describe Projects::DestroyService, services: true do context 'images deletion fails' do before do - expect_any_instance_of(ContainerRepository).to receive(:delete_tags).and_return(false) + expect_any_instance_of(ContainerRepository) + .to receive(:delete_tags!).and_return(false) end subject { destroy_project(project, user, {}) } From 60cdd2bcc894cf9cce4892570bf6a146dc45e536 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 12:27:05 +0200 Subject: [PATCH 51/97] Add specs for container repository factory method --- app/models/container_repository.rb | 2 +- ...ntainer_registry_authentication_service.rb | 2 +- spec/models/container_repository_spec.rb | 53 +++++++++++++++---- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 052d93c3bdc..e27369c10d6 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -58,7 +58,7 @@ class ContainerRepository < ActiveRecord::Base end end - def self.create_from_path(path) + def self.create_from_path!(path) self.create(project: path.repository_project, name: path.repository_name) end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index dcb728b6151..d58ff589be1 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -94,7 +94,7 @@ module Auth return if path.has_repository? return unless actions.include?('push') - ContainerRepository.create_from_path(path) + ContainerRepository.create_from_path!(path) end def can_access?(requested_project, requested_action) diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index 92dccf76d71..884eac43719 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -85,28 +85,63 @@ describe ContainerRepository do end end - describe '#from_repository_path' do - context 'when received multi-level repository path' do - let(:repository) do - described_class.from_repository_path('group/test/some/image/name') - end + describe '.create_from_path!' do + let(:repository) do + described_class.create_from_path!(ContainerRegistry::Path.new(path)) + end - pending 'fabricates object within a correct project' do + let(:repository_path) { ContainerRegistry::Path.new(path) } + + context 'when received multi-level repository path' do + let(:path) { project.full_path + '/some/image' } + + it 'fabricates repository assigned to a correct project' do expect(repository.project).to eq project end - pending 'it fabricates project with a correct name' do - expect(repository.name).to eq 'some/image/name' + it 'fabricates repository with a correct name' do + expect(repository.name).to eq 'some/image' end end - context 'when path contains too many nodes' do + context 'when path is too long' do + let(:path) do + project.full_path + '/a/b/c/d/e/f/g/h/i/j/k/l/n/o/p/s/t/u/x/y/z' + end + + it 'does not create repository and raises error' do + expect { repository }.to raise_error( + ContainerRegistry::Path::InvalidRegistryPathError) + end end context 'when received multi-level repository with nested groups' do + let(:group) { create(:group, :nested, name: 'nested') } + let(:path) { project.full_path + '/some/image' } + + it 'fabricates repository assigned to a correct project' do + expect(repository.project).to eq project + end + + it 'fabricates repository with a correct name' do + expect(repository.name).to eq 'some/image' + end + + it 'has path including a nested group' do + expect(repository.path).to include 'nested/test/some/image' + end end context 'when received root repository path' do + let(:path) { project.full_path } + + it 'fabricates repository assigned to a correct project' do + expect(repository.project).to eq project + end + + it 'fabricates repository with an empty name' do + expect(repository.name).to be_empty + end end end end From 4726ff9dbee74d00544c7eb1ea188ecdfe16d7e8 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 12:37:44 +0200 Subject: [PATCH 52/97] Add test example for invalid registry access request --- .../container_registry_authentication_service.rb | 2 -- ...ontainer_registry_authentication_service_spec.rb | 13 +++++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index d58ff589be1..5e151b0f044 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -64,8 +64,6 @@ module Auth end def process_repository_access(type, path, actions) - # TODO, add specs for invalid paths - # return unless path.valid? requested_project = path.repository_project diff --git a/spec/services/auth/container_registry_authentication_service_spec.rb b/spec/services/auth/container_registry_authentication_service_spec.rb index a4a6430011e..e273dfe1552 100644 --- a/spec/services/auth/container_registry_authentication_service_spec.rb +++ b/spec/services/auth/container_registry_authentication_service_spec.rb @@ -81,13 +81,13 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end shared_examples 'container repository factory' do - it 'creates a new containe repository resource' do + it 'creates a new container repository resource' do expect { subject } .to change { project.container_repositories.count }.by(1) end end - shared_examples 'container repository factory' do + shared_examples 'not a container repository factory' do it 'does not create a new container repository resource' do expect { subject }.not_to change { ContainerRepository.count } end @@ -183,6 +183,15 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do it_behaves_like 'an inaccessible' it_behaves_like 'not a container repository factory' end + + context 'when repository name is invalid' do + let(:current_params) do + { scope: 'repository:invalid:push' } + end + + it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' + end end context 'for internal project' do From 41956773839ba010bb316e6bbe8d48c1ad7177de Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 13:08:09 +0200 Subject: [PATCH 53/97] Reorganize container repository controllers and views --- .../projects/container_registry_controller.rb | 53 ------------------ .../registry/application_controller.rb | 10 ++++ .../registry/repositories_controller.rb | 54 +++++++++++++++++++ .../projects/registry/tags_controller.rb | 7 +++ .../repositories}/_image.html.haml | 0 .../repositories}/_tag.html.haml | 0 .../repositories}/index.html.haml | 0 config/routes/project.rb | 5 +- 8 files changed, 75 insertions(+), 54 deletions(-) delete mode 100644 app/controllers/projects/container_registry_controller.rb create mode 100644 app/controllers/projects/registry/application_controller.rb create mode 100644 app/controllers/projects/registry/repositories_controller.rb create mode 100644 app/controllers/projects/registry/tags_controller.rb rename app/views/projects/{container_registry => registry/repositories}/_image.html.haml (100%) rename app/views/projects/{container_registry => registry/repositories}/_tag.html.haml (100%) rename app/views/projects/{container_registry => registry/repositories}/index.html.haml (100%) diff --git a/app/controllers/projects/container_registry_controller.rb b/app/controllers/projects/container_registry_controller.rb deleted file mode 100644 index 8929bd0aa55..00000000000 --- a/app/controllers/projects/container_registry_controller.rb +++ /dev/null @@ -1,53 +0,0 @@ -class Projects::ContainerRegistryController < Projects::ApplicationController - before_action :verify_registry_enabled - before_action :authorize_read_container_image! - before_action :authorize_update_container_image!, only: [:destroy] - layout 'project' - - def index - @images = project.container_repositories - end - - def destroy - if tag - delete_tag - else - delete_image - end - end - - private - - def registry_url - @registry_url ||= namespace_project_container_registry_index_path(project.namespace, project) - end - - def verify_registry_enabled - render_404 unless Gitlab.config.registry.enabled - end - - def delete_image - if image.destroy - redirect_to registry_url - else - redirect_to registry_url, alert: 'Failed to remove image' - end - end - - def delete_tag - if tag.delete - image.destroy if image.tags.empty? - redirect_to registry_url - else - redirect_to registry_url, alert: 'Failed to remove tag' - end - end - - def image - @image ||= project.container_repositories.find_by(id: params[:id]) - end - - def tag - @tag ||= image.tag(params[:tag]) if params[:tag].present? - end -end diff --git a/app/controllers/projects/registry/application_controller.rb b/app/controllers/projects/registry/application_controller.rb new file mode 100644 index 00000000000..8710c219553 --- /dev/null +++ b/app/controllers/projects/registry/application_controller.rb @@ -0,0 +1,10 @@ +module Projects + module Registry + class ApplicationController < Projects::ApplicationController + layout 'project' + + before_action :verify_registry_enabled + before_action :authorize_read_container_image! + end + end +end diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb new file mode 100644 index 00000000000..b953d5b3378 --- /dev/null +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -0,0 +1,54 @@ +module Projects + module Registry + class RepositoriesController < ::Projects::Registry::ApplicationController + before_action :authorize_update_container_image!, only: [:destroy] + + def index + @images = project.container_repositories + end + + def destroy + if tag + delete_tag + else + delete_image + end + end + + private + + def registry_url + @registry_url ||= namespace_project_container_registry_index_path(project.namespace, project) + end + + def verify_registry_enabled + render_404 unless Gitlab.config.registry.enabled + end + + def delete_image + if image.destroy + redirect_to registry_url + else + redirect_to registry_url, alert: 'Failed to remove image' + end + end + + def delete_tag + if tag.delete + image.destroy if image.tags.empty? + redirect_to registry_url + else + redirect_to registry_url, alert: 'Failed to remove tag' + end + end + + def image + @image ||= project.container_repositories.find_by(id: params[:id]) + end + + def tag + @tag ||= image.tag(params[:tag]) if params[:tag].present? + end + end + end +end diff --git a/app/controllers/projects/registry/tags_controller.rb b/app/controllers/projects/registry/tags_controller.rb new file mode 100644 index 00000000000..e40489f67ad --- /dev/null +++ b/app/controllers/projects/registry/tags_controller.rb @@ -0,0 +1,7 @@ +module Projects + module Registry + class TagsController < ::Projects::Registry::ApplicationController + before_action :authorize_update_container_image!, only: [:destroy] + end + end +end diff --git a/app/views/projects/container_registry/_image.html.haml b/app/views/projects/registry/repositories/_image.html.haml similarity index 100% rename from app/views/projects/container_registry/_image.html.haml rename to app/views/projects/registry/repositories/_image.html.haml diff --git a/app/views/projects/container_registry/_tag.html.haml b/app/views/projects/registry/repositories/_tag.html.haml similarity index 100% rename from app/views/projects/container_registry/_tag.html.haml rename to app/views/projects/registry/repositories/_tag.html.haml diff --git a/app/views/projects/container_registry/index.html.haml b/app/views/projects/registry/repositories/index.html.haml similarity index 100% rename from app/views/projects/container_registry/index.html.haml rename to app/views/projects/registry/repositories/index.html.haml diff --git a/config/routes/project.rb b/config/routes/project.rb index 44b8ae7aedd..34f4bd917f7 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -219,7 +219,10 @@ constraints(ProjectUrlConstrainer.new) do end end - resources :container_registry, only: [:index, :destroy], constraints: { id: Gitlab::Regex.container_registry_reference_regex } + resources :container_registry, + controller: 'registry/repositories', + only: [:index, :destroy], + constraints: { id: Gitlab::Regex.container_registry_reference_regex } resources :milestones, constraints: { id: /\d+/ } do member do From f32d269cfa08f280c0d0dd86c4b50c4dcfb7409f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 13:16:16 +0200 Subject: [PATCH 54/97] Remove unused method from container registry client --- lib/container_registry/client.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 196cdd36a88..7f5f6d9ddb6 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -15,10 +15,6 @@ module ContainerRegistry @options = options end - def update_token(token) - @options[:token] = token - end - def repository_tags(name) response_body faraday.get("/v2/#{name}/tags/list") end From 00319e595ab52906d12ef027a10e08ac92ea1337 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 13:56:07 +0200 Subject: [PATCH 55/97] Move code related to registry to multiple controllers --- .../registry/application_controller.rb | 8 ++++- .../registry/repositories_controller.rb | 36 +++---------------- .../projects/registry/tags_controller.rb | 20 +++++++++++ .../registry/repositories/_tag.html.haml | 9 +++-- config/routes/project.rb | 6 ++++ 5 files changed, 44 insertions(+), 35 deletions(-) diff --git a/app/controllers/projects/registry/application_controller.rb b/app/controllers/projects/registry/application_controller.rb index 8710c219553..a56f9c58726 100644 --- a/app/controllers/projects/registry/application_controller.rb +++ b/app/controllers/projects/registry/application_controller.rb @@ -3,8 +3,14 @@ module Projects class ApplicationController < Projects::ApplicationController layout 'project' - before_action :verify_registry_enabled + before_action :verify_registry_enabled! before_action :authorize_read_container_image! + + private + + def verify_registry_enabled! + render_404 unless Gitlab.config.registry.enabled + end end end end diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb index b953d5b3378..e3e30176b30 100644 --- a/app/controllers/projects/registry/repositories_controller.rb +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -8,47 +8,19 @@ module Projects end def destroy - if tag - delete_tag + if image.destroy + redirect_to project_container_registry_path(@project) else - delete_image + redirect_to project_container_registry_path(@project), + alert: 'Failed to remove images repository!' end end private - def registry_url - @registry_url ||= namespace_project_container_registry_index_path(project.namespace, project) - end - - def verify_registry_enabled - render_404 unless Gitlab.config.registry.enabled - end - - def delete_image - if image.destroy - redirect_to registry_url - else - redirect_to registry_url, alert: 'Failed to remove image' - end - end - - def delete_tag - if tag.delete - image.destroy if image.tags.empty? - redirect_to registry_url - else - redirect_to registry_url, alert: 'Failed to remove tag' - end - end - def image @image ||= project.container_repositories.find_by(id: params[:id]) end - - def tag - @tag ||= image.tag(params[:tag]) if params[:tag].present? - end end end end diff --git a/app/controllers/projects/registry/tags_controller.rb b/app/controllers/projects/registry/tags_controller.rb index e40489f67ad..8f0a1aff394 100644 --- a/app/controllers/projects/registry/tags_controller.rb +++ b/app/controllers/projects/registry/tags_controller.rb @@ -2,6 +2,26 @@ module Projects module Registry class TagsController < ::Projects::Registry::ApplicationController before_action :authorize_update_container_image!, only: [:destroy] + + def destroy + if tag.delete + redirect_to project_container_registry_path(@project) + else + redirect_to project_container_registry_path(@project), + alert: 'Failed to remove repository tag!' + end + end + + private + + def repository + @image ||= project.container_repositories + .find_by(id: params[:repository_id]) + end + + def tag + @tag ||= repository.tag(params[:id]) if params[:id].present? + end end end end diff --git a/app/views/projects/registry/repositories/_tag.html.haml b/app/views/projects/registry/repositories/_tag.html.haml index f7161e85428..c9b3644ff93 100644 --- a/app/views/projects/registry/repositories/_tag.html.haml +++ b/app/views/projects/registry/repositories/_tag.html.haml @@ -25,5 +25,10 @@ - if can?(current_user, :update_container_image, @project) %td.content .controls.hidden-xs.pull-right - = link_to namespace_project_container_registry_path(@project.namespace, @project, { id: tag.repository.id, tag: tag.name} ), class: 'btn btn-remove has-tooltip', title: "Remove tag", data: { confirm: "Due to a Docker limitation, all tags with the same ID will also be deleted. Are you sure?" }, method: :delete do - = icon("trash cred") + - notice = 'Due to a Docker limitation, all tags with the same ID will also be deleted. Are you sure?' + = link_to namespace_project_registry_repository_tag_path(@project.namespace, @project, tag.repository, tag.name), + method: :delete, + class: 'btn btn-remove has-tooltip', + title: 'Remove image tag', + data: { confirm: notice } do + = icon('trash cred') diff --git a/config/routes/project.rb b/config/routes/project.rb index 34f4bd917f7..0269857f9fb 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -224,6 +224,12 @@ constraints(ProjectUrlConstrainer.new) do only: [:index, :destroy], constraints: { id: Gitlab::Regex.container_registry_reference_regex } + namespace :registry do + resources :repository, only: [] do + resources :tags, only: [:destroy] + end + end + resources :milestones, constraints: { id: /\d+/ } do member do put :sort_issues From 83d1fe9b5aeb947c1387666205ecaca81f2bf3a2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 15:10:15 +0200 Subject: [PATCH 56/97] Add serveral minor improvements to container registry --- .../projects/registry/repositories_controller.rb | 3 ++- app/controllers/projects/registry/tags_controller.rb | 3 ++- app/models/container_repository.rb | 3 ++- app/views/projects/registry/repositories/_image.html.haml | 6 +++++- lib/container_registry/tag.rb | 4 +--- 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb index e3e30176b30..2901d83fcef 100644 --- a/app/controllers/projects/registry/repositories_controller.rb +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -9,7 +9,8 @@ module Projects def destroy if image.destroy - redirect_to project_container_registry_path(@project) + redirect_to project_container_registry_path(@project), + notice: 'Images repository has been removed successfully!' else redirect_to project_container_registry_path(@project), alert: 'Failed to remove images repository!' diff --git a/app/controllers/projects/registry/tags_controller.rb b/app/controllers/projects/registry/tags_controller.rb index 8f0a1aff394..aab130787e9 100644 --- a/app/controllers/projects/registry/tags_controller.rb +++ b/app/controllers/projects/registry/tags_controller.rb @@ -5,7 +5,8 @@ module Projects def destroy if tag.delete - redirect_to project_container_registry_path(@project) + redirect_to project_container_registry_path(@project), + notice: 'Tag removed successfull!' else redirect_to project_container_registry_path(@project), alert: 'Failed to remove repository tag!' diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index e27369c10d6..ceb82af2e95 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -4,7 +4,8 @@ class ContainerRepository < ActiveRecord::Base validates :name, length: { minimum: 0, allow_nil: false } delegate :client, to: :registry - before_destroy :delete_tags + + before_destroy :delete_tags! def registry @registry ||= begin diff --git a/app/views/projects/registry/repositories/_image.html.haml b/app/views/projects/registry/repositories/_image.html.haml index 64b11e375c6..c3f8580d25b 100644 --- a/app/views/projects/registry/repositories/_image.html.haml +++ b/app/views/projects/registry/repositories/_image.html.haml @@ -10,7 +10,11 @@ = escape_once(image.path) = clipboard_button(clipboard_text: "docker pull #{image.path}") .controls.hidden-xs.pull-right - = link_to namespace_project_container_registry_path(@project.namespace, @project, image.id), class: 'btn btn-remove has-tooltip', title: "Remove image", data: { confirm: "Are you sure?" }, method: :delete do + = link_to namespace_project_container_registry_path(@project.namespace, @project, image), + class: 'btn btn-remove has-tooltip', + title: 'Remove repository', + data: { confirm: 'Are you sure?' }, + method: :delete do = icon("trash cred") diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index d653deb3bf1..d00e6191e7e 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -36,9 +36,7 @@ module ContainerRegistry end def digest - return @digest if defined?(@digest) - - @digest = client.repository_tag_digest(repository.path, name) + @digest ||= client.repository_tag_digest(repository.path, name) end def config_blob From 662d2e68173ecc8ee1bfe9dddf8cc38766bd310b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 10:38:37 +0200 Subject: [PATCH 57/97] Refactor feature specs for container registry --- .../registry/repositories/_tag.html.haml | 2 +- spec/features/container_registry_spec.rb | 67 ++++++++++--------- spec/support/stub_gitlab_calls.rb | 16 ++--- 3 files changed, 43 insertions(+), 42 deletions(-) diff --git a/app/views/projects/registry/repositories/_tag.html.haml b/app/views/projects/registry/repositories/_tag.html.haml index c9b3644ff93..73566689d57 100644 --- a/app/views/projects/registry/repositories/_tag.html.haml +++ b/app/views/projects/registry/repositories/_tag.html.haml @@ -29,6 +29,6 @@ = link_to namespace_project_registry_repository_tag_path(@project.namespace, @project, tag.repository, tag.name), method: :delete, class: 'btn btn-remove has-tooltip', - title: 'Remove image tag', + title: 'Remove tag', data: { confirm: notice } do = icon('trash cred') diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index 530e6af92d3..42431cbe731 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -1,59 +1,60 @@ require 'spec_helper' describe "Container Registry" do + let(:user) { create(:user) } let(:project) { create(:empty_project) } - let(:registry) { project.container_registry } - let(:tag_name) { 'latest' } - let(:tags) { [tag_name] } - let(:container_repository) { create(:container_repository) } - let(:image_name) { container_repository.name } + + let(:container_repository) do + create(:container_repository, name: 'my/image') + end before do - login_as(:user) - project.team << [@user, :developer] + login_as(user) + project.add_developer(user) stub_container_registry_config(enabled: true) - stub_container_registry_tags(*tags) - project.container_repositories << container_repository unless container_repository.nil? + stub_container_registry_tags(%w[latest]) end - describe 'GET /:project/container_registry' do - before do - visit namespace_project_container_registry_index_path(project.namespace, project) - end + context 'when there are no image repositories' do + scenario 'user visits container registry main page' do + visit_container_registry - context 'when no images' do - let(:container_repository) { } - - it { expect(page).to have_content('No container images in Container Registry for this project') } - end - - context 'when there are images' do - it { expect(page).to have_content(image_name) } + expect(page).to have_content 'No container images' end end - describe 'DELETE /:project/container_registry/:image_id' do + context 'when there are image repositories' do before do - visit namespace_project_container_registry_index_path(project.namespace, project) + project.container_repositories << container_repository end - it do + scenario 'user wants to see multi-level container repository' do + visit_container_registry + + expect(page).to have_content('my/image') + end + + scenario 'user removes entire container repository' do + visit_container_registry + expect_any_instance_of(ContainerRepository) .to receive(:delete_tags!).and_return(true) - click_on 'Remove image' - end - end - - describe 'DELETE /:project/container_registry/tag' do - before do - visit namespace_project_container_registry_index_path(project.namespace, project) + click_on 'Remove repository' end - it do - expect_any_instance_of(::ContainerRegistry::Tag).to receive(:delete).and_return(true) + scenario 'user removes a specific tag from container repository' do + visit_container_registry + + expect_any_instance_of(ContainerRegistry::Tag) + .to receive(:delete).and_return(true) click_on 'Remove tag' end end + + def visit_container_registry + visit namespace_project_container_registry_index_path( + project.namespace, project) + end end diff --git a/spec/support/stub_gitlab_calls.rb b/spec/support/stub_gitlab_calls.rb index 3949784aabb..dbf3ace37c3 100644 --- a/spec/support/stub_gitlab_calls.rb +++ b/spec/support/stub_gitlab_calls.rb @@ -32,15 +32,15 @@ module StubGitlabCalls end def stub_container_registry_tags(*tags) - allow_any_instance_of(ContainerRegistry::Client).to receive(:repository_tags).and_return( - { "tags" => tags } - ) - allow_any_instance_of(ContainerRegistry::Client).to receive(:repository_manifest).and_return( - JSON.parse(File.read(Rails.root + 'spec/fixtures/container_registry/tag_manifest.json')) - ) + allow_any_instance_of(ContainerRegistry::Client) + .to receive(:repository_tags).and_return({ 'tags' => tags }) + + allow_any_instance_of(ContainerRegistry::Client) + .to receive(:repository_manifest).and_return( + JSON.parse(File.read(Rails.root + 'spec/fixtures/container_registry/tag_manifest.json'))) + allow_any_instance_of(ContainerRegistry::Client).to receive(:blob).and_return( - File.read(Rails.root + 'spec/fixtures/container_registry/config_blob.json') - ) + File.read(Rails.root + 'spec/fixtures/container_registry/config_blob.json')) end private From 10b0fb1b4241e6c93f62a07fd1eac58835a8c7e2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 10:53:16 +0200 Subject: [PATCH 58/97] Remove redundant stubs from container image tag specs --- spec/lib/container_registry/tag_spec.rb | 35 ++++++++++++------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/spec/lib/container_registry/tag_spec.rb b/spec/lib/container_registry/tag_spec.rb index 37eaa10f4a4..bc1912d8e6c 100644 --- a/spec/lib/container_registry/tag_spec.rb +++ b/spec/lib/container_registry/tag_spec.rb @@ -5,10 +5,9 @@ describe ContainerRegistry::Tag do let(:project) { create(:project, path: 'test', group: group) } let(:repository) do - create(:container_repository, name: '', tags: %w[latest], project: project) + create(:container_repository, name: '', project: project) end - # TODO, move stubs to helper with this header let(:headers) do { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' } end @@ -41,7 +40,7 @@ describe ContainerRegistry::Tag do context 'when tag belongs to first-level repository' do let(:repository) do create(:container_repository, name: 'my_image', - tags: %w[latest], + tags: %w[tag], project: project) end @@ -158,29 +157,29 @@ describe ContainerRegistry::Tag do end end - context 'manifest digest' do + context 'with stubbed digest' do before do - stub_request(:head, 'http://registry.gitlab/v2/group/test/manifests/tag'). - with(headers: headers). - to_return(status: 200, headers: { 'Docker-Content-Digest' => 'sha256:digest' }) + stub_request(:head, 'http://registry.gitlab/v2/group/test/manifests/tag') + .with(headers: headers) + .to_return(status: 200, headers: { 'Docker-Content-Digest' => 'sha256:digest' }) end - context '#digest' do - subject { tag.digest } - - it { is_expected.to eq('sha256:digest') } + describe '#digest' do + it 'returns a correct tag digest' do + expect(tag.digest).to eq 'sha256:digest' + end end - context '#delete' do + describe '#delete' do before do - stub_request(:delete, 'http://registry.gitlab/v2/group/test/manifests/sha256:digest'). - with(headers: headers). - to_return(status: 200) + stub_request(:delete, 'http://registry.gitlab/v2/group/test/manifests/sha256:digest') + .with(headers: headers) + .to_return(status: 200) end - subject { tag.delete } - - it { is_expected.to be_truthy } + it 'correctly deletes the tag' do + expect(tag.delete).to be_truthy + end end end end From 01280a5ad56f67ae653dade815faa5649bcee81f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 10:57:12 +0200 Subject: [PATCH 59/97] Add missing test example for container repository tags --- app/models/container_repository.rb | 2 -- spec/models/container_repository_spec.rb | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index ceb82af2e95..ab04258ab8d 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -43,8 +43,6 @@ class ContainerRepository < ActiveRecord::Base ContainerRegistry::Blob.new(self, config) end - # TODO, specs needed - # def has_tags? tags.to_a.any? end diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index 884eac43719..f794085fc45 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -57,6 +57,12 @@ describe ContainerRepository do it { is_expected.not_to be_empty } end + describe '#has_tags?' do + it 'has tags' do + expect(container_repository).to have_tags + end + end + describe '#delete_tags!' do let(:container_repository) do create(:container_repository, name: 'my_image', From 1a47986b3d7cd8e6d5bdbfbc20a841cf5586f773 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 11:38:39 +0200 Subject: [PATCH 60/97] Check registry repository name against regexp This regexp is extracted from Docker Distribution 2.4.1 docs, contains additional `/` element that can be a separator of components. --- lib/container_registry/path.rb | 4 +++- lib/gitlab/regex.rb | 7 +++++++ spec/lib/container_registry/path_spec.rb | 24 +++++++++++++++--------- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 27e0e7897ff..6e8d62b77c7 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -22,7 +22,9 @@ module ContainerRegistry end def valid? - @nodes.size > 1 && @nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED + @path =~ Gitlab::Regex.container_repository_name_regex && + @nodes.size > 1 && + @nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED end def components diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 5e5f5ff1589..e599dd4a656 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -121,6 +121,13 @@ module Gitlab git_reference_regex end + ## + # Docker Distribution Registry 2.4.1 repository name rules + # + def container_repository_name_regex + @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z} + end + def environment_name_regex @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze end diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index 68732b12542..1973da65f0a 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -36,25 +36,31 @@ describe ContainerRegistry::Path do context 'when path has less than two components' do let(:path) { 'something/' } - it 'is not valid' do - expect(subject).not_to be_valid - end + it { is_expected.not_to be_valid } end context 'when path has more than allowed number of components' do let(:path) { 'a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/r/s/t/u/w/y/z' } - it 'is not valid' do - expect(subject).not_to be_valid - end + it { is_expected.not_to be_valid } + end + + context 'when path has invalid characters' do + let(:path) { 'some\path' } + + it { is_expected.not_to be_valid } end context 'when path has two or more components' do let(:path) { 'some/path' } - it 'is valid' do - expect(subject).to be_valid - end + it { is_expected.to be_valid } + end + + context 'when path is related to multi-level image' do + let(:path) { 'some/path/my/image' } + + it { is_expected.to be_valid } end end From e10dae3e3c2a5fb6077244c72e44c912b7281349 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 11:42:37 +0200 Subject: [PATCH 61/97] Improve code in container repository path class --- lib/container_registry/path.rb | 23 +++++++++++++---------- spec/lib/container_registry/path_spec.rb | 8 ++++++++ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 6e8d62b77c7..f76c6489381 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -14,24 +14,23 @@ module ContainerRegistry def initialize(path) @path = path - @nodes = path.to_s.split('/') - end - - def to_s - @path end def valid? @path =~ Gitlab::Regex.container_repository_name_regex && - @nodes.size > 1 && - @nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED + nodes.size > 1 && + nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED + end + + def nodes + @nodes ||= @path.to_s.split('/') end def components raise InvalidRegistryPathError unless valid? - @components ||= @nodes.size.downto(2).map do |length| - @nodes.take(length).join('/') + @components ||= nodes.size.downto(2).map do |length| + nodes.take(length).join('/') end end @@ -51,7 +50,7 @@ module ContainerRegistry end def repository_project - @project ||= Project.where_full_path_in(components.first(3))&.first + @project ||= Project.where_full_path_in(components.first(3)).first end def repository_name @@ -59,5 +58,9 @@ module ContainerRegistry @path.remove(%r(^?#{Regexp.escape(repository_project.full_path)}/?)) end + + def to_s + @path + end end end diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index 1973da65f0a..825c61beeb3 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -3,6 +3,14 @@ require 'spec_helper' describe ContainerRegistry::Path do subject { described_class.new(path) } + describe '#nodes' do + let(:path) { 'path/to/some/project' } + + it 'splits elements by a forward slash' do + expect(subject.nodes).to eq %w[path to some project] + end + end + describe '#components' do context 'when repository path is valid' do let(:path) { 'path/to/some/project' } From 6fefa794304a6233368b592422ea0fb71a2700f0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 11:51:13 +0200 Subject: [PATCH 62/97] Validate uniqueness of container repository name --- app/models/container_repository.rb | 1 + spec/models/container_repository_spec.rb | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index ab04258ab8d..0cf781799e9 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -2,6 +2,7 @@ class ContainerRepository < ActiveRecord::Base belongs_to :project validates :name, length: { minimum: 0, allow_nil: false } + validates :name, uniqueness: { scope: :project_id } delegate :client, to: :registry diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index f794085fc45..503da6556c0 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -21,6 +21,13 @@ describe ContainerRepository do headers: { 'Content-Type' => 'application/json' }) end + describe 'validations' do + it 'validates uniqueness of name scoped to project' do + expect(subject).to validate_uniqueness_of(:name) + .scoped_to(:project_id) + end + end + describe 'associations' do it 'belongs to the project' do expect(container_repository).to belong_to(:project) From fd30b3d4972180b6e91d0180b140f32622c48b1d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 12:49:54 +0200 Subject: [PATCH 63/97] Ensure root container repository when visiting registry Root container repository is a images repository that had been created before 9.1, before we introduced multi-level images support. --- .../registry/repositories_controller.rb | 16 ++++ app/models/container_repository.rb | 12 ++- .../registry/repositories_controller_spec.rb | 82 +++++++++++++++++++ spec/factories/container_repositories.rb | 4 + spec/models/container_repository_spec.rb | 37 +++++++++ 5 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 spec/controllers/projects/registry/repositories_controller_spec.rb diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb index 2901d83fcef..ab4344bcf3c 100644 --- a/app/controllers/projects/registry/repositories_controller.rb +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -2,6 +2,7 @@ module Projects module Registry class RepositoriesController < ::Projects::Registry::ApplicationController before_action :authorize_update_container_image!, only: [:destroy] + before_action :ensure_root_container_repository!, only: [:index] def index @images = project.container_repositories @@ -22,6 +23,21 @@ module Projects def image @image ||= project.container_repositories.find_by(id: params[:id]) end + + ## + # Container repository object for root project path. + # + # Needed to maintain a backwards compatibility. + # + def ensure_root_container_repository! + ContainerRegistry::Path.new(@project.full_path).tap do |path| + return if path.has_repository? + + ContainerRepository.build_from_path(path).tap do |repository| + repository.save if repository.has_tags? + end + end + end end end end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 0cf781799e9..36158d75ae8 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -48,6 +48,10 @@ class ContainerRepository < ActiveRecord::Base tags.to_a.any? end + def root_repository? + name.empty? + end + def delete_tags! return unless has_tags? @@ -58,8 +62,12 @@ class ContainerRepository < ActiveRecord::Base end end + def self.build_from_path(path) + self.new(project: path.repository_project, + name: path.repository_name) + end + def self.create_from_path!(path) - self.create(project: path.repository_project, - name: path.repository_name) + build_from_path(path).tap(&:save!) end end diff --git a/spec/controllers/projects/registry/repositories_controller_spec.rb b/spec/controllers/projects/registry/repositories_controller_spec.rb new file mode 100644 index 00000000000..e514f535f1d --- /dev/null +++ b/spec/controllers/projects/registry/repositories_controller_spec.rb @@ -0,0 +1,82 @@ +require 'spec_helper' + +describe Projects::Registry::RepositoriesController do + let(:user) { create(:user) } + let(:project) { create(:empty_project, :private) } + + before do + sign_in(user) + end + + context 'when user has access to registry' do + before do + project.add_developer(user) + end + + describe 'GET index' do + context 'when root container repository exists' do + before do + create(:container_repository, :root, project: project) + end + + it 'does not create root container repository' do + expect { go_to_index }.not_to change { ContainerRepository.all.count } + end + end + + context 'when root container repository does not exist' do + context 'when there are tags for this repository' do + before do + stub_container_registry_tags(%w[rc1 latest]) + end + + it 'successfully renders container repositories' do + go_to_index + + expect(response).to have_http_status(:ok) + end + + it 'creates a root container repository' do + expect { go_to_index }.to change { ContainerRepository.all.count }.by(1) + expect(ContainerRepository.first).to be_root_repository + end + end + + context 'when there are no tags for this repository' do + before do + stub_container_registry_tags(*[]) + end + + it 'successfully renders container repositories' do + go_to_index + + expect(response).to have_http_status(:ok) + end + + it 'does not ensure root container repository' do + expect { go_to_index }.not_to change { ContainerRepository.all.count } + end + end + end + end + end + + context 'when user does not have access to registry' do + describe 'GET index' do + it 'responds with 404' do + go_to_index + + expect(response).to have_http_status(:not_found) + end + + it 'does not ensure root container repository' do + expect { go_to_index }.not_to change { ContainerRepository.all.count } + end + end + end + + def go_to_index + get :index, namespace_id: project.namespace, + project_id: project + end +end diff --git a/spec/factories/container_repositories.rb b/spec/factories/container_repositories.rb index 4919a03cdf2..3fcad9fd4b3 100644 --- a/spec/factories/container_repositories.rb +++ b/spec/factories/container_repositories.rb @@ -7,6 +7,10 @@ FactoryGirl.define do tags [] end + trait :root do + name '' + end + after(:build) do |repository, evaluator| next if evaluator.tags.to_a.none? diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index 503da6556c0..b23ed8e30b6 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -98,6 +98,43 @@ describe ContainerRepository do end end + describe '#root_repository?' do + context 'when repository is a root repository' do + let(:repository) { create(:container_repository, :root) } + + it 'returns true' do + expect(repository).to be_root_repository + end + end + + context 'when repository is not a root repository' do + it 'returns false' do + expect(container_repository).not_to be_root_repository + end + end + end + + describe '.build_from_path' do + let(:path) { project.full_path + '/some/image' } + let(:repository_path) { ContainerRegistry::Path.new(path) } + + let(:repository) do + described_class.build_from_path(ContainerRegistry::Path.new(path)) + end + + it 'fabricates repository assigned to a correct project' do + expect(repository.project).to eq project + end + + it 'fabricates repository with a correct name' do + expect(repository.name).to eq 'some/image' + end + + it 'is not persisted' do + expect(repository).not_to be_persisted + end + end + describe '.create_from_path!' do let(:repository) do described_class.create_from_path!(ContainerRegistry::Path.new(path)) From 0af4cbc57266cd7d2c433442c537da5c8970a3da Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 13:05:46 +0200 Subject: [PATCH 64/97] Simplify container repository build method specs --- spec/models/container_repository_spec.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index b23ed8e30b6..1a29cc9a096 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -115,11 +115,12 @@ describe ContainerRepository do end describe '.build_from_path' do - let(:path) { project.full_path + '/some/image' } - let(:repository_path) { ContainerRegistry::Path.new(path) } + let(:registry_path) do + ContainerRegistry::Path.new(project.full_path + '/some/image') + end let(:repository) do - described_class.build_from_path(ContainerRegistry::Path.new(path)) + described_class.build_from_path(registry_path) end it 'fabricates repository assigned to a correct project' do From baa00d542478759be225a45dc805d0314e1921d2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 15:52:24 +0200 Subject: [PATCH 65/97] Refactor container registry repository tag stubs --- .../registry/repositories_controller_spec.rb | 7 +++-- spec/features/container_registry_spec.rb | 3 +- .../security/project/internal_access_spec.rb | 2 +- .../security/project/private_access_spec.rb | 2 +- .../security/project/public_access_spec.rb | 2 +- spec/models/namespace_spec.rb | 2 +- spec/models/project_spec.rb | 2 +- .../services/projects/destroy_service_spec.rb | 2 +- .../projects/transfer_service_spec.rb | 2 +- spec/support/stub_gitlab_calls.rb | 30 ++++++++++++++----- 10 files changed, 36 insertions(+), 18 deletions(-) diff --git a/spec/controllers/projects/registry/repositories_controller_spec.rb b/spec/controllers/projects/registry/repositories_controller_spec.rb index e514f535f1d..29f0a65483f 100644 --- a/spec/controllers/projects/registry/repositories_controller_spec.rb +++ b/spec/controllers/projects/registry/repositories_controller_spec.rb @@ -24,10 +24,11 @@ describe Projects::Registry::RepositoriesController do end end - context 'when root container repository does not exist' do + context 'when root container repository is not created' do context 'when there are tags for this repository' do before do - stub_container_registry_tags(%w[rc1 latest]) + stub_container_registry_tags(repository: project.full_path, + tags: %w[rc1 latest]) end it 'successfully renders container repositories' do @@ -44,7 +45,7 @@ describe Projects::Registry::RepositoriesController do context 'when there are no tags for this repository' do before do - stub_container_registry_tags(*[]) + stub_container_registry_tags(repository: :any, tags: []) end it 'successfully renders container repositories' do diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index 42431cbe731..fa7adbe71ea 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -12,7 +12,7 @@ describe "Container Registry" do login_as(user) project.add_developer(user) stub_container_registry_config(enabled: true) - stub_container_registry_tags(%w[latest]) + stub_container_registry_tags(repository: :any, tags: []) end context 'when there are no image repositories' do @@ -25,6 +25,7 @@ describe "Container Registry" do context 'when there are image repositories' do before do + stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest]) project.container_repositories << container_repository end diff --git a/spec/features/security/project/internal_access_spec.rb b/spec/features/security/project/internal_access_spec.rb index a961d8b4f69..6ecdc8cbb71 100644 --- a/spec/features/security/project/internal_access_spec.rb +++ b/spec/features/security/project/internal_access_spec.rb @@ -446,7 +446,7 @@ describe "Internal Project Access", feature: true do let(:container_repository) { create(:container_repository) } before do - stub_container_registry_tags('latest') + stub_container_registry_tags(repository: :any, tags: ['latest']) stub_container_registry_config(enabled: true) project.container_repositories << container_repository end diff --git a/spec/features/security/project/private_access_spec.rb b/spec/features/security/project/private_access_spec.rb index b7e42e67d82..a8fc0624588 100644 --- a/spec/features/security/project/private_access_spec.rb +++ b/spec/features/security/project/private_access_spec.rb @@ -435,7 +435,7 @@ describe "Private Project Access", feature: true do let(:container_repository) { create(:container_repository) } before do - stub_container_registry_tags('latest') + stub_container_registry_tags(repository: :any, tags: ['latest']) stub_container_registry_config(enabled: true) project.container_repositories << container_repository end diff --git a/spec/features/security/project/public_access_spec.rb b/spec/features/security/project/public_access_spec.rb index 02660984b29..08cf2bf5291 100644 --- a/spec/features/security/project/public_access_spec.rb +++ b/spec/features/security/project/public_access_spec.rb @@ -446,7 +446,7 @@ describe "Public Project Access", feature: true do let(:container_repository) { create(:container_repository) } before do - stub_container_registry_tags('latest') + stub_container_registry_tags(repository: :any, tags:['latest']) stub_container_registry_config(enabled: true) project.container_repositories << container_repository end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index e1bfd20a6aa..197fd558dd7 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -153,7 +153,7 @@ describe Namespace, models: true do before do stub_container_registry_config(enabled: true) - stub_container_registry_tags('tag') + stub_container_registry_tags(repository: :any, tags: ['tag']) create(:empty_project, namespace: @namespace, container_repositories: [container_repository]) diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 841c7d4cb5b..062d7fdd4ae 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1190,7 +1190,7 @@ describe Project, models: true do before do stub_container_registry_config(enabled: true) - stub_container_registry_tags('tag') + stub_container_registry_tags(repository: :any, tags: ['tag']) project.container_repositories << container_repository end diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index cf1f90becfd..5ef07c8275e 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -94,7 +94,7 @@ describe Projects::DestroyService, services: true do before do stub_container_registry_config(enabled: true) - stub_container_registry_tags('tag') + stub_container_registry_tags(repository: :any, tags: ['tag']) project.container_repositories << container_repository end diff --git a/spec/services/projects/transfer_service_spec.rb b/spec/services/projects/transfer_service_spec.rb index 81e15f9dba6..29ccce59c53 100644 --- a/spec/services/projects/transfer_service_spec.rb +++ b/spec/services/projects/transfer_service_spec.rb @@ -33,7 +33,7 @@ describe Projects::TransferService, services: true do before do stub_container_registry_config(enabled: true) - stub_container_registry_tags('tag') + stub_container_registry_tags(repository: :any, tags: ['tag']) project.container_repositories << container_repository end diff --git a/spec/support/stub_gitlab_calls.rb b/spec/support/stub_gitlab_calls.rb index dbf3ace37c3..ded2d593059 100644 --- a/spec/support/stub_gitlab_calls.rb +++ b/spec/support/stub_gitlab_calls.rb @@ -31,20 +31,36 @@ module StubGitlabCalls .to receive(:full_access_token).and_return('token') end - def stub_container_registry_tags(*tags) - allow_any_instance_of(ContainerRegistry::Client) - .to receive(:repository_tags).and_return({ 'tags' => tags }) + def stub_container_registry_tags(repository: :any, tags:) + repository = any_args if repository == :any allow_any_instance_of(ContainerRegistry::Client) - .to receive(:repository_manifest).and_return( - JSON.parse(File.read(Rails.root + 'spec/fixtures/container_registry/tag_manifest.json'))) + .to receive(:repository_tags).with(repository) + .and_return({ 'tags' => tags }) - allow_any_instance_of(ContainerRegistry::Client).to receive(:blob).and_return( - File.read(Rails.root + 'spec/fixtures/container_registry/config_blob.json')) + allow_any_instance_of(ContainerRegistry::Client) + .to receive(:repository_manifest).with(repository) + .and_return(stub_container_registry_tag_manifest) + + allow_any_instance_of(ContainerRegistry::Client) + .to receive(:blob).with(repository) + .and_return(stub_container_registry_blob) end private + def stub_container_registry_tag_manifest + fixture_path = 'spec/fixtures/container_registry/tag_manifest.json' + + JSON.parse(File.read(Rails.root + fixture_path)) + end + + def stub_container_registry_blob + fixture_path = 'spec/fixtures/container_registry/config_blob.json' + + File.read(Rails.root + fixture_path) + end + def gitlab_url Gitlab.config.gitlab.url end From fc5eb3157082d923b80596485b302504ce0671ea Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 15:53:51 +0200 Subject: [PATCH 66/97] Fix Rubocop offenses in code related to the registry --- app/controllers/projects/registry/repositories_controller.rb | 2 +- spec/features/security/project/public_access_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb index ab4344bcf3c..cb1ea45dd04 100644 --- a/app/controllers/projects/registry/repositories_controller.rb +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -31,7 +31,7 @@ module Projects # def ensure_root_container_repository! ContainerRegistry::Path.new(@project.full_path).tap do |path| - return if path.has_repository? + break if path.has_repository? ContainerRepository.build_from_path(path).tap do |repository| repository.save if repository.has_tags? diff --git a/spec/features/security/project/public_access_spec.rb b/spec/features/security/project/public_access_spec.rb index 08cf2bf5291..c4d2f50ca14 100644 --- a/spec/features/security/project/public_access_spec.rb +++ b/spec/features/security/project/public_access_spec.rb @@ -446,7 +446,7 @@ describe "Public Project Access", feature: true do let(:container_repository) { create(:container_repository) } before do - stub_container_registry_tags(repository: :any, tags:['latest']) + stub_container_registry_tags(repository: :any, tags: ['latest']) stub_container_registry_config(enabled: true) project.container_repositories << container_repository end From a9b221d91a5529c514615b640cdbbaf6b99bf790 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 16:29:11 +0200 Subject: [PATCH 67/97] Refine method for checking project registry tags --- app/models/project.rb | 20 +++++++++++++ spec/models/project_spec.rb | 60 +++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/app/models/project.rb b/app/models/project.rb index 649d0e100c3..0adec807f34 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -406,6 +406,11 @@ class Project < ActiveRecord::Base end end + def has_container_registry_tags? + container_repositories.to_a.any?(&:has_tags?) || + has_root_container_repository_tags? + end + def commit(ref = 'HEAD') repository.commit(ref) end @@ -1373,4 +1378,19 @@ class Project < ActiveRecord::Base Project.unscoped.where(pending_delete: true).find_by_full_path(path_with_namespace) end + + ## + # This method is here because of support for legacy container repository + # which has exactly the same path like project does, but which might not be + # persisted in `container_repositories` table. + # + def has_root_container_repository_tags? + return false unless Gitlab.config.registry.enabled + + ContainerRegistry::Path.new(self.full_path).tap do |path| + ContainerRepository.build_from_path(path).tap do |repository| + return repository.has_tags? + end + end + end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 062d7fdd4ae..4c13e53d831 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1414,6 +1414,66 @@ describe Project, models: true do end end + describe '#has_container_registry_tags?' do + let(:project) { create(:empty_project) } + + context 'when container registry is enabled' do + before { stub_container_registry_config(enabled: true) } + + context 'when tags are present for multi-level registries' do + before do + create(:container_repository, project: project, name: 'image') + + stub_container_registry_tags(repository: /image/, + tags: %w[latest rc1]) + end + + it 'should have image tags' do + expect(project).to have_container_registry_tags + end + end + + context 'when tags are present for root repository' do + before do + stub_container_registry_tags(repository: project.full_path, + tags: %w[latest rc1 pre1]) + end + + it 'should have image tags' do + expect(project).to have_container_registry_tags + end + end + + context 'when there are no tags at all' do + before do + stub_container_registry_tags(repository: :any, tags: []) + end + + it 'should not have image tags' do + expect(project).not_to have_container_registry_tags + end + end + end + + context 'when container registry is disabled' do + before { stub_container_registry_config(enabled: false) } + + it 'should not have image tags' do + expect(project).not_to have_container_registry_tags + end + + it 'should not check root repository tags' do + expect(project).not_to receive(:full_path) + expect(project).not_to have_container_registry_tags + end + + it 'should iterate through container repositories' do + expect(project).to receive(:container_repositories) + expect(project).not_to have_container_registry_tags + end + end + end + describe '#latest_successful_builds_for' do def create_pipeline(status = 'success') create(:ci_pipeline, project: project, From e9d5e95c4403d39072b6d29555569b2d09b02fe6 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 16:34:18 +0200 Subject: [PATCH 68/97] Revert changes in services related to moving projects --- app/models/namespace.rb | 2 +- app/models/project.rb | 4 ++-- app/services/projects/transfer_service.rb | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 2c3473f9186..b57ed258af7 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -150,7 +150,7 @@ class Namespace < ActiveRecord::Base end def any_project_has_container_registry_images? - projects.joins(:container_repositories).any? + projects.any?(&:has_container_registry_tags?) end def send_update_instructions diff --git a/app/models/project.rb b/app/models/project.rb index 0adec807f34..fa64ccbf7e4 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -894,8 +894,8 @@ class Project < ActiveRecord::Base expire_caches_before_rename(old_path_with_namespace) - if container_repositories.present? - Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry images are present" + if has_container_registry_tags? + Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" # we currently doesn't support renaming repository if it contains images in container registry raise StandardError.new('Project cannot be renamed, because images are present in its container registry') diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index f46bf884e37..c2c2addeebe 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -36,9 +36,9 @@ module Projects raise TransferError.new("Project with same path in target namespace already exists") end - unless project.container_repositories.empty? + if project.has_container_registry_tags? # we currently doesn't support renaming repository if it contains images in container registry - raise TransferError.new('Project cannot be transferred, because images are present in its container registry') + raise TransferError.new('Project cannot be transferred, because tags are present in its container registry') end project.expire_caches_before_rename(old_path) From 325908ad3931f12dcada654d164ae09ff5f00c6e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 16:38:51 +0200 Subject: [PATCH 69/97] Remove changes unnecessary changes from namespace model --- app/models/namespace.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/namespace.rb b/app/models/namespace.rb index b57ed258af7..1d4b1f7d590 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -113,8 +113,8 @@ class Namespace < ActiveRecord::Base end def move_dir - if any_project_has_container_registry_images? - raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has images in container registry') + if any_project_has_container_registry_tags? + raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry') end # Move the namespace directory in all storages paths used by member projects @@ -149,7 +149,7 @@ class Namespace < ActiveRecord::Base end end - def any_project_has_container_registry_images? + def any_project_has_container_registry_tags? projects.any?(&:has_container_registry_tags?) end From d2a9c5d81fce78540cf2c2a70f020fc0b234ce5a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 16:41:56 +0200 Subject: [PATCH 70/97] Remove unecessary changes from project transfer service --- app/services/projects/transfer_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index c2c2addeebe..da6e6acd4a7 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -37,7 +37,7 @@ module Projects end if project.has_container_registry_tags? - # we currently doesn't support renaming repository if it contains images in container registry + # we currently doesn't support renaming repository if it contains tags in container registry raise TransferError.new('Project cannot be transferred, because tags are present in its container registry') end From 1d6ddb05bb12dce9e90c76b73a22a21082506091 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 21:25:52 +0200 Subject: [PATCH 71/97] Fix namespace specs related to container registry --- spec/models/namespace_spec.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 197fd558dd7..a7e565ec645 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -161,7 +161,9 @@ describe Namespace, models: true do allow(@namespace).to receive(:path).and_return('new_path') end - it { expect { @namespace.move_dir }.to raise_error('Namespace cannot be moved, because at least one project has images in container registry') } + it 'raises an error about not movable project' do + expect { @namespace.move_dir }.to raise_error(/Namespace cannot be moved/) + end end context 'renaming a sub-group' do From 4fab9f24c07af110441a6db6d4f4d26197c155b7 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 21:27:08 +0200 Subject: [PATCH 72/97] Fix documentation related to container registry --- doc/user/project/container_registry.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/user/project/container_registry.md b/doc/user/project/container_registry.md index 7524e70957f..6de75e43ed3 100644 --- a/doc/user/project/container_registry.md +++ b/doc/user/project/container_registry.md @@ -85,8 +85,7 @@ and click **Registry** in the project menu. This view will show you all tags in your project and will easily allow you to delete them. -![Container Registry panel](image-needs-update) -[//]: # (img/container_registry_panel.png) +![Container Registry panel](img/container_registry_panel.png) ## Build and push images using GitLab CI From 7e46d0a95fc4ce8d0d66e75e6b1c4ef473606e79 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 11:13:12 +0200 Subject: [PATCH 73/97] Fix container registry controller specs --- .../projects/registry/repositories_controller_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/controllers/projects/registry/repositories_controller_spec.rb b/spec/controllers/projects/registry/repositories_controller_spec.rb index 29f0a65483f..464302824a8 100644 --- a/spec/controllers/projects/registry/repositories_controller_spec.rb +++ b/spec/controllers/projects/registry/repositories_controller_spec.rb @@ -6,6 +6,7 @@ describe Projects::Registry::RepositoriesController do before do sign_in(user) + stub_container_registry_config(enabled: true) end context 'when user has access to registry' do From 008441c49e493b885586ba38993f54bdcb03799c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 11:46:14 +0200 Subject: [PATCH 74/97] Improve container registry repository view partials --- .../registry/repositories/_image.html.haml | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/app/views/projects/registry/repositories/_image.html.haml b/app/views/projects/registry/repositories/_image.html.haml index c3f8580d25b..e0d74789207 100644 --- a/app/views/projects/registry/repositories/_image.html.haml +++ b/app/views/projects/registry/repositories/_image.html.haml @@ -1,29 +1,21 @@ -- expanded = false .container-image.js-toggle-container .container-image-head = link_to "#", class: "js-toggle-button" do - - if expanded - = icon("chevron-up") - - else - = icon("chevron-down") - + = icon("chevron-down") = escape_once(image.path) + = clipboard_button(clipboard_text: "docker pull #{image.path}") + .controls.hidden-xs.pull-right = link_to namespace_project_container_registry_path(@project.namespace, @project, image), class: 'btn btn-remove has-tooltip', title: 'Remove repository', data: { confirm: 'Are you sure?' }, method: :delete do - = icon("trash cred") + = icon('trash cred') - - .container-image-tags.js-toggle-content{ class: ("hide" unless expanded) } - - if image.tags.blank? - %li - .nothing-here-block No tags in Container Registry for this container image. - - - else + .container-image-tags.js-toggle-content{ class: 'hide' } + - if image.has_tags? .table-holder %table.table.tags %thead @@ -34,5 +26,8 @@ %th Created - if can?(current_user, :update_container_image, @project) %th - = render partial: 'tag', collection: image.tags + - else + %li + .nothing-here-block No tags in Container Registry for this container image. + From 59b5843eb0d329f314a58c074c42ecaa4bd24f25 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 11:50:26 +0200 Subject: [PATCH 75/97] Improve wording in registry notifications in the UI --- app/controllers/projects/registry/repositories_controller.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb index cb1ea45dd04..2e18075d40f 100644 --- a/app/controllers/projects/registry/repositories_controller.rb +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -11,10 +11,10 @@ module Projects def destroy if image.destroy redirect_to project_container_registry_path(@project), - notice: 'Images repository has been removed successfully!' + notice: 'Image repository has been removed successfully!' else redirect_to project_container_registry_path(@project), - alert: 'Failed to remove images repository!' + alert: 'Failed to remove image repository!' end end From 7cf91f7b78225957d8b79e96f7d0080cdd15c0cd Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 11:52:56 +0200 Subject: [PATCH 76/97] Improve wording in partials related to the registry --- app/views/projects/registry/repositories/_tag.html.haml | 3 +-- app/views/projects/registry/repositories/index.html.haml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/views/projects/registry/repositories/_tag.html.haml b/app/views/projects/registry/repositories/_tag.html.haml index 73566689d57..ee1ec0e8f9a 100644 --- a/app/views/projects/registry/repositories/_tag.html.haml +++ b/app/views/projects/registry/repositories/_tag.html.haml @@ -25,10 +25,9 @@ - if can?(current_user, :update_container_image, @project) %td.content .controls.hidden-xs.pull-right - - notice = 'Due to a Docker limitation, all tags with the same ID will also be deleted. Are you sure?' = link_to namespace_project_registry_repository_tag_path(@project.namespace, @project, tag.repository, tag.name), method: :delete, class: 'btn btn-remove has-tooltip', title: 'Remove tag', - data: { confirm: notice } do + data: { confirm: 'Are you sure you want to delete this tag?' } do = icon('trash cred') diff --git a/app/views/projects/registry/repositories/index.html.haml b/app/views/projects/registry/repositories/index.html.haml index 1b5d000e801..be128e92fa7 100644 --- a/app/views/projects/registry/repositories/index.html.haml +++ b/app/views/projects/registry/repositories/index.html.haml @@ -20,7 +20,7 @@ docker push #{escape_once(@project.container_registry_url)}/image - if @images.blank? - .nothing-here-block No container images in Container Registry for this project. + .nothing-here-block No container image repositories in Container Registry for this project. - else = render partial: 'image', collection: @images From 97c6cf59b361a0fe1b22cc999a2b2dd64b0a30f3 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 11:56:32 +0200 Subject: [PATCH 77/97] Swap method names in containe registry path class --- lib/container_registry/path.rb | 18 +++++++++--------- spec/lib/container_registry/path_spec.rb | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index f76c6489381..2291291d1fc 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -18,19 +18,19 @@ module ContainerRegistry def valid? @path =~ Gitlab::Regex.container_repository_name_regex && - nodes.size > 1 && - nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED - end - - def nodes - @nodes ||= @path.to_s.split('/') + components.size > 1 && + components.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED end def components + @components ||= @path.to_s.split('/') + end + + def nodes raise InvalidRegistryPathError unless valid? - @components ||= nodes.size.downto(2).map do |length| - nodes.take(length).join('/') + @nodes ||= components.size.downto(2).map do |length| + components.take(length).join('/') end end @@ -50,7 +50,7 @@ module ContainerRegistry end def repository_project - @project ||= Project.where_full_path_in(components.first(3)).first + @project ||= Project.where_full_path_in(nodes.first(3)).first end def repository_name diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index 825c61beeb3..b9c4572c269 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -3,22 +3,22 @@ require 'spec_helper' describe ContainerRegistry::Path do subject { described_class.new(path) } - describe '#nodes' do + describe '#components' do let(:path) { 'path/to/some/project' } - it 'splits elements by a forward slash' do - expect(subject.nodes).to eq %w[path to some project] + it 'splits components by a forward slash' do + expect(subject.components).to eq %w[path to some project] end end - describe '#components' do + describe '#nodes' do context 'when repository path is valid' do let(:path) { 'path/to/some/project' } - it 'return all project-like components in reverse order' do - expect(subject.components).to eq %w[path/to/some/project - path/to/some - path/to] + it 'return all project path like node in reverse order' do + expect(subject.nodes).to eq %w[path/to/some/project + path/to/some + path/to] end end @@ -26,7 +26,7 @@ describe ContainerRegistry::Path do let(:path) { '' } it 'rasises en error' do - expect { subject.components } + expect { subject.nodes } .to raise_error described_class::InvalidRegistryPathError end end From 1c91d52a70407e15f9b106bafc6505895214f3b8 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 11:58:15 +0200 Subject: [PATCH 78/97] Remove unneeded char in registry repository path --- lib/container_registry/path.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 2291291d1fc..89973b2e7b8 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -56,7 +56,7 @@ module ContainerRegistry def repository_name return unless has_project? - @path.remove(%r(^?#{Regexp.escape(repository_project.full_path)}/?)) + @path.remove(%r(^#{Regexp.escape(repository_project.full_path)}/?)) end def to_s From cb2ce8452fe2e9e156add5ccfe8fd2ec5cda9ace Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 12:57:38 +0200 Subject: [PATCH 79/97] Remove legacy registry tags when deleting a project --- app/models/container_repository.rb | 4 + app/services/projects/destroy_service.rb | 23 +++++- spec/models/container_repository_spec.rb | 18 +++++ .../services/projects/destroy_service_spec.rb | 78 +++++++++++++------ 4 files changed, 97 insertions(+), 26 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 36158d75ae8..463eb5b7d69 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -70,4 +70,8 @@ class ContainerRepository < ActiveRecord::Base def self.create_from_path!(path) build_from_path(path).tap(&:save!) end + + def self.build_root_repository(project) + self.new(project: project, name: '') + end end diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index 4e1964f79dd..a47e74ba9b0 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -29,15 +29,20 @@ module Projects Project.transaction do project.team.truncate - project.destroy! + + unless remove_legacy_registry_tags + raise_error('Failed to remove some tags in project container registry. Please try again or contact administrator.') + end unless remove_repository(repo_path) - raise_error('Failed to remove project repository. Please try again or contact administrator') + raise_error('Failed to remove project repository. Please try again or contact administrator.') end unless remove_repository(wiki_path) - raise_error('Failed to remove wiki repository. Please try again or contact administrator') + raise_error('Failed to remove wiki repository. Please try again or contact administrator.') end + + project.destroy! end log_info("Project \"#{project.path_with_namespace}\" was removed") @@ -64,6 +69,18 @@ module Projects end end + ## + # This method makes sure that we correctly remove registry tags + # for legacy image repository (when repository path equals project path). + # + def remove_legacy_registry_tags + return true unless Gitlab.config.registry.enabled + + ContainerRepository.build_root_repository(project).tap do |repository| + return repository.delete_tags! if repository.has_tags? + end + end + def raise_error(message) raise DestroyError.new(message) end diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index 1a29cc9a096..3e6082ec326 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -195,4 +195,22 @@ describe ContainerRepository do end end end + + describe '.build_root_repository' do + let(:repository) do + described_class.build_root_repository(project) + end + + it 'fabricates a root repository object' do + expect(repository).to be_root_repository + end + + it 'assignes it to the correct project' do + expect(repository.project).to eq project + end + + it 'does not persist it' do + expect(repository).not_to be_persisted + end + end end diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index 5ef07c8275e..4b8589b2736 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -7,6 +7,11 @@ describe Projects::DestroyService, services: true do let!(:remove_path) { path.sub(/\.git\Z/, "+#{project.id}+deleted.git") } let!(:async) { false } # execute or async_execute + before do + stub_container_registry_config(enabled: true) + stub_container_registry_tags(repository: :any, tags: []) + end + shared_examples 'deleting the project' do it 'deletes the project' do expect(Project.unscoped.all).not_to include(project) @@ -89,37 +94,64 @@ describe Projects::DestroyService, services: true do it_behaves_like 'deleting the project with pipeline and build' end - context 'container registry' do - let(:container_repository) { create(:container_repository) } + describe 'container registry' do + context 'when there are regular container repositories' do + let(:container_repository) { create(:container_repository) } - before do - stub_container_registry_config(enabled: true) - stub_container_registry_tags(repository: :any, tags: ['tag']) - project.container_repositories << container_repository - end - - context 'images deletion succeeds' do - it do - expect_any_instance_of(ContainerRepository) - .to receive(:delete_tags!).and_return(true) - - destroy_project(project, user, {}) - end - end - - context 'images deletion fails' do before do - expect_any_instance_of(ContainerRepository) - .to receive(:delete_tags!).and_return(false) + stub_container_registry_tags(repository: project.full_path + '/image', + tags: ['tag']) + project.container_repositories << container_repository end - subject { destroy_project(project, user, {}) } + context 'when image repository deletion succeeds' do + it 'removes tags' do + expect_any_instance_of(ContainerRepository) + .to receive(:delete_tags!).and_return(true) - it { expect{subject}.to raise_error(ActiveRecord::RecordNotDestroyed) } + destroy_project(project, user) + end + end + + context 'when image repository deletion fails' do + it 'raises an exception' do + expect_any_instance_of(ContainerRepository) + .to receive(:delete_tags!).and_return(false) + + expect{ destroy_project(project, user) } + .to raise_error(ActiveRecord::RecordNotDestroyed) + end + end + end + + context 'when there are tags for legacy root repository' do + before do + stub_container_registry_tags(repository: project.full_path, + tags: ['tag']) + end + + context 'when image repository tags deletion succeeds' do + it 'removes tags' do + expect_any_instance_of(ContainerRepository) + .to receive(:delete_tags!).and_return(true) + + destroy_project(project, user) + end + end + + context 'when image repository tags deletion fails' do + it 'raises an exception' do + expect_any_instance_of(ContainerRepository) + .to receive(:delete_tags!).and_return(false) + + expect { destroy_project(project, user) } + .to raise_error(Projects::DestroyService::DestroyError) + end + end end end - def destroy_project(project, user, params) + def destroy_project(project, user, params = {}) if async Projects::DestroyService.new(project, user, params).async_execute else From f60820ed0345513d43a1fd9a8b93caf4d39756f4 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 13:02:52 +0200 Subject: [PATCH 80/97] Fix wording in registry tags controller notifications --- app/controllers/projects/registry/tags_controller.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/projects/registry/tags_controller.rb b/app/controllers/projects/registry/tags_controller.rb index aab130787e9..138f9d14ee1 100644 --- a/app/controllers/projects/registry/tags_controller.rb +++ b/app/controllers/projects/registry/tags_controller.rb @@ -6,10 +6,10 @@ module Projects def destroy if tag.delete redirect_to project_container_registry_path(@project), - notice: 'Tag removed successfull!' + notice: 'Registry tag has been removed successfully!' else redirect_to project_container_registry_path(@project), - alert: 'Failed to remove repository tag!' + alert: 'Failed to remove registry tag!' end end From 666ba382e900b0972d457af4f8784ef61324be5c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 13:06:11 +0200 Subject: [PATCH 81/97] Simplify registry-related code in project model --- app/models/project.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index fa64ccbf7e4..2bbfba90b6b 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1387,10 +1387,6 @@ class Project < ActiveRecord::Base def has_root_container_repository_tags? return false unless Gitlab.config.registry.enabled - ContainerRegistry::Path.new(self.full_path).tap do |path| - ContainerRepository.build_from_path(path).tap do |repository| - return repository.has_tags? - end - end + ContainerRepository.build_root_repository(self).has_tags? end end From 33329d40df85c81b0a932c53a6152e03fc1f7363 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 13:13:51 +0200 Subject: [PATCH 82/97] Fix project specs related to container registry --- spec/models/project_spec.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 4c13e53d831..da1372fe761 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1157,11 +1157,12 @@ describe Project, models: true do # Project#gitlab_shell returns a new instance of Gitlab::Shell on every # call. This makes testing a bit easier. allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) - allow(project).to receive(:previous_changes).and_return('path' => ['foo']) end it 'renames a repository' do + stub_container_registry_config(enabled: false) + expect(gitlab_shell).to receive(:mv_repository). ordered. with(project.repository_storage_path, "#{project.namespace.full_path}/foo", "#{project.full_path}"). From b03f1699c47ce8a08f67ef458107d22cbafbc0bd Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 13:24:13 +0200 Subject: [PATCH 83/97] Extend registry docs regarding multi-level repositories --- doc/user/project/container_registry.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/user/project/container_registry.md b/doc/user/project/container_registry.md index 6de75e43ed3..34762c1bf46 100644 --- a/doc/user/project/container_registry.md +++ b/doc/user/project/container_registry.md @@ -10,7 +10,7 @@ - Starting from GitLab 8.12, if you have 2FA enabled in your account, you need to pass a personal access token instead of your password in order to login to GitLab's Container Registry. -- Multiple level image names support was added in GitLab 8.15 +- Multiple level image names support was added in GitLab 9.1 With the Docker Container Registry integrated into GitLab, every project can have its own space to store its Docker images. @@ -65,6 +65,16 @@ Your image will be named after the following scheme: /// ``` +GitLab supports up to three levels of image repository names. + +Following image repository names are valid: + +``` +registry.example.com// +registry.example.com///image +registry.example.com///image/type +``` + ## Use images from GitLab Container Registry To download and run a container from images hosted in GitLab Container Registry, From ec28f0b59b54a00ddd4ac33dfa74fa6adfb7f09e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 13:39:42 +0200 Subject: [PATCH 84/97] Add changelog entry for multi-level image repositories --- .../feature-multi-level-container-registry-images.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelogs/unreleased/feature-multi-level-container-registry-images.yml diff --git a/changelogs/unreleased/feature-multi-level-container-registry-images.yml b/changelogs/unreleased/feature-multi-level-container-registry-images.yml new file mode 100644 index 00000000000..6d39a6c17c0 --- /dev/null +++ b/changelogs/unreleased/feature-multi-level-container-registry-images.yml @@ -0,0 +1,4 @@ +--- +title: Add support for multi-level container image repository names +merge_request: 10109 +author: André Guede From 2f939b4a81be2b3f09b33313b0d13db28d0b1a31 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 14:58:51 +0200 Subject: [PATCH 85/97] Fix container registry specs after changing texts --- spec/features/container_registry_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index fa7adbe71ea..b86609e07c5 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -19,7 +19,7 @@ describe "Container Registry" do scenario 'user visits container registry main page' do visit_container_registry - expect(page).to have_content 'No container images' + expect(page).to have_content 'No container image repositories' end end From 22d68f2bc38b0c55a33b41410ed3587b1d7d5b74 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 15:41:38 +0200 Subject: [PATCH 86/97] Fix HAML lint offense in repository image partial --- app/views/projects/registry/repositories/_image.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/projects/registry/repositories/_image.html.haml b/app/views/projects/registry/repositories/_image.html.haml index e0d74789207..b0b09354e02 100644 --- a/app/views/projects/registry/repositories/_image.html.haml +++ b/app/views/projects/registry/repositories/_image.html.haml @@ -14,7 +14,7 @@ method: :delete do = icon('trash cred') - .container-image-tags.js-toggle-content{ class: 'hide' } + .container-image-tags.js-toggle-content.hide - if image.has_tags? .table-holder %table.table.tags From ee6f50027c3b8366d97a0170f99c24dfc23cd101 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 14:13:46 +0200 Subject: [PATCH 87/97] Improve container repository tags controller route --- config/routes/project.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/config/routes/project.rb b/config/routes/project.rb index f85521fe6d3..909a12a0204 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -221,14 +221,13 @@ constraints(ProjectUrlConstrainer.new) do end end - resources :container_registry, - controller: 'registry/repositories', - only: [:index, :destroy], - constraints: { id: Gitlab::Regex.container_registry_reference_regex } + resources :container_registry, only: [:index, :destroy], + controller: 'registry/repositories' namespace :registry do resources :repository, only: [] do - resources :tags, only: [:destroy] + resources :tags, only: [:destroy], + constraints: { id: Gitlab::Regex.container_registry_reference_regex } end end From ed0547b7c069eb1df002f6441cfa313ba6c6381e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 14:18:42 +0200 Subject: [PATCH 88/97] Require container registry entities in controllers --- .../projects/registry/repositories_controller.rb | 2 +- app/controllers/projects/registry/tags_controller.rb | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb index 2e18075d40f..737f8424ebf 100644 --- a/app/controllers/projects/registry/repositories_controller.rb +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -21,7 +21,7 @@ module Projects private def image - @image ||= project.container_repositories.find_by(id: params[:id]) + @image ||= project.container_repositories.find(params[:id]) end ## diff --git a/app/controllers/projects/registry/tags_controller.rb b/app/controllers/projects/registry/tags_controller.rb index 138f9d14ee1..7a9f290e946 100644 --- a/app/controllers/projects/registry/tags_controller.rb +++ b/app/controllers/projects/registry/tags_controller.rb @@ -17,11 +17,13 @@ module Projects def repository @image ||= project.container_repositories - .find_by(id: params[:repository_id]) + .find(params[:repository_id]) end def tag - @tag ||= repository.tag(params[:id]) if params[:id].present? + return render_404 unless params[:id].present? + + @tag ||= repository.tag(params[:id]) end end end From b31c53320ea930f85dabbea674103d2faa59471f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 14:20:33 +0200 Subject: [PATCH 89/97] Revert unneeded change in project destroy service --- app/services/projects/destroy_service.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index a47e74ba9b0..f2d12402239 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -29,6 +29,7 @@ module Projects Project.transaction do project.team.truncate + project.destroy! unless remove_legacy_registry_tags raise_error('Failed to remove some tags in project container registry. Please try again or contact administrator.') @@ -41,8 +42,6 @@ module Projects unless remove_repository(wiki_path) raise_error('Failed to remove wiki repository. Please try again or contact administrator.') end - - project.destroy! end log_info("Project \"#{project.path_with_namespace}\" was removed") From b611da70afeffbffd4a8fc8d5ddf61aa2362617d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 14:32:45 +0200 Subject: [PATCH 90/97] Fix status when removing legacy tags from registry --- app/services/projects/destroy_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index f2d12402239..06d8d143231 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -76,7 +76,7 @@ module Projects return true unless Gitlab.config.registry.enabled ContainerRepository.build_root_repository(project).tap do |repository| - return repository.delete_tags! if repository.has_tags? + return repository.has_tags? ? repository.delete_tags! : true end end From c753975b087cf71d82712624fa5fcd9fa6d79844 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 14:37:50 +0200 Subject: [PATCH 91/97] Add some minor improvements into registry partials --- app/assets/stylesheets/pages/container_registry.scss | 2 +- app/views/projects/registry/repositories/_image.html.haml | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/app/assets/stylesheets/pages/container_registry.scss b/app/assets/stylesheets/pages/container_registry.scss index 92543d7d714..3266714396e 100644 --- a/app/assets/stylesheets/pages/container_registry.scss +++ b/app/assets/stylesheets/pages/container_registry.scss @@ -8,7 +8,7 @@ .container-image-head { padding: 0 16px; - line-height: 4; + line-height: 4em; } .table.tags { diff --git a/app/views/projects/registry/repositories/_image.html.haml b/app/views/projects/registry/repositories/_image.html.haml index b0b09354e02..d183ce34a3a 100644 --- a/app/views/projects/registry/repositories/_image.html.haml +++ b/app/views/projects/registry/repositories/_image.html.haml @@ -1,7 +1,7 @@ .container-image.js-toggle-container .container-image-head = link_to "#", class: "js-toggle-button" do - = icon("chevron-down") + = icon('chevron-down', 'aria-hidden': 'true') = escape_once(image.path) = clipboard_button(clipboard_text: "docker pull #{image.path}") @@ -12,7 +12,7 @@ title: 'Remove repository', data: { confirm: 'Are you sure?' }, method: :delete do - = icon('trash cred') + = icon('trash cred', 'aria-hidden': 'true') .container-image-tags.js-toggle-content.hide - if image.has_tags? @@ -28,6 +28,5 @@ %th = render partial: 'tag', collection: image.tags - else - %li - .nothing-here-block No tags in Container Registry for this container image. + .nothing-here-block No tags in Container Registry for this container image. From 000af87190bee6c99ad4e734818b3ba37c21a292 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 14:44:35 +0200 Subject: [PATCH 92/97] Remove redundant code from container registry classes --- app/controllers/projects/registry/repositories_controller.rb | 2 +- app/models/container_repository.rb | 4 ++-- spec/lib/container_registry/blob_spec.rb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb index 737f8424ebf..17f391ba07f 100644 --- a/app/controllers/projects/registry/repositories_controller.rb +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -34,7 +34,7 @@ module Projects break if path.has_repository? ContainerRepository.build_from_path(path).tap do |repository| - repository.save if repository.has_tags? + repository.save! if repository.has_tags? end end end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 463eb5b7d69..9682df3a586 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -28,7 +28,7 @@ class ContainerRepository < ActiveRecord::Base end def manifest - @manifest ||= client.repository_tags(self.path) + @manifest ||= client.repository_tags(path) end def tags @@ -45,7 +45,7 @@ class ContainerRepository < ActiveRecord::Base end def has_tags? - tags.to_a.any? + tags.any? end def root_repository? diff --git a/spec/lib/container_registry/blob_spec.rb b/spec/lib/container_registry/blob_spec.rb index 76ea29666ea..f06e5fd54a2 100644 --- a/spec/lib/container_registry/blob_spec.rb +++ b/spec/lib/container_registry/blob_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' describe ContainerRegistry::Blob do let(:group) { create(:group, name: 'group') } - let(:project) { create(:project, path: 'test', group: group) } + let(:project) { create(:empty_project, path: 'test', group: group) } let(:repository) do create(:container_repository, name: 'image', From 6b565f534a17ce893ce288e8afd1e3e4d5d3dd3a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 14:47:28 +0200 Subject: [PATCH 93/97] Improve docs for multi-level container registry images --- doc/user/project/container_registry.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/user/project/container_registry.md b/doc/user/project/container_registry.md index 34762c1bf46..6a2ca7fb428 100644 --- a/doc/user/project/container_registry.md +++ b/doc/user/project/container_registry.md @@ -67,12 +67,12 @@ Your image will be named after the following scheme: GitLab supports up to three levels of image repository names. -Following image repository names are valid: +Following examples of image tags are valid: ``` -registry.example.com// -registry.example.com///image -registry.example.com///image/type +registry.example.com/group/project:some-tag +registry.example.com/group/project/image:latest +registry.example.com/group/project/my/image:rc1 ``` ## Use images from GitLab Container Registry From 1e54181523f42c2734056b6ea548ebc2134f57a6 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 15:12:29 +0200 Subject: [PATCH 94/97] Improve migration for container repositories table --- .../20170322013926_create_container_repository.rb | 9 +++++++-- db/schema.rb | 10 ++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/db/migrate/20170322013926_create_container_repository.rb b/db/migrate/20170322013926_create_container_repository.rb index 87a1523724c..32eba3903b1 100644 --- a/db/migrate/20170322013926_create_container_repository.rb +++ b/db/migrate/20170322013926_create_container_repository.rb @@ -5,8 +5,13 @@ class CreateContainerRepository < ActiveRecord::Migration def change create_table :container_repositories do |t| - t.integer :project_id - t.string :name + t.references :project, foreign_key: true, null: false + t.string :name, null: false + + t.timestamps null: false end + + add_index :container_repositories, :project_id + add_index :container_repositories, [:project_id, :name], unique: true end end diff --git a/db/schema.rb b/db/schema.rb index 3c11e68da2c..9ee1d0e6aef 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -324,10 +324,15 @@ ActiveRecord::Schema.define(version: 20170329124448) do add_index "ci_variables", ["project_id"], name: "index_ci_variables_on_project_id", using: :btree create_table "container_repositories", force: :cascade do |t| - t.integer "project_id" - t.string "name" + t.integer "project_id", null: false + t.string "name", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false end + add_index "container_repositories", ["project_id", "name"], name: "index_container_repositories_on_project_id_and_name", unique: true, using: :btree + add_index "container_repositories", ["project_id"], name: "index_container_repositories_on_project_id", using: :btree + create_table "deploy_keys_projects", force: :cascade do |t| t.integer "deploy_key_id", null: false t.integer "project_id", null: false @@ -1304,6 +1309,7 @@ ActiveRecord::Schema.define(version: 20170329124448) do add_foreign_key "boards", "projects" add_foreign_key "chat_teams", "namespaces", on_delete: :cascade add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade + add_foreign_key "container_repositories", "projects" add_foreign_key "issue_metrics", "issues", on_delete: :cascade add_foreign_key "label_priorities", "labels", on_delete: :cascade add_foreign_key "label_priorities", "projects", on_delete: :cascade From 82dea6cffe339456c5ba0fd090464926e970dccf Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 15:19:48 +0200 Subject: [PATCH 95/97] Fix Rubocop offenses related to registry routes --- config/routes/project.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/routes/project.rb b/config/routes/project.rb index 909a12a0204..757e1852da4 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -222,12 +222,12 @@ constraints(ProjectUrlConstrainer.new) do end resources :container_registry, only: [:index, :destroy], - controller: 'registry/repositories' + controller: 'registry/repositories' namespace :registry do resources :repository, only: [] do resources :tags, only: [:destroy], - constraints: { id: Gitlab::Regex.container_registry_reference_regex } + constraints: { id: Gitlab::Regex.container_registry_reference_regex } end end From 714c408f222cc3bfef577b477f7bab0556f50599 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 6 Apr 2017 10:23:51 +0200 Subject: [PATCH 96/97] Add minor improvements to container registry code --- app/controllers/projects/registry/tags_controller.rb | 6 ++---- lib/container_registry/path.rb | 8 ++++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/app/controllers/projects/registry/tags_controller.rb b/app/controllers/projects/registry/tags_controller.rb index 7a9f290e946..d689cade3ab 100644 --- a/app/controllers/projects/registry/tags_controller.rb +++ b/app/controllers/projects/registry/tags_controller.rb @@ -15,15 +15,13 @@ module Projects private - def repository + def image @image ||= project.container_repositories .find(params[:repository_id]) end def tag - return render_404 unless params[:id].present? - - @tag ||= repository.tag(params[:id]) + @tag ||= image.tag(params[:id]) end end end diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 89973b2e7b8..a4b5f2aba6c 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -1,6 +1,6 @@ module ContainerRegistry ## - # Class reponsible for extracting project and repository name from + # Class responsible for extracting project and repository name from # image repository path provided by a containers registry API response. # # Example: @@ -12,6 +12,8 @@ module ContainerRegistry class Path InvalidRegistryPathError = Class.new(StandardError) + LEVELS_SUPPORTED = 3 + def initialize(path) @path = path end @@ -50,7 +52,9 @@ module ContainerRegistry end def repository_project - @project ||= Project.where_full_path_in(nodes.first(3)).first + @project ||= Project + .where_full_path_in(nodes.first(LEVELS_SUPPORTED)) + .first end def repository_name From 163e9f99ab29524bca204fbd0d0aabc1a1813e4a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 6 Apr 2017 12:25:09 +0200 Subject: [PATCH 97/97] Fix indexes in container repositories table --- db/migrate/20170322013926_create_container_repository.rb | 3 +-- spec/models/container_repository_spec.rb | 7 ------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/db/migrate/20170322013926_create_container_repository.rb b/db/migrate/20170322013926_create_container_repository.rb index 32eba3903b1..91540bc88bd 100644 --- a/db/migrate/20170322013926_create_container_repository.rb +++ b/db/migrate/20170322013926_create_container_repository.rb @@ -5,13 +5,12 @@ class CreateContainerRepository < ActiveRecord::Migration def change create_table :container_repositories do |t| - t.references :project, foreign_key: true, null: false + t.references :project, foreign_key: true, index: true, null: false t.string :name, null: false t.timestamps null: false end - add_index :container_repositories, :project_id add_index :container_repositories, [:project_id, :name], unique: true end end diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index 3e6082ec326..f7ee0b57072 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -21,13 +21,6 @@ describe ContainerRepository do headers: { 'Content-Type' => 'application/json' }) end - describe 'validations' do - it 'validates uniqueness of name scoped to project' do - expect(subject).to validate_uniqueness_of(:name) - .scoped_to(:project_id) - end - end - describe 'associations' do it 'belongs to the project' do expect(container_repository).to belong_to(:project)