kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Move storage specific code from Namespace and Project to concerns

Browse source
This commit is contained in:
Gabriel Mazetto 2017-07-05 02:19:02 +02:00
parent faa2a12391
commit 9861531888
4 changed files with 194 additions and 168 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

109
app/models/concerns/storage/legacy_namespace.rb Normal file
View file

@ -0,0 +1,109 @@
module Storage
module LegacyNamespace
extend ActiveSupport::Concern
def move_dir
if any_project_has_container_registry_tags?
raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry')
end
# Move the namespace directory in all storages paths used by member projects
repository_storage_paths.each do |repository_storage_path|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, full_path_was)
unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path)
Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}"
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise Gitlab::UpdatePathError.new('namespace directory cannot be moved')
end
end
Gitlab::UploadsTransfer.new.rename_namespace(full_path_was, full_path)
Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path)
remove_exports!
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
# So we basically we mute exceptions in next actions
begin
send_update_instructions
true
rescue
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
end
# Hooks
# Save the storage paths before the projects are destroyed to use them on after destroy
def prepare_for_destroy
old_repository_storage_paths
end
private
def send_update_instructions
projects.each do |project|
project.send_move_instructions("#{full_path_was}/#{project.path}")
end
end
def old_repository_storage_paths
@old_repository_storage_paths ||= repository_storage_paths
end
def repository_storage_paths
# We need to get the storage paths for all the projects, even the ones that are
# pending delete. Unscoping also get rids of the default order, which causes
# problems with SELECT DISTINCT.
Project.unscoped do
all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
end
end
def rm_dir
# Remove the namespace directory in all storages paths used by member projects
old_repository_storage_paths.each do |repository_storage_path|
# Move namespace directory into trash.
# We will remove it later async
new_path = "#{full_path}+#{id}+deleted"
if gitlab_shell.mv_namespace(repository_storage_path, full_path, new_path)
message = "Namespace directory \"#{full_path}\" moved to \"#{new_path}\""
Gitlab::AppLogger.info message
# Remove namespace directroy async with delay so
# GitLab has time to remove all projects first
run_after_commit do
GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
end
end
end
remove_exports!
end
def remove_exports!
Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete))
end
def export_path
File.join(Gitlab::ImportExport.storage_path, full_path_was)
end
def full_path_was
if parent
parent.full_path + '/' + path_was
else
path_was
end
end
end
end

76
app/models/concerns/storage/legacy_project.rb Normal file
View file

@ -0,0 +1,76 @@
module Storage
module LegacyProject
extend ActiveSupport::Concern
def disk_path
full_path
end
def ensure_dir_exist
gitlab_shell.add_namespace(repository_storage_path, namespace.full_path)
end
def rename_repo
path_was = previous_changes['path'].first
old_path_with_namespace = File.join(namespace.full_path, path_was)
new_path_with_namespace = File.join(namespace.full_path, path)
Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}"
if has_container_registry_tags?
Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!"
# we currently doesn't support renaming repository if it contains images in container registry
raise StandardError.new('Project cannot be renamed, because images are present in its container registry')
end
expire_caches_before_rename(old_path_with_namespace)
if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
send_move_instructions(old_path_with_namespace)
expires_full_path_cache
@old_path_with_namespace = old_path_with_namespace
SystemHooksService.new.execute_hooks_for(self, :rename)
@repository = nil
rescue => e
Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}"
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
else
Rails.logger.error "Repository could not be renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise StandardError.new('repository cannot be renamed')
end
Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path)
Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path)
end
def create_repository(force: false)
# Forked import is handled asynchronously
return if forked? && !force
if gitlab_shell.add_repository(repository_storage_path, path_with_namespace)
repository.after_create
true
else
errors.add(:base, 'Failed to create repository via gitlab-shell')
false
end
end
end
end

104
app/models/namespace.rb
View file

@ -8,6 +8,7 @@ class Namespace < ActiveRecord::Base
include Gitlab::VisibilityLevel
include Routable
include AfterCommitQueue
include Storage::LegacyNamespace
# Prevent users from creating unreasonably deep level of nesting.
# The number 20 was taken based on maximum nesting level of
@ -41,10 +42,11 @@ class Namespace < ActiveRecord::Base
delegate :name, to: :owner, allow_nil: true, prefix: true
after_update :move_dir, if: :path_changed?
after_commit :refresh_access_of_projects_invited_groups, on: :update, if: -> { previous_changes.key?('share_with_group_lock') }
# Save the storage paths before the projects are destroyed to use them on after destroy
# Legacy Storage specific hooks
after_update :move_dir, if: :path_changed?
before_destroy(prepend: true) { prepare_for_destroy }
after_destroy :rm_dir
@ -118,53 +120,10 @@ class Namespace < ActiveRecord::Base
owner_name
end
def move_dir
if any_project_has_container_registry_tags?
raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry')
end
# Move the namespace directory in all storages paths used by member projects
repository_storage_paths.each do |repository_storage_path|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, full_path_was)
unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path)
Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}"
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise Gitlab::UpdatePathError.new('namespace directory cannot be moved')
end
end
Gitlab::UploadsTransfer.new.rename_namespace(full_path_was, full_path)
Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path)
remove_exports!
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
# So we basically we mute exceptions in next actions
begin
send_update_instructions
rescue
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
end
def any_project_has_container_registry_tags?
all_projects.any?(&:has_container_registry_tags?)
end
def send_update_instructions
projects.each do |project|
project.send_move_instructions("#{full_path_was}/#{project.path}")
end
end
def kind
type == 'Group' ? 'group' : 'user'
end
@ -206,14 +165,6 @@ class Namespace < ActiveRecord::Base
parent_id_changed?
end
def prepare_for_destroy
old_repository_storage_paths
end
def old_repository_storage_paths
@old_repository_storage_paths ||= repository_storage_paths
end
# Includes projects from this namespace and projects from all subgroups
# that belongs to this namespace
def all_projects
@ -232,37 +183,6 @@ class Namespace < ActiveRecord::Base
private
def repository_storage_paths
# We need to get the storage paths for all the projects, even the ones that are
# pending delete. Unscoping also get rids of the default order, which causes
# problems with SELECT DISTINCT.
Project.unscoped do
all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
end
end
def rm_dir
# Remove the namespace directory in all storages paths used by member projects
old_repository_storage_paths.each do |repository_storage_path|
# Move namespace directory into trash.
# We will remove it later async
new_path = "#{full_path}+#{id}+deleted"
if gitlab_shell.mv_namespace(repository_storage_path, full_path, new_path)
message = "Namespace directory \"#{full_path}\" moved to \"#{new_path}\""
Gitlab::AppLogger.info message
# Remove namespace directroy async with delay so
# GitLab has time to remove all projects first
run_after_commit do
GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
end
end
end
remove_exports!
end
def refresh_access_of_projects_invited_groups
Group
.joins(project_group_links: :project)
@ -270,22 +190,6 @@ class Namespace < ActiveRecord::Base
.find_each(&:refresh_members_authorized_projects)
end
def remove_exports!
Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete))
end
def export_path
File.join(Gitlab::ImportExport.storage_path, full_path_was)
end
def full_path_was
if parent
parent.full_path + '/' + path_was
else
path_was
end
end
def nesting_level_allowed
if ancestors.count > Group::NUMBER_OF_ANCESTORS_ALLOWED
errors.add(:parent_id, "has too deep level of nesting")

73
app/models/project.rb
View file

@ -17,6 +17,7 @@ class Project < ActiveRecord::Base
include ProjectFeaturesCompatibility
include SelectForProjectAuthorization
include Routable
include Storage::LegacyProject
extend Gitlab::ConfigHelper
@ -45,7 +46,6 @@ class Project < ActiveRecord::Base
after_create :ensure_dir_exist
after_create :create_project_feature, unless: :project_feature
after_save :ensure_dir_exist, if: :namespace_id_changed?
after_save :update_project_statistics, if: :namespace_id_changed?
# set last_activity_at to the same as created_at
@ -67,6 +67,10 @@ class Project < ActiveRecord::Base
after_validation :check_pending_delete
# Legacy Storage specific hooks
after_save :ensure_dir_exist, if: :namespace_id_changed?
acts_as_taggable
attr_accessor :new_default_branch
@ -974,56 +978,6 @@ class Project < ActiveRecord::Base
!group
end
def rename_repo
path_was = previous_changes['path'].first
old_path_with_namespace = File.join(namespace.full_path, path_was)
new_path_with_namespace = File.join(namespace.full_path, path)
Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}"
if has_container_registry_tags?
Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!"
# we currently doesn't support renaming repository if it contains images in container registry
raise StandardError.new('Project cannot be renamed, because images are present in its container registry')
end
expire_caches_before_rename(old_path_with_namespace)
if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
send_move_instructions(old_path_with_namespace)
expires_full_path_cache
@old_path_with_namespace = old_path_with_namespace
SystemHooksService.new.execute_hooks_for(self, :rename)
@repository = nil
rescue => e
Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}"
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
else
Rails.logger.error "Repository could not be renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise StandardError.new('repository cannot be renamed')
end
Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path)
Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path)
end
# Expires various caches before a project is renamed.
def expire_caches_before_rename(old_path)
repo = Repository.new(old_path, self)
@ -1109,19 +1063,6 @@ class Project < ActiveRecord::Base
merge_requests.where(source_project_id: self.id)
end
def create_repository(force: false)
# Forked import is handled asynchronously
return if forked? && !force
if gitlab_shell.add_repository(repository_storage_path, path_with_namespace)
repository.after_create
true
else
errors.add(:base, 'Failed to create repository via gitlab-shell')
false
end
end
def ensure_repository
create_repository(force: true) unless repository_exists?
end
@ -1327,10 +1268,6 @@ class Project < ActiveRecord::Base
status.zero?
end
def ensure_dir_exist
gitlab_shell.add_namespace(repository_storage_path, namespace.full_path)
end
def predefined_variables
[
{ key: 'CI_PROJECT_ID', value: id.to_s, public: true },