kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Move ReservedPathsMigration into V1 namespace

Browse source
This commit is contained in:
Bob Van Landuyt 2017-04-25 14:10:04 +02:00
parent 12735eefcd
commit 99a03fd6e9
16 changed files with 392 additions and 379 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
db/post_migrate/20170412174900_rename_reserved_dynamic_paths.rb
View file

@ -2,33 +2,36 @@
# for more information on how to write migrations for GitLab.
class RenameReservedDynamicPaths < ActiveRecord::Migration
include Gitlab::Database::RenameReservedPathsMigration
include Gitlab::Database::RenameReservedPathsMigration::V1
DOWNTIME = false
disable_ddl_transaction!
DISALLOWED_ROOT_PATHS = %w[
-
abuse_reports
api
autocomplete
member
explore
uploads
import
notification_settings
abuse_reports
invites
koding
health_check
import
invites
jwt
koding
member
notification_settings
oauth
sent_notifications
-
uploads
users
]
DISALLOWED_WILDCARD_PATHS = %w[info/lfs/objects gitlab-lfs/objects
environments/folders]
DISALLOWED_WILDCARD_PATHS = %w[
environments/folders
gitlab-lfs/objects
info/lfs/objects
]
def up
rename_root_paths(DISALLOWED_ROOT_PATHS)

20
lib/gitlab/database/rename_reserved_paths_migration.rb
View file

@ -1,20 +0,0 @@
module Gitlab
module Database
module RenameReservedPathsMigration
def self.included(kls)
kls.include(MigrationHelpers)
end
def rename_wildcard_paths(one_or_more_paths)
paths = Array(one_or_more_paths)
RenameNamespaces.new(paths, self).rename_namespaces(type: :wildcard)
RenameProjects.new(paths, self).rename_projects
end
def rename_root_paths(paths)
paths = Array(paths)
RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level)
end
end
end
end

94
lib/gitlab/database/rename_reserved_paths_migration/migration_classes.rb
View file

@ -1,94 +0,0 @@
module Gitlab
module Database
module RenameReservedPathsMigration
module MigrationClasses
module Routable
def full_path
if route && route.path.present?
@full_path ||= route.path
else
update_route if persisted?
build_full_path
end
end
def build_full_path
if parent && path
parent.full_path + '/' + path
else
path
end
end
def update_route
prepare_route
route.save
end
def prepare_route
route || build_route(source: self)
route.path = build_full_path
route.name = build_full_name
@full_path = nil
@full_name = nil
end
def build_full_name
if parent && name
parent.human_name + ' / ' + name
else
name
end
end
def human_name
owner&.name
end
end
class User < ActiveRecord::Base
self.table_name = 'users'
end
class Namespace < ActiveRecord::Base
include MigrationClasses::Routable
self.table_name = 'namespaces'
belongs_to :parent,
class_name: "#{MigrationClasses.name}::Namespace"
has_one :route, as: :source
has_many :children,
class_name: "#{MigrationClasses.name}::Namespace",
foreign_key: :parent_id
belongs_to :owner,
class_name: "#{MigrationClasses.name}::User"
# Overridden to have the correct `source_type` for the `route` relation
def self.name
'Namespace'
end
end
class Route < ActiveRecord::Base
self.table_name = 'routes'
belongs_to :source, polymorphic: true
end
class Project < ActiveRecord::Base
include MigrationClasses::Routable
has_one :route, as: :source
self.table_name = 'projects'
def repository_storage_path
Gitlab.config.repositories.storages[repository_storage]['path']
end
# Overridden to have the correct `source_type` for the `route` relation
def self.name
'Project'
end
end
end
end
end
end

125
lib/gitlab/database/rename_reserved_paths_migration/rename_base.rb
View file

@ -1,125 +0,0 @@
module Gitlab
module Database
module RenameReservedPathsMigration
class RenameBase
attr_reader :paths, :migration
delegate :update_column_in_batches,
:replace_sql,
to: :migration
def initialize(paths, migration)
@paths = paths
@migration = migration
end
def path_patterns
@path_patterns ||= paths.map { |path| "%#{path}" }
end
def rename_path_for_routable(routable)
old_path = routable.path
old_full_path = routable.full_path
# Only remove the last occurrence of the path name to get the parent namespace path
namespace_path = remove_last_occurrence(old_full_path, old_path)
new_path = rename_path(namespace_path, old_path)
new_full_path = join_routable_path(namespace_path, new_path)
# skips callbacks & validations
routable.class.where(id: routable).
update_all(path: new_path)
rename_routes(old_full_path, new_full_path)
[old_full_path, new_full_path]
end
def rename_routes(old_full_path, new_full_path)
replace_statement = replace_sql(Route.arel_table[:path],
old_full_path,
new_full_path)
update_column_in_batches(:routes, :path, replace_statement) do |table, query|
query.where(MigrationClasses::Route.arel_table[:path].matches("#{old_full_path}%"))
end
end
def rename_path(namespace_path, path_was)
counter = 0
path = "#{path_was}#{counter}"
while route_exists?(join_routable_path(namespace_path, path))
counter += 1
path = "#{path_was}#{counter}"
end
path
end
def remove_last_occurrence(string, pattern)
string.reverse.sub(pattern.reverse, "").reverse
end
def join_routable_path(namespace_path, top_level)
if namespace_path.present?
File.join(namespace_path, top_level)
else
top_level
end
end
def route_exists?(full_path)
MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any?
end
def move_pages(old_path, new_path)
move_folders(pages_dir, old_path, new_path)
end
def move_uploads(old_path, new_path)
return unless file_storage?
move_folders(uploads_dir, old_path, new_path)
end
def move_folders(directory, old_relative_path, new_relative_path)
old_path = File.join(directory, old_relative_path)
return unless File.directory?(old_path)
new_path = File.join(directory, new_relative_path)
FileUtils.mv(old_path, new_path)
end
def remove_cached_html_for_projects(project_ids)
update_column_in_batches(:projects, :description_html, nil) do |table, query|
query.where(table[:id].in(project_ids))
end
update_column_in_batches(:issues, :description_html, nil) do |table, query|
query.where(table[:project_id].in(project_ids))
end
update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
query.where(table[:target_project_id].in(project_ids))
end
update_column_in_batches(:notes, :note_html, nil) do |table, query|
query.where(table[:project_id].in(project_ids))
end
end
def file_storage?
CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
end
def uploads_dir
File.join(CarrierWave.root, "uploads")
end
def pages_dir
Settings.pages.path
end
end
end
end
end

70
lib/gitlab/database/rename_reserved_paths_migration/rename_namespaces.rb
View file

@ -1,70 +0,0 @@
module Gitlab
module Database
module RenameReservedPathsMigration
class RenameNamespaces < RenameBase
include Gitlab::ShellAdapter
def rename_namespaces(type:)
namespaces_for_paths(type: type).each do |namespace|
rename_namespace(namespace)
end
end
def namespaces_for_paths(type:)
namespaces = case type
when :wildcard
MigrationClasses::Namespace.where.not(parent_id: nil)
when :top_level
MigrationClasses::Namespace.where(parent_id: nil)
end
with_paths = MigrationClasses::Route.arel_table[:path].
matches_any(path_patterns)
namespaces.joins(:route).where(with_paths)
end
def rename_namespace(namespace)
old_full_path, new_full_path = rename_path_for_routable(namespace)
move_repositories(namespace, old_full_path, new_full_path)
move_uploads(old_full_path, new_full_path)
move_pages(old_full_path, new_full_path)
remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id))
end
def move_repositories(namespace, old_full_path, new_full_path)
repo_paths_for_namespace(namespace).each do |repository_storage_path|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, old_full_path)
unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path)
message = "Exception moving path #{repository_storage_path} \
from #{old_full_path} to #{new_full_path}"
Rails.logger.error message
end
end
end
def repo_paths_for_namespace(namespace)
projects_for_namespace(namespace).distinct.select(:repository_storage).
map(&:repository_storage_path)
end
def projects_for_namespace(namespace)
namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id])
namespace_or_children = MigrationClasses::Project.
arel_table[:namespace_id].
in(namespace_ids)
MigrationClasses::Project.where(namespace_or_children)
end
def child_ids_for_parent(namespace, ids: [])
namespace.children.each do |child|
ids << child.id
child_ids_for_parent(child, ids: ids) if child.children.any?
end
ids
end
end
end
end
end

43
lib/gitlab/database/rename_reserved_paths_migration/rename_projects.rb
View file

@ -1,43 +0,0 @@
module Gitlab
module Database
module RenameReservedPathsMigration
class RenameProjects < RenameBase
include Gitlab::ShellAdapter
def rename_projects
projects_for_paths.each do |project|
rename_project(project)
end
remove_cached_html_for_projects(projects_for_paths.map(&:id))
end
def rename_project(project)
old_full_path, new_full_path = rename_path_for_routable(project)
move_repository(project, old_full_path, new_full_path)
move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki")
move_uploads(old_full_path, new_full_path)
move_pages(old_full_path, new_full_path)
end
def move_repository(project, old_path, new_path)
unless gitlab_shell.mv_repository(project.repository_storage_path,
old_path,
new_path)
Rails.logger.error "Error moving #{old_path} to #{new_path}"
end
end
def projects_for_paths
return @projects_for_paths if @projects_for_paths
with_paths = MigrationClasses::Route.arel_table[:path]
.matches_any(path_patterns)
@projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths)
end
end
end
end
end

22
lib/gitlab/database/rename_reserved_paths_migration/v1.rb Normal file
View file

@ -0,0 +1,22 @@
module Gitlab
module Database
module RenameReservedPathsMigration
module V1
def self.included(kls)
kls.include(MigrationHelpers)
end
def rename_wildcard_paths(one_or_more_paths)
paths = Array(one_or_more_paths)
RenameNamespaces.new(paths, self).rename_namespaces(type: :wildcard)
RenameProjects.new(paths, self).rename_projects
end
def rename_root_paths(paths)
paths = Array(paths)
RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level)
end
end
end
end
end

96
lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb Normal file
View file

@ -0,0 +1,96 @@
module Gitlab
module Database
module RenameReservedPathsMigration
module V1
module MigrationClasses
module Routable
def full_path
if route && route.path.present?
@full_path ||= route.path
else
update_route if persisted?
build_full_path
end
end
def build_full_path
if parent && path
parent.full_path + '/' + path
else
path
end
end
def update_route
prepare_route
route.save
end
def prepare_route
route || build_route(source: self)
route.path = build_full_path
route.name = build_full_name
@full_path = nil
@full_name = nil
end
def build_full_name
if parent && name
parent.human_name + ' / ' + name
else
name
end
end
def human_name
owner&.name
end
end
class User < ActiveRecord::Base
self.table_name = 'users'
end
class Namespace < ActiveRecord::Base
include MigrationClasses::Routable
self.table_name = 'namespaces'
belongs_to :parent,
class_name: "#{MigrationClasses.name}::Namespace"
has_one :route, as: :source
has_many :children,
class_name: "#{MigrationClasses.name}::Namespace",
foreign_key: :parent_id
belongs_to :owner,
class_name: "#{MigrationClasses.name}::User"
# Overridden to have the correct `source_type` for the `route` relation
def self.name
'Namespace'
end
end
class Route < ActiveRecord::Base
self.table_name = 'routes'
belongs_to :source, polymorphic: true
end
class Project < ActiveRecord::Base
include MigrationClasses::Routable
has_one :route, as: :source
self.table_name = 'projects'
def repository_storage_path
Gitlab.config.repositories.storages[repository_storage]['path']
end
# Overridden to have the correct `source_type` for the `route` relation
def self.name
'Project'
end
end
end
end
end
end
end

127
lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb Normal file
View file

@ -0,0 +1,127 @@
module Gitlab
module Database
module RenameReservedPathsMigration
module V1
class RenameBase
attr_reader :paths, :migration
delegate :update_column_in_batches,
:replace_sql,
to: :migration
def initialize(paths, migration)
@paths = paths
@migration = migration
end
def path_patterns
@path_patterns ||= paths.map { |path| "%#{path}" }
end
def rename_path_for_routable(routable)
old_path = routable.path
old_full_path = routable.full_path
# Only remove the last occurrence of the path name to get the parent namespace path
namespace_path = remove_last_occurrence(old_full_path, old_path)
new_path = rename_path(namespace_path, old_path)
new_full_path = join_routable_path(namespace_path, new_path)
# skips callbacks & validations
routable.class.where(id: routable).
update_all(path: new_path)
rename_routes(old_full_path, new_full_path)
[old_full_path, new_full_path]
end
def rename_routes(old_full_path, new_full_path)
replace_statement = replace_sql(Route.arel_table[:path],
old_full_path,
new_full_path)
update_column_in_batches(:routes, :path, replace_statement) do |table, query|
query.where(MigrationClasses::Route.arel_table[:path].matches("#{old_full_path}%"))
end
end
def rename_path(namespace_path, path_was)
counter = 0
path = "#{path_was}#{counter}"
while route_exists?(join_routable_path(namespace_path, path))
counter += 1
path = "#{path_was}#{counter}"
end
path
end
def remove_last_occurrence(string, pattern)
string.reverse.sub(pattern.reverse, "").reverse
end
def join_routable_path(namespace_path, top_level)
if namespace_path.present?
File.join(namespace_path, top_level)
else
top_level
end
end
def route_exists?(full_path)
MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any?
end
def move_pages(old_path, new_path)
move_folders(pages_dir, old_path, new_path)
end
def move_uploads(old_path, new_path)
return unless file_storage?
move_folders(uploads_dir, old_path, new_path)
end
def move_folders(directory, old_relative_path, new_relative_path)
old_path = File.join(directory, old_relative_path)
return unless File.directory?(old_path)
new_path = File.join(directory, new_relative_path)
FileUtils.mv(old_path, new_path)
end
def remove_cached_html_for_projects(project_ids)
update_column_in_batches(:projects, :description_html, nil) do |table, query|
query.where(table[:id].in(project_ids))
end
update_column_in_batches(:issues, :description_html, nil) do |table, query|
query.where(table[:project_id].in(project_ids))
end
update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
query.where(table[:target_project_id].in(project_ids))
end
update_column_in_batches(:notes, :note_html, nil) do |table, query|
query.where(table[:project_id].in(project_ids))
end
end
def file_storage?
CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
end
def uploads_dir
File.join(CarrierWave.root, "uploads")
end
def pages_dir
Settings.pages.path
end
end
end
end
end
end

72
lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb Normal file
View file

@ -0,0 +1,72 @@
module Gitlab
module Database
module RenameReservedPathsMigration
module V1
class RenameNamespaces < RenameBase
include Gitlab::ShellAdapter
def rename_namespaces(type:)
namespaces_for_paths(type: type).each do |namespace|
rename_namespace(namespace)
end
end
def namespaces_for_paths(type:)
namespaces = case type
when :wildcard
MigrationClasses::Namespace.where.not(parent_id: nil)
when :top_level
MigrationClasses::Namespace.where(parent_id: nil)
end
with_paths = MigrationClasses::Route.arel_table[:path].
matches_any(path_patterns)
namespaces.joins(:route).where(with_paths)
end
def rename_namespace(namespace)
old_full_path, new_full_path = rename_path_for_routable(namespace)
move_repositories(namespace, old_full_path, new_full_path)
move_uploads(old_full_path, new_full_path)
move_pages(old_full_path, new_full_path)
remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id))
end
def move_repositories(namespace, old_full_path, new_full_path)
repo_paths_for_namespace(namespace).each do |repository_storage_path|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, old_full_path)
unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path)
message = "Exception moving path #{repository_storage_path} \
from #{old_full_path} to #{new_full_path}"
Rails.logger.error message
end
end
end
def repo_paths_for_namespace(namespace)
projects_for_namespace(namespace).distinct.select(:repository_storage).
map(&:repository_storage_path)
end
def projects_for_namespace(namespace)
namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id])
namespace_or_children = MigrationClasses::Project.
arel_table[:namespace_id].
in(namespace_ids)
MigrationClasses::Project.where(namespace_or_children)
end
def child_ids_for_parent(namespace, ids: [])
namespace.children.each do |child|
ids << child.id
child_ids_for_parent(child, ids: ids) if child.children.any?
end
ids
end
end
end
end
end
end

45
lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb Normal file
View file

@ -0,0 +1,45 @@
module Gitlab
module Database
module RenameReservedPathsMigration
module V1
class RenameProjects < RenameBase
include Gitlab::ShellAdapter
def rename_projects
projects_for_paths.each do |project|
rename_project(project)
end
remove_cached_html_for_projects(projects_for_paths.map(&:id))
end
def rename_project(project)
old_full_path, new_full_path = rename_path_for_routable(project)
move_repository(project, old_full_path, new_full_path)
move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki")
move_uploads(old_full_path, new_full_path)
move_pages(old_full_path, new_full_path)
end
def move_repository(project, old_path, new_path)
unless gitlab_shell.mv_repository(project.repository_storage_path,
old_path,
new_path)
Rails.logger.error "Error moving #{old_path} to #{new_path}"
end
end
def projects_for_paths
return @projects_for_paths if @projects_for_paths
with_paths = MigrationClasses::Route.arel_table[:path]
.matches_any(path_patterns)
@projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths)
end
end
end
end
end
end

8
spec/lib/gitlab/database/rename_reserved_paths_migration/rename_base_spec.rb → spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb
View file

@ -1,7 +1,7 @@
require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration::RenameBase do
let(:migration) { FakeRenameReservedPathMigration.new }
describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase do
let(:migration) { FakeRenameReservedPathMigrationV1.new }
let(:subject) { described_class.new(['the-path'], migration) }
before do
@ -9,12 +9,12 @@ describe Gitlab::Database::RenameReservedPathsMigration::RenameBase do
end
def migration_namespace(namespace)
Gitlab::Database::RenameReservedPathsMigration::MigrationClasses::
Gitlab::Database::RenameReservedPathsMigration::V1::MigrationClasses::
Namespace.find(namespace.id)
end
def migration_project(project)
Gitlab::Database::RenameReservedPathsMigration::MigrationClasses::
Gitlab::Database::RenameReservedPathsMigration::V1::MigrationClasses::
Project.find(project.id)
end

6
spec/lib/gitlab/database/rename_reserved_paths_migration/rename_namespaces_spec.rb → spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces_spec.rb
View file

@ -1,7 +1,7 @@
require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration::RenameNamespaces do
let(:migration) { FakeRenameReservedPathMigration.new }
describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces do
let(:migration) { FakeRenameReservedPathMigrationV1.new }
let(:subject) { described_class.new(['the-path'], migration) }
before do
@ -9,7 +9,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::RenameNamespaces do
end
def migration_namespace(namespace)
Gitlab::Database::RenameReservedPathsMigration::MigrationClasses::
Gitlab::Database::RenameReservedPathsMigration::V1::MigrationClasses::
Namespace.find(namespace.id)
end

4
spec/lib/gitlab/database/rename_reserved_paths_migration/rename_projects_spec.rb → spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb
View file

@ -1,7 +1,7 @@
require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration::RenameProjects do
let(:migration) { FakeRenameReservedPathMigration.new }
describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects do
let(:migration) { FakeRenameReservedPathMigrationV1.new }
let(:subject) { described_class.new(['the-path'], migration) }
before do

10
spec/lib/gitlab/database/rename_reserved_paths_migration_spec.rb
View file

@ -1,7 +1,7 @@
require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration do
let(:subject) { FakeRenameReservedPathMigration.new }
describe Gitlab::Database::RenameReservedPathsMigration::V1 do
let(:subject) { FakeRenameReservedPathMigrationV1.new }
before do
allow(subject).to receive(:say)
@ -10,7 +10,7 @@ describe Gitlab::Database::RenameReservedPathsMigration do
describe '#rename_wildcard_paths' do
it 'should rename namespaces' do
rename_namespaces = double
expect(Gitlab::Database::RenameReservedPathsMigration::RenameNamespaces).
expect(described_class::RenameNamespaces).
to receive(:new).with(['first-path', 'second-path'], subject).
and_return(rename_namespaces)
expect(rename_namespaces).to receive(:rename_namespaces).
@ -21,7 +21,7 @@ describe Gitlab::Database::RenameReservedPathsMigration do
it 'should rename projects' do
rename_projects = double
expect(Gitlab::Database::RenameReservedPathsMigration::RenameProjects).
expect(described_class::RenameProjects).
to receive(:new).with(['the-path'], subject).
and_return(rename_projects)
@ -34,7 +34,7 @@ describe Gitlab::Database::RenameReservedPathsMigration do
describe '#rename_root_paths' do
it 'should rename namespaces' do
rename_namespaces = double
expect(Gitlab::Database::RenameReservedPathsMigration::RenameNamespaces).
expect(described_class::RenameNamespaces).
to receive(:new).with(['the-path'], subject).
and_return(rename_namespaces)
expect(rename_namespaces).to receive(:rename_namespaces).

4
spec/support/fake_migration_classes.rb
View file

@ -1,3 +1,3 @@
class FakeRenameReservedPathMigration < ActiveRecord::Migration
include Gitlab::Database::RenameReservedPathsMigration
class FakeRenameReservedPathMigrationV1 < ActiveRecord::Migration
include Gitlab::Database::RenameReservedPathsMigration::V1
end