kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Use objects for renaming namespaces and projects

Browse source
This commit is contained in:
Bob Van Landuyt 2017-04-14 19:29:22 +02:00
parent 0369ef1452
commit 27f54bebb2
10 changed files with 306 additions and 248 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

97
lib/gitlab/database/rename_reserved_paths_migration.rb
View file

@ -1,104 +1,19 @@
module Gitlab
module Database
module RenameReservedPathsMigration
include MigrationHelpers
include Namespaces
include Projects
def self.included(kls)
kls.include(MigrationHelpers)
end
def rename_wildcard_paths(one_or_more_paths)
paths = Array(one_or_more_paths)
rename_namespaces(paths, type: :wildcard)
rename_projects(paths)
RenameNamespaces.new(paths, self).rename_namespaces(type: :wildcard)
RenameProjects.new(paths, self).rename_projects
end
def rename_root_paths(paths)
paths = Array(paths)
rename_namespaces(paths, type: :top_level)
end
def rename_path_for_routable(routable)
old_path = routable.path
old_full_path = routable.full_path
# Only remove the last occurrence of the path name to get the parent namespace path
namespace_path = remove_last_occurrence(old_full_path, old_path)
new_path = rename_path(namespace_path, old_path)
new_full_path = join_routable_path(namespace_path, new_path)
# skips callbacks & validations
routable.class.where(id: routable).
update_all(path: new_path)
rename_routes(old_full_path, new_full_path)
[old_full_path, new_full_path]
end
def rename_routes(old_full_path, new_full_path)
replace_statement = replace_sql(Route.arel_table[:path],
old_full_path,
new_full_path)
update_column_in_batches(:routes, :path, replace_statement) do |table, query|
query.where(MigrationClasses::Route.arel_table[:path].matches("#{old_full_path}%"))
end
end
def rename_path(namespace_path, path_was)
counter = 0
path = "#{path_was}#{counter}"
while route_exists?(join_routable_path(namespace_path, path))
counter += 1
path = "#{path_was}#{counter}"
end
path
end
def remove_last_occurrence(string, pattern)
string.reverse.sub(pattern.reverse, "").reverse
end
def join_routable_path(namespace_path, top_level)
if namespace_path.present?
File.join(namespace_path, top_level)
else
top_level
end
end
def route_exists?(full_path)
MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any?
end
def move_pages(old_path, new_path)
move_folders(pages_dir, old_path, new_path)
end
def move_uploads(old_path, new_path)
return unless file_storage?
move_folders(uploads_dir, old_path, new_path)
end
def move_folders(directory, old_relative_path, new_relative_path)
old_path = File.join(directory, old_relative_path)
return unless File.directory?(old_path)
new_path = File.join(directory, new_relative_path)
FileUtils.mv(old_path, new_path)
end
def file_storage?
CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
end
def uploads_dir
File.join(CarrierWave.root, "uploads")
end
def pages_dir
Settings.pages.path
RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level)
end
end
end

1
lib/gitlab/database/rename_reserved_paths_migration/migration_classes.rb
View file

@ -76,6 +76,7 @@ module Gitlab
class Project < ActiveRecord::Base
include MigrationClasses::Routable
has_one :route, as: :source
self.table_name = 'projects'
def repository_storage_path

103
lib/gitlab/database/rename_reserved_paths_migration/rename_base.rb Normal file
View file

@ -0,0 +1,103 @@
module Gitlab
module Database
module RenameReservedPathsMigration
class RenameBase
attr_reader :paths, :migration
delegate :update_column_in_batches,
:replace_sql,
to: :migration
def initialize(paths, migration)
@paths = paths
@migration = migration
end
def rename_path_for_routable(routable)
old_path = routable.path
old_full_path = routable.full_path
# Only remove the last occurrence of the path name to get the parent namespace path
namespace_path = remove_last_occurrence(old_full_path, old_path)
new_path = rename_path(namespace_path, old_path)
new_full_path = join_routable_path(namespace_path, new_path)
# skips callbacks & validations
routable.class.where(id: routable).
update_all(path: new_path)
rename_routes(old_full_path, new_full_path)
[old_full_path, new_full_path]
end
def rename_routes(old_full_path, new_full_path)
replace_statement = replace_sql(Route.arel_table[:path],
old_full_path,
new_full_path)
update_column_in_batches(:routes, :path, replace_statement) do |table, query|
query.where(MigrationClasses::Route.arel_table[:path].matches("#{old_full_path}%"))
end
end
def rename_path(namespace_path, path_was)
counter = 0
path = "#{path_was}#{counter}"
while route_exists?(join_routable_path(namespace_path, path))
counter += 1
path = "#{path_was}#{counter}"
end
path
end
def remove_last_occurrence(string, pattern)
string.reverse.sub(pattern.reverse, "").reverse
end
def join_routable_path(namespace_path, top_level)
if namespace_path.present?
File.join(namespace_path, top_level)
else
top_level
end
end
def route_exists?(full_path)
MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any?
end
def move_pages(old_path, new_path)
move_folders(pages_dir, old_path, new_path)
end
def move_uploads(old_path, new_path)
return unless file_storage?
move_folders(uploads_dir, old_path, new_path)
end
def move_folders(directory, old_relative_path, new_relative_path)
old_path = File.join(directory, old_relative_path)
return unless File.directory?(old_path)
new_path = File.join(directory, new_relative_path)
FileUtils.mv(old_path, new_path)
end
def file_storage?
CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
end
def uploads_dir
File.join(CarrierWave.root, "uploads")
end
def pages_dir
Settings.pages.path
end
end
end
end
end

10
lib/gitlab/database/rename_reserved_paths_migration/namespaces.rb → lib/gitlab/database/rename_reserved_paths_migration/rename_namespaces.rb
View file

@ -1,16 +1,16 @@
module Gitlab
module Database
module RenameReservedPathsMigration
module Namespaces
class RenameNamespaces < RenameBase
include Gitlab::ShellAdapter
def rename_namespaces(paths, type:)
namespaces_for_paths(paths, type: type).each do |namespace|
def rename_namespaces(type:)
namespaces_for_paths(type: type).each do |namespace|
rename_namespace(namespace)
end
end
def namespaces_for_paths(paths, type:)
def namespaces_for_paths(type:)
namespaces = if type == :wildcard
MigrationClasses::Namespace.where.not(parent_id: nil)
elsif type == :top_level
@ -52,7 +52,7 @@ module Gitlab
namespace_or_children = MigrationClasses::Project.
arel_table[:namespace_id].
in(namespace_ids)
MigrationClasses::Project.unscoped.where(namespace_or_children)
MigrationClasses::Project.where(namespace_or_children)
end
# This won't scale to huge trees, but it should do for a handful of

8
lib/gitlab/database/rename_reserved_paths_migration/projects.rb → lib/gitlab/database/rename_reserved_paths_migration/rename_projects.rb
View file

@ -1,11 +1,11 @@
module Gitlab
module Database
module RenameReservedPathsMigration
module Projects
class RenameProjects < RenameBase
include Gitlab::ShellAdapter
def rename_projects(paths)
projects_for_paths(paths).each do |project|
def rename_projects
projects_for_paths.each do |project|
rename_project(project)
end
end
@ -27,7 +27,7 @@ module Gitlab
end
end
def projects_for_paths(paths)
def projects_for_paths
with_paths = MigrationClasses::Project.arel_table[:path]
.matches_any(paths)
MigrationClasses::Project.where(with_paths)

147
spec/lib/gitlab/database/rename_reserved_paths_migration/rename_base_spec.rb Normal file
View file

@ -0,0 +1,147 @@
require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration::RenameBase do
let(:migration) { FakeRenameReservedPathMigration.new }
let(:subject) { described_class.new(['the-path'], migration) }
before do
allow(migration).to receive(:say)
end
def migration_namespace(namespace)
Gitlab::Database::RenameReservedPathsMigration::MigrationClasses::
Namespace.find(namespace.id)
end
def migration_project(project)
Gitlab::Database::RenameReservedPathsMigration::MigrationClasses::
Project.find(project.id)
end
describe "#remove_last_ocurrence" do
it "removes only the last occurance of a string" do
input = "this/is/a-word-to-replace/namespace/with/a-word-to-replace"
expect(subject.remove_last_occurrence(input, "a-word-to-replace"))
.to eq("this/is/a-word-to-replace/namespace/with/")
end
end
describe '#rename_path_for_routable' do
context 'for namespaces' do
let(:namespace) { create(:namespace, path: 'the-path') }
it "renames namespaces called the-path" do
subject.rename_path_for_routable(migration_namespace(namespace))
expect(namespace.reload.path).to eq("the-path0")
end
it "renames the route to the namespace" do
subject.rename_path_for_routable(migration_namespace(namespace))
expect(Namespace.find(namespace.id).full_path).to eq("the-path0")
end
it "renames the route for projects of the namespace" do
project = create(:project, path: "project-path", namespace: namespace)
subject.rename_path_for_routable(migration_namespace(namespace))
expect(project.route.reload.path).to eq("the-path0/project-path")
end
it 'returns the old & the new path' do
old_path, new_path = subject.rename_path_for_routable(migration_namespace(namespace))
expect(old_path).to eq('the-path')
expect(new_path).to eq('the-path0')
end
context "the-path namespace -> subgroup -> the-path0 project" do
it "updates the route of the project correctly" do
subgroup = create(:group, path: "subgroup", parent: namespace)
project = create(:project, path: "the-path0", namespace: subgroup)
subject.rename_path_for_routable(migration_namespace(namespace))
expect(project.route.reload.path).to eq("the-path0/subgroup/the-path0")
end
end
end
context 'for projects' do
let(:parent) { create(:namespace, path: 'the-parent') }
let(:project) { create(:empty_project, path: 'the-path', namespace: parent) }
it 'renames the project called `the-path`' do
subject.rename_path_for_routable(migration_project(project))
expect(project.reload.path).to eq('the-path0')
end
it 'renames the route for the project' do
subject.rename_path_for_routable(project)
expect(project.reload.route.path).to eq('the-parent/the-path0')
end
it 'returns the old & new path' do
old_path, new_path = subject.rename_path_for_routable(migration_project(project))
expect(old_path).to eq('the-parent/the-path')
expect(new_path).to eq('the-parent/the-path0')
end
end
end
describe '#move_pages' do
it 'moves the pages directory' do
expect(subject).to receive(:move_folders)
.with(TestEnv.pages_path, 'old-path', 'new-path')
subject.move_pages('old-path', 'new-path')
end
end
describe "#move_uploads" do
let(:test_dir) { File.join(Rails.root, 'tmp', 'tests', 'rename_reserved_paths') }
let(:uploads_dir) { File.join(test_dir, 'public', 'uploads') }
it 'moves subdirectories in the uploads folder' do
expect(subject).to receive(:uploads_dir).and_return(uploads_dir)
expect(subject).to receive(:move_folders).with(uploads_dir, 'old_path', 'new_path')
subject.move_uploads('old_path', 'new_path')
end
it "doesn't move uploads when they are stored in object storage" do
expect(subject).to receive(:file_storage?).and_return(false)
expect(subject).not_to receive(:move_folders)
subject.move_uploads('old_path', 'new_path')
end
end
describe '#move_folders' do
let(:test_dir) { File.join(Rails.root, 'tmp', 'tests', 'rename_reserved_paths') }
let(:uploads_dir) { File.join(test_dir, 'public', 'uploads') }
before do
FileUtils.remove_dir(test_dir) if File.directory?(test_dir)
FileUtils.mkdir_p(uploads_dir)
allow(subject).to receive(:uploads_dir).and_return(uploads_dir)
end
it 'moves a folder with files' do
source = File.join(uploads_dir, 'parent-group', 'sub-group')
FileUtils.mkdir_p(source)
destination = File.join(uploads_dir, 'parent-group', 'moved-group')
FileUtils.touch(File.join(source, 'test.txt'))
expected_file = File.join(destination, 'test.txt')
subject.move_folders(uploads_dir, File.join('parent-group', 'sub-group'), File.join('parent-group', 'moved-group'))
expect(File.exist?(expected_file)).to be(true)
end
end
end

23
spec/lib/gitlab/database/rename_reserved_paths_migration/namespaces_spec.rb → spec/lib/gitlab/database/rename_reserved_paths_migration/rename_namespaces_spec.rb
View file

@ -1,14 +1,11 @@
require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration::Namespaces, :truncate do
let(:subject) do
ActiveRecord::Migration.new.extend(
Gitlab::Database::RenameReservedPathsMigration
)
end
describe Gitlab::Database::RenameReservedPathsMigration::RenameNamespaces do
let(:migration) { FakeRenameReservedPathMigration.new }
let(:subject) { described_class.new(['the-path'], migration) }
before do
allow(subject).to receive(:say)
allow(migration).to receive(:say)
end
def migration_namespace(namespace)
@ -27,7 +24,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::Namespaces, :truncate d
path: 'the-path',
parent: create(:namespace))
found_ids = subject.namespaces_for_paths(['the-PATH'], type: :wildcard).
found_ids = subject.namespaces_for_paths(type: :wildcard).
map(&:id)
expect(found_ids).to contain_exactly(namespace.id)
end
@ -41,7 +38,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::Namespaces, :truncate d
path: 'the-path',
parent: create(:namespace))
found_ids = subject.namespaces_for_paths(['the-path'], type: :top_level).
found_ids = subject.namespaces_for_paths(type: :top_level).
map(&:id)
expect(found_ids).to contain_exactly(root_namespace.id)
end
@ -98,12 +95,14 @@ describe Gitlab::Database::RenameReservedPathsMigration::Namespaces, :truncate d
describe "#rename_namespace" do
let(:namespace) { create(:namespace, path: 'the-path') }
it 'renames paths & routes for the namesapce' do
it 'renames paths & routes for the namespace' do
expect(subject).to receive(:rename_path_for_routable).
with(namespace).
and_call_original
subject.rename_namespace(namespace)
expect(namespace.reload.path).to eq('the-path0')
end
it "moves the the repository for a project in the namespace" do
@ -140,14 +139,14 @@ describe Gitlab::Database::RenameReservedPathsMigration::Namespaces, :truncate d
expect(subject).to receive(:rename_namespace).
with(migration_namespace(top_level_namespace))
subject.rename_namespaces(['the-path'], type: :top_level)
subject.rename_namespaces(type: :top_level)
end
it 'renames child namespaces' do
expect(subject).to receive(:rename_namespace).
with(migration_namespace(child_namespace))
subject.rename_namespaces(['the-path'], type: :wildcard)
subject.rename_namespaces(type: :wildcard)
end
end
end

15
spec/lib/gitlab/database/rename_reserved_paths_migration/projects_spec.rb → spec/lib/gitlab/database/rename_reserved_paths_migration/rename_projects_spec.rb
View file

@ -1,14 +1,11 @@
require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration::Projects, :truncate do
let(:subject) do
ActiveRecord::Migration.new.extend(
Gitlab::Database::RenameReservedPathsMigration
)
end
describe Gitlab::Database::RenameReservedPathsMigration::RenameProjects do
let(:migration) { FakeRenameReservedPathMigration.new }
let(:subject) { described_class.new(['the-path'], migration) }
before do
allow(subject).to receive(:say)
allow(migration).to receive(:say)
end
describe '#projects_for_paths' do
@ -16,7 +13,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::Projects, :truncate do
project = create(:empty_project, path: 'THE-path')
_other_project = create(:empty_project)
result_ids = subject.projects_for_paths(['the-PATH']).map(&:id)
result_ids = subject.projects_for_paths.map(&:id)
expect(result_ids).to contain_exactly(project.id)
end
@ -35,6 +32,8 @@ describe Gitlab::Database::RenameReservedPathsMigration::Projects, :truncate do
and_call_original
subject.rename_project(project)
expect(project.reload.path).to eq('the-path0')
end
it 'moves the wiki & the repo' do

147
spec/lib/gitlab/database/rename_reserved_paths_migration_spec.rb
View file

@ -1,11 +1,7 @@
require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration do
let(:subject) do
ActiveRecord::Migration.new.extend(
Gitlab::Database::RenameReservedPathsMigration
)
end
let(:subject) { FakeRenameReservedPathMigration.new }
before do
allow(subject).to receive(:say)
@ -13,14 +9,23 @@ describe Gitlab::Database::RenameReservedPathsMigration do
describe '#rename_wildcard_paths' do
it 'should rename namespaces' do
expect(subject).to receive(:rename_namespaces).
with(['first-path', 'second-path'], type: :wildcard)
rename_namespaces = double
expect(Gitlab::Database::RenameReservedPathsMigration::RenameNamespaces).
to receive(:new).with(['first-path', 'second-path'], subject).
and_return(rename_namespaces)
expect(rename_namespaces).to receive(:rename_namespaces).
with(type: :wildcard)
subject.rename_wildcard_paths(['first-path', 'second-path'])
end
it 'should rename projects' do
expect(subject).to receive(:rename_projects).with(['the-path'])
rename_projects = double
expect(Gitlab::Database::RenameReservedPathsMigration::RenameProjects).
to receive(:new).with(['the-path'], subject).
and_return(rename_projects)
expect(rename_projects).to receive(:rename_projects)
subject.rename_wildcard_paths(['the-path'])
end
@ -28,128 +33,14 @@ describe Gitlab::Database::RenameReservedPathsMigration do
describe '#rename_root_paths' do
it 'should rename namespaces' do
expect(subject).to receive(:rename_namespaces).
with(['the-path'], type: :top_level)
rename_namespaces = double
expect(Gitlab::Database::RenameReservedPathsMigration::RenameNamespaces).
to receive(:new).with(['the-path'], subject).
and_return(rename_namespaces)
expect(rename_namespaces).to receive(:rename_namespaces).
with(type: :top_level)
subject.rename_root_paths('the-path')
end
end
describe "#remove_last_ocurrence" do
it "removes only the last occurance of a string" do
input = "this/is/a-word-to-replace/namespace/with/a-word-to-replace"
expect(subject.remove_last_occurrence(input, "a-word-to-replace"))
.to eq("this/is/a-word-to-replace/namespace/with/")
end
end
describe '#rename_path_for_routable' do
context 'for namespaces' do
let(:namespace) { create(:namespace, path: 'the-path') }
it "renames namespaces called the-path" do
subject.rename_path_for_routable(namespace)
expect(namespace.reload.path).to eq("the-path0")
end
it "renames the route to the namespace" do
subject.rename_path_for_routable(namespace)
expect(Namespace.find(namespace.id).full_path).to eq("the-path0")
end
it "renames the route for projects of the namespace" do
project = create(:project, path: "project-path", namespace: namespace)
subject.rename_path_for_routable(namespace)
expect(project.route.reload.path).to eq("the-path0/project-path")
end
it 'returns the old & the new path' do
old_path, new_path = subject.rename_path_for_routable(namespace)
expect(old_path).to eq('the-path')
expect(new_path).to eq('the-path0')
end
context "the-path namespace -> subgroup -> the-path0 project" do
it "updates the route of the project correctly" do
subgroup = create(:group, path: "subgroup", parent: namespace)
project = create(:project, path: "the-path0", namespace: subgroup)
subject.rename_path_for_routable(namespace)
expect(project.route.reload.path).to eq("the-path0/subgroup/the-path0")
end
end
end
context 'for projects' do
let(:parent) { create(:namespace, path: 'the-parent') }
let(:project) { create(:empty_project, path: 'the-path', namespace: parent) }
it 'renames the project called `the-path`' do
subject.rename_path_for_routable(project)
expect(project.reload.path).to eq('the-path0')
end
it 'renames the route for the project' do
subject.rename_path_for_routable(project)
expect(project.reload.route.path).to eq('the-parent/the-path0')
end
it 'returns the old & new path' do
old_path, new_path = subject.rename_path_for_routable(project)
expect(old_path).to eq('the-parent/the-path')
expect(new_path).to eq('the-parent/the-path0')
end
end
end
describe "#move_uploads" do
let(:test_dir) { File.join(Rails.root, 'tmp', 'tests', 'rename_reserved_paths') }
let(:uploads_dir) { File.join(test_dir, 'public', 'uploads') }
it 'moves subdirectories in the uploads folder' do
expect(subject).to receive(:uploads_dir).and_return(uploads_dir)
expect(subject).to receive(:move_folders).with(uploads_dir, 'old_path', 'new_path')
subject.move_uploads('old_path', 'new_path')
end
it "doesn't move uploads when they are stored in object storage" do
expect(subject).to receive(:file_storage?).and_return(false)
expect(subject).not_to receive(:move_folders)
subject.move_uploads('old_path', 'new_path')
end
end
describe '#move_folders' do
let(:test_dir) { File.join(Rails.root, 'tmp', 'tests', 'rename_reserved_paths') }
let(:uploads_dir) { File.join(test_dir, 'public', 'uploads') }
before do
FileUtils.remove_dir(test_dir) if File.directory?(test_dir)
FileUtils.mkdir_p(uploads_dir)
allow(subject).to receive(:uploads_dir).and_return(uploads_dir)
end
it 'moves a folder with files' do
source = File.join(uploads_dir, 'parent-group', 'sub-group')
FileUtils.mkdir_p(source)
destination = File.join(uploads_dir, 'parent-group', 'moved-group')
FileUtils.touch(File.join(source, 'test.txt'))
expected_file = File.join(destination, 'test.txt')
subject.move_folders(uploads_dir, File.join('parent-group', 'sub-group'), File.join('parent-group', 'moved-group'))
expect(File.exist?(expected_file)).to be(true)
end
end
end

3
spec/support/fake_migration_classes.rb Normal file
View file

@ -0,0 +1,3 @@
class FakeRenameReservedPathMigration < ActiveRecord::Migration
include Gitlab::Database::RenameReservedPathsMigration
end