kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

New storage is now "Hashed" instead of "UUID"

Browse source
This commit is contained in:
Gabriel Mazetto 2017-08-04 07:30:42 +02:00
parent 5340339957
commit 9e6fa996ea
8 changed files with 203 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
app/models/concerns/storage/uuid_project.rb → app/models/concerns/storage/hashed_project.rb
View file

@ -1,17 +1,23 @@
module Storage
module UUIDProject
module HashedProject
extend ActiveSupport::Concern
def uuid_dir
%Q(#{uuid[0..1]}/#{uuid[2..3]})
# Base directory
#
# @return [String] directory where repository is stored
def base_dir
%Q(#{disk_hash[0..1]}/#{disk_hash[2..3]}) if disk_hash
end
# Disk path is used to build repository and project's wiki path on disk
#
# @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
def disk_path
%Q(#{uuid_dir}/#{uuid})
%Q(#{base_dir}/#{disk_hash})
end
def ensure_storage_path_exist
gitlab_shell.add_namespace(repository_storage_path, uuid_dir)
gitlab_shell.add_namespace(repository_storage_path, base_dir)
end
def rename_repo
@ -54,5 +60,13 @@ module Storage
Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path)
Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path)
end
private
# Generates the hash for the project path and name on disk
# If you need to refer to the repository on disk, use the `#disk_path`
def disk_hash
@disk_hash ||= Digest::SHA2.hexdigest(self.id.to_s) if self.id
end
end
end

12
app/models/concerns/storage/legacy_project.rb
View file

@ -2,12 +2,22 @@ module Storage
module LegacyProject
extend ActiveSupport::Concern
# Base directory
#
# @return [String] directory where repository is stored
def base_dir
namespace.full_path
end
# Disk path is used to build repository and project's wiki path on disk
#
# @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
def disk_path
full_path
end
def ensure_storage_path_exist
gitlab_shell.add_namespace(repository_storage_path, namespace.full_path)
gitlab_shell.add_namespace(repository_storage_path, base_dir)
end
def rename_repo

8
app/models/project.rb
View file

@ -1424,10 +1424,12 @@ class Project < ActiveRecord::Base
private
def load_storage
if self.storage_version > 1
self.class.include Storage::UUIDProject
return unless has_attribute?(:storage_version)
if self.storage_version && self.storage_version >= 1
self.extend Storage::HashedProject
else
self.class.include Storage::LegacyProject
self.extend Storage::LegacyProject
end
end

24
db/migrate/20170802013652_add_storage_fields_to_project.rb
View file

@ -4,36 +4,16 @@
class AddStorageFieldsToProject < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
# Set this constant to true if this migration requires downtime.
DOWNTIME = false
# When a migration requires downtime you **must** uncomment the following
# constant and define a short and easy to understand explanation as to why the
# migration requires downtime.
# DOWNTIME_REASON = ''
# When using the methods "add_concurrent_index", "remove_concurrent_index" or
# "add_column_with_default" you must disable the use of transactions
# as these methods can not run in an existing transaction.
# When using "add_concurrent_index" or "remove_concurrent_index" methods make sure
# that either of them is the _only_ method called in the migration,
# any other changes should go in a separate migration.
# This ensures that upon failure _only_ the index creation or removing fails
# and can be retried or reverted easily.
#
# To disable transactions uncomment the following line and remove these
# comments:
disable_ddl_transaction!
def up
# rubocop:disable Migration/AddColumnWithDefaultToLargeTable
add_column :projects, :uuid, :uuid
add_column_with_default :projects, :storage_version, :integer, default: 0, limit: 1
add_concurrent_index :projects, :uuid
add_column :projects, :storage_version, :integer, limit: 2
add_concurrent_index :projects, :storage_version
end
def down
remove_column :projects, :uuid
remove_column :projects, :storage_version
end
end

5
db/schema.rb
View file

@ -1208,8 +1208,7 @@ ActiveRecord::Schema.define(version: 20170820100558) do
t.datetime "last_repository_updated_at"
t.string "ci_config_path"
t.text "delete_error"
t.uuid "uuid"
t.integer "storage_version", limit: 2, default: 0, null: false
t.integer "storage_version", limit: 2
end
add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
@ -1226,7 +1225,7 @@ ActiveRecord::Schema.define(version: 20170820100558) do
add_index "projects", ["pending_delete"], name: "index_projects_on_pending_delete", using: :btree
add_index "projects", ["runners_token"], name: "index_projects_on_runners_token", using: :btree
add_index "projects", ["star_count"], name: "index_projects_on_star_count", using: :btree
add_index "projects", ["uuid"], name: "index_projects_on_uuid", using: :btree
add_index "projects", ["storage_version"], name: "index_projects_on_storage_version", using: :btree
add_index "projects", ["visibility_level"], name: "index_projects_on_visibility_level", using: :btree
create_table "protected_branch_merge_access_levels", force: :cascade do |t|

1
lib/gitlab/import_export/import_export.yml
View file

@ -98,6 +98,7 @@ excluded_attributes:
- :last_activity_at
- :last_repository_updated_at
- :last_repository_check_at
- :storage_version
snippets:
- :expired_at
merge_request_diff:

4
spec/factories/projects.rb
View file

@ -81,6 +81,10 @@ FactoryGirl.define do
archived true
end
trait :hashed do
storage_version 1
end
trait :access_requestable do
request_access_enabled true
end

213
spec/models/project_spec.rb
View file

@ -1251,60 +1251,6 @@ describe Project do
end
end
describe '#rename_repo' do
let(:project) { create(:project, :repository) }
let(:gitlab_shell) { Gitlab::Shell.new }
before do
# Project#gitlab_shell returns a new instance of Gitlab::Shell on every
# call. This makes testing a bit easier.
allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
allow(project).to receive(:previous_changes).and_return('path' => ['foo'])
end
it 'renames a repository' do
stub_container_registry_config(enabled: false)
expect(gitlab_shell).to receive(:mv_repository)
.ordered
.with(project.repository_storage_path, "#{project.namespace.full_path}/foo", "#{project.full_path}")
.and_return(true)
expect(gitlab_shell).to receive(:mv_repository)
.ordered
.with(project.repository_storage_path, "#{project.namespace.full_path}/foo.wiki", "#{project.full_path}.wiki")
.and_return(true)
expect_any_instance_of(SystemHooksService)
.to receive(:execute_hooks_for)
.with(project, :rename)
expect_any_instance_of(Gitlab::UploadsTransfer)
.to receive(:rename_project)
.with('foo', project.path, project.namespace.full_path)
expect(project).to receive(:expire_caches_before_rename)
expect(project).to receive(:expires_full_path_cache)
project.rename_repo
end
context 'container registry with images' do
let(:container_repository) { create(:container_repository) }
before do
stub_container_registry_config(enabled: true)
stub_container_registry_tags(repository: :any, tags: ['tag'])
project.container_repositories << container_repository
end
subject { project.rename_repo }
it { expect {subject}.to raise_error(StandardError) }
end
end
describe '#expire_caches_before_rename' do
let(:project) { create(:project, :repository) }
let(:repo) { double(:repo, exists?: true) }
@ -2367,4 +2313,163 @@ describe Project do
expect(project.forks_count).to eq(1)
end
end
context 'legacy storage' do
let(:project) { create(:project, :repository) }
let(:gitlab_shell) { Gitlab::Shell.new }
describe '#base_dir' do
it 'returns base_dir based on namespace only' do
expect(project.base_dir).to eq(project.namespace.full_path)
end
end
describe '#disk_path' do
it 'returns disk_path based on namespace and project path' do
expect(project.disk_path).to eq("#{project.namespace.full_path}/#{project.path}")
end
end
describe '#ensure_storage_path_exist' do
before do
allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
end
it 'delegates to gitlab_shell to ensure namespace is created' do
expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, project.base_dir)
project.ensure_storage_path_exist
end
end
describe '#rename_repo' do
before do
# Project#gitlab_shell returns a new instance of Gitlab::Shell on every
# call. This makes testing a bit easier.
allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
allow(project).to receive(:previous_changes).and_return('path' => ['foo'])
end
it 'renames a repository' do
stub_container_registry_config(enabled: false)
expect(gitlab_shell).to receive(:mv_repository)
.ordered
.with(project.repository_storage_path, "#{project.namespace.full_path}/foo", "#{project.full_path}")
.and_return(true)
expect(gitlab_shell).to receive(:mv_repository)
.ordered
.with(project.repository_storage_path, "#{project.namespace.full_path}/foo.wiki", "#{project.full_path}.wiki")
.and_return(true)
expect_any_instance_of(SystemHooksService)
.to receive(:execute_hooks_for)
.with(project, :rename)
expect_any_instance_of(Gitlab::UploadsTransfer)
.to receive(:rename_project)
.with('foo', project.path, project.namespace.full_path)
expect(project).to receive(:expire_caches_before_rename)
expect(project).to receive(:expires_full_path_cache)
project.rename_repo
end
context 'container registry with images' do
let(:container_repository) { create(:container_repository) }
before do
stub_container_registry_config(enabled: true)
stub_container_registry_tags(repository: :any, tags: ['tag'])
project.container_repositories << container_repository
end
subject { project.rename_repo }
it { expect{subject}.to raise_error(StandardError) }
end
end
end
context 'hashed storage' do
let(:project) { create(:project, :repository, :hashed) }
let(:gitlab_shell) { Gitlab::Shell.new }
let(:hash) { '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' }
before do
allow(Digest::SHA2).to receive(:hexdigest) { hash }
end
describe '#base_dir' do
it 'returns base_dir based on hash of project id' do
expect(project.base_dir).to eq('6b/86')
end
end
describe '#disk_path' do
it 'returns disk_path based on has of project id' do
hashed_path = '6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b'
expect(project.disk_path).to eq(hashed_path)
end
end
describe '#ensure_storage_path_exist' do
before do
allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
end
it 'delegates to gitlab_shell to ensure namespace is created' do
expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, '6b/86')
project.ensure_storage_path_exist
end
end
describe '#rename_repo' do
before do
# Project#gitlab_shell returns a new instance of Gitlab::Shell on every
# call. This makes testing a bit easier.
allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
allow(project).to receive(:previous_changes).and_return('path' => ['foo'])
end
it 'renames a repository' do
stub_container_registry_config(enabled: false)
expect(gitlab_shell).not_to receive(:mv_repository)
expect_any_instance_of(SystemHooksService)
.to receive(:execute_hooks_for)
.with(project, :rename)
expect_any_instance_of(Gitlab::UploadsTransfer)
.to receive(:rename_project)
.with('foo', project.path, project.namespace.full_path)
expect(project).to receive(:expire_caches_before_rename)
expect(project).to receive(:expires_full_path_cache)
project.rename_repo
end
context 'container registry with images' do
let(:container_repository) { create(:container_repository) }
before do
stub_container_registry_config(enabled: true)
stub_container_registry_tags(repository: :any, tags: ['tag'])
project.container_repositories << container_repository
end
subject { project.rename_repo }
it { expect{subject}.to raise_error(StandardError) }
end
end
end
end