Merge branch '52300-pool-repositories' into 'master'
Start tracking pool repositories Closes #38238 and #52300 See merge request gitlab-org/gitlab-ce!22482
This commit is contained in:
commit
19c5468a27
|
@ -0,0 +1,22 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class PoolRepository < ActiveRecord::Base
|
||||
POOL_PREFIX = '@pools'
|
||||
|
||||
belongs_to :shard
|
||||
validates :shard, presence: true
|
||||
|
||||
# For now, only pool repositories are tracked in the database. However, we may
|
||||
# want to add other repository types in the future
|
||||
self.table_name = 'repositories'
|
||||
|
||||
has_many :pool_member_projects, class_name: 'Project', foreign_key: :pool_repository_id
|
||||
|
||||
def shard_name
|
||||
shard&.name
|
||||
end
|
||||
|
||||
def shard_name=(name)
|
||||
self.shard = Shard.by_name(name)
|
||||
end
|
||||
end
|
|
@ -124,6 +124,7 @@ class Project < ActiveRecord::Base
|
|||
alias_attribute :title, :name
|
||||
|
||||
# Relations
|
||||
belongs_to :pool_repository
|
||||
belongs_to :creator, class_name: 'User'
|
||||
belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id'
|
||||
belongs_to :namespace
|
||||
|
|
|
@ -0,0 +1,26 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class Shard < ActiveRecord::Base
|
||||
# Store shard names from the configuration file in the database. This is not a
|
||||
# list of active shards - we just want to assign an immutable, unique ID to
|
||||
# every shard name for easy indexing / referencing.
|
||||
def self.populate!
|
||||
return unless table_exists?
|
||||
|
||||
# The GitLab config does not change for the lifecycle of the process
|
||||
in_config = Gitlab.config.repositories.storages.keys.map(&:to_s)
|
||||
|
||||
transaction do
|
||||
in_db = all.pluck(:name)
|
||||
missing = in_config - in_db
|
||||
|
||||
missing.map { |name| by_name(name) }
|
||||
end
|
||||
end
|
||||
|
||||
def self.by_name(name)
|
||||
find_or_create_by(name: name)
|
||||
rescue ActiveRecord::RecordNotUnique
|
||||
retry
|
||||
end
|
||||
end
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
title: Start tracking shards and pool repositories in the database
|
||||
merge_request: 22482
|
||||
author:
|
||||
type: other
|
|
@ -0,0 +1,4 @@
|
|||
return unless Shard.connected?
|
||||
return if Gitlab::Database.read_only?
|
||||
|
||||
Shard.populate!
|
|
@ -0,0 +1,11 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class AddShardsTable < ActiveRecord::Migration
|
||||
DOWNTIME = false
|
||||
|
||||
def change
|
||||
create_table :shards do |t|
|
||||
t.string :name, null: false, index: { unique: true }
|
||||
end
|
||||
end
|
||||
end
|
|
@ -0,0 +1,15 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class AddRepositoriesTable < ActiveRecord::Migration
|
||||
DOWNTIME = false
|
||||
|
||||
def change
|
||||
create_table :repositories, id: :bigserial do |t|
|
||||
t.references :shard, null: false, index: true, foreign_key: { on_delete: :restrict }
|
||||
t.string :disk_path, null: false, index: { unique: true }
|
||||
end
|
||||
|
||||
add_column :projects, :pool_repository_id, :bigint
|
||||
add_index :projects, :pool_repository_id, where: 'pool_repository_id IS NOT NULL'
|
||||
end
|
||||
end
|
|
@ -0,0 +1,22 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class AddProjectsPoolRepositoryIdForeignKey < ActiveRecord::Migration
|
||||
include Gitlab::Database::MigrationHelpers
|
||||
|
||||
DOWNTIME = false
|
||||
|
||||
disable_ddl_transaction!
|
||||
|
||||
def up
|
||||
add_concurrent_foreign_key(
|
||||
:projects,
|
||||
:repositories,
|
||||
column: :pool_repository_id,
|
||||
on_delete: :nullify
|
||||
)
|
||||
end
|
||||
|
||||
def down
|
||||
remove_foreign_key(:projects, column: :pool_repository_id)
|
||||
end
|
||||
end
|
18
db/schema.rb
18
db/schema.rb
|
@ -1703,6 +1703,7 @@ ActiveRecord::Schema.define(version: 20181101144347) do
|
|||
t.integer "jobs_cache_index"
|
||||
t.boolean "pages_https_only", default: true
|
||||
t.boolean "remote_mirror_available_overridden"
|
||||
t.integer "pool_repository_id", limit: 8
|
||||
end
|
||||
|
||||
add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
|
||||
|
@ -1719,6 +1720,7 @@ ActiveRecord::Schema.define(version: 20181101144347) do
|
|||
add_index "projects", ["path"], name: "index_projects_on_path", using: :btree
|
||||
add_index "projects", ["path"], name: "index_projects_on_path_trigram", using: :gin, opclasses: {"path"=>"gin_trgm_ops"}
|
||||
add_index "projects", ["pending_delete"], name: "index_projects_on_pending_delete", using: :btree
|
||||
add_index "projects", ["pool_repository_id"], name: "index_projects_on_pool_repository_id", where: "(pool_repository_id IS NOT NULL)", using: :btree
|
||||
add_index "projects", ["repository_storage", "created_at"], name: "idx_project_repository_check_partial", where: "(last_repository_check_at IS NULL)", using: :btree
|
||||
add_index "projects", ["repository_storage"], name: "index_projects_on_repository_storage", using: :btree
|
||||
add_index "projects", ["runners_token"], name: "index_projects_on_runners_token", using: :btree
|
||||
|
@ -1851,6 +1853,14 @@ ActiveRecord::Schema.define(version: 20181101144347) do
|
|||
add_index "remote_mirrors", ["last_successful_update_at"], name: "index_remote_mirrors_on_last_successful_update_at", using: :btree
|
||||
add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree
|
||||
|
||||
create_table "repositories", id: :bigserial, force: :cascade do |t|
|
||||
t.integer "shard_id", null: false
|
||||
t.string "disk_path", null: false
|
||||
end
|
||||
|
||||
add_index "repositories", ["disk_path"], name: "index_repositories_on_disk_path", unique: true, using: :btree
|
||||
add_index "repositories", ["shard_id"], name: "index_repositories_on_shard_id", using: :btree
|
||||
|
||||
create_table "repository_languages", id: false, force: :cascade do |t|
|
||||
t.integer "project_id", null: false
|
||||
t.integer "programming_language_id", null: false
|
||||
|
@ -1931,6 +1941,12 @@ ActiveRecord::Schema.define(version: 20181101144347) do
|
|||
add_index "services", ["project_id"], name: "index_services_on_project_id", using: :btree
|
||||
add_index "services", ["template"], name: "index_services_on_template", using: :btree
|
||||
|
||||
create_table "shards", force: :cascade do |t|
|
||||
t.string "name", null: false
|
||||
end
|
||||
|
||||
add_index "shards", ["name"], name: "index_shards_on_name", unique: true, using: :btree
|
||||
|
||||
create_table "site_statistics", force: :cascade do |t|
|
||||
t.integer "repositories_count", default: 0, null: false
|
||||
end
|
||||
|
@ -2450,6 +2466,7 @@ ActiveRecord::Schema.define(version: 20181101144347) do
|
|||
add_foreign_key "project_import_data", "projects", name: "fk_ffb9ee3a10", on_delete: :cascade
|
||||
add_foreign_key "project_mirror_data", "projects", on_delete: :cascade
|
||||
add_foreign_key "project_statistics", "projects", on_delete: :cascade
|
||||
add_foreign_key "projects", "repositories", column: "pool_repository_id", name: "fk_6e5c14658a", on_delete: :nullify
|
||||
add_foreign_key "prometheus_metrics", "projects", on_delete: :cascade
|
||||
add_foreign_key "protected_branch_merge_access_levels", "protected_branches", name: "fk_8a3072ccb3", on_delete: :cascade
|
||||
add_foreign_key "protected_branch_push_access_levels", "protected_branches", name: "fk_9ffc86a3d9", on_delete: :cascade
|
||||
|
@ -2461,6 +2478,7 @@ ActiveRecord::Schema.define(version: 20181101144347) do
|
|||
add_foreign_key "push_event_payloads", "events", name: "fk_36c74129da", on_delete: :cascade
|
||||
add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade
|
||||
add_foreign_key "remote_mirrors", "projects", on_delete: :cascade
|
||||
add_foreign_key "repositories", "shards", on_delete: :restrict
|
||||
add_foreign_key "repository_languages", "projects", on_delete: :cascade
|
||||
add_foreign_key "resource_label_events", "issues", on_delete: :cascade
|
||||
add_foreign_key "resource_label_events", "labels", on_delete: :nullify
|
||||
|
|
|
@ -92,6 +92,7 @@ excluded_attributes:
|
|||
- :path
|
||||
- :namespace_id
|
||||
- :creator_id
|
||||
- :pool_repository_id
|
||||
- :import_url
|
||||
- :import_status
|
||||
- :avatar
|
||||
|
|
|
@ -299,6 +299,7 @@ project:
|
|||
- ci_cd_settings
|
||||
- import_export_upload
|
||||
- repository_languages
|
||||
- pool_repository
|
||||
award_emoji:
|
||||
- awardable
|
||||
- user
|
||||
|
|
|
@ -8,6 +8,7 @@ describe Project do
|
|||
it { is_expected.to belong_to(:group) }
|
||||
it { is_expected.to belong_to(:namespace) }
|
||||
it { is_expected.to belong_to(:creator).class_name('User') }
|
||||
it { is_expected.to belong_to(:pool_repository) }
|
||||
it { is_expected.to have_many(:users) }
|
||||
it { is_expected.to have_many(:services) }
|
||||
it { is_expected.to have_many(:events) }
|
||||
|
|
|
@ -0,0 +1,50 @@
|
|||
# frozen_string_literals: true
|
||||
require 'spec_helper'
|
||||
|
||||
describe Shard do
|
||||
describe '.populate!' do
|
||||
it 'creates shards based on the config file' do
|
||||
expect(described_class.all).to be_empty
|
||||
|
||||
stub_storage_settings(foo: {}, bar: {}, baz: {})
|
||||
|
||||
described_class.populate!
|
||||
|
||||
expect(described_class.all.map(&:name)).to match_array(%w[default foo bar baz])
|
||||
end
|
||||
end
|
||||
|
||||
describe '.by_name' do
|
||||
let(:default_shard) { described_class.find_by(name: 'default') }
|
||||
|
||||
before do
|
||||
described_class.populate!
|
||||
end
|
||||
|
||||
it 'returns an existing shard' do
|
||||
expect(described_class.by_name('default')).to eq(default_shard)
|
||||
end
|
||||
|
||||
it 'creates a new shard' do
|
||||
result = described_class.by_name('foo')
|
||||
|
||||
expect(result).not_to eq(default_shard)
|
||||
expect(result.name).to eq('foo')
|
||||
end
|
||||
|
||||
it 'retries if creation races' do
|
||||
expect(described_class)
|
||||
.to receive(:find_or_create_by)
|
||||
.with(name: 'default')
|
||||
.and_raise(ActiveRecord::RecordNotUnique, 'fail')
|
||||
.once
|
||||
|
||||
expect(described_class)
|
||||
.to receive(:find_or_create_by)
|
||||
.with(name: 'default')
|
||||
.and_call_original
|
||||
|
||||
expect(described_class.by_name('default')).to eq(default_shard)
|
||||
end
|
||||
end
|
||||
end
|
Loading…
Reference in New Issue