diff --git a/app/models/shard.rb b/app/models/shard.rb new file mode 100644 index 00000000000..2fa22bd040c --- /dev/null +++ b/app/models/shard.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class Shard < ActiveRecord::Base + # Store shard names from the configuration file in the database. This is not a + # list of active shards - we just want to assign an immutable, unique ID to + # every shard name for easy indexing / referencing. + def self.populate! + return unless table_exists? + + # The GitLab config does not change for the lifecycle of the process + in_config = Gitlab.config.repositories.storages.keys.map(&:to_s) + + transaction do + in_db = all.pluck(:name) + missing = in_config - in_db + + missing.map { |name| by_name(name) } + end + end + + def self.by_name(name) + find_or_create_by(name: name) + rescue ActiveRecord::RecordNotUnique + retry + end +end diff --git a/config/initializers/fill_shards.rb b/config/initializers/fill_shards.rb new file mode 100644 index 00000000000..d5071c7d556 --- /dev/null +++ b/config/initializers/fill_shards.rb @@ -0,0 +1,3 @@ +return if Gitlab::Database.read_only? + +Shard.populate! diff --git a/db/migrate/20181019032400_add_shards_table.rb b/db/migrate/20181019032400_add_shards_table.rb new file mode 100644 index 00000000000..5e0a6960548 --- /dev/null +++ b/db/migrate/20181019032400_add_shards_table.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class AddShardsTable < ActiveRecord::Migration + DOWNTIME = false + + def change + create_table :shards do |t| + t.string :name, null: false, index: { unique: true } + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 1a8b556228d..439f70b0198 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1931,6 +1931,12 @@ ActiveRecord::Schema.define(version: 20181101144347) do add_index "services", ["project_id"], name: "index_services_on_project_id", using: :btree add_index "services", ["template"], name: "index_services_on_template", using: :btree + create_table "shards", force: :cascade do |t| + t.string "name", null: false + end + + add_index "shards", ["name"], name: "index_shards_on_name", unique: true, using: :btree + create_table "site_statistics", force: :cascade do |t| t.integer "repositories_count", default: 0, null: false end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index d059854214f..84326724118 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -8,6 +8,7 @@ describe Project do it { is_expected.to belong_to(:group) } it { is_expected.to belong_to(:namespace) } it { is_expected.to belong_to(:creator).class_name('User') } + it { is_expected.to belong_to(:pool_repository) } it { is_expected.to have_many(:users) } it { is_expected.to have_many(:services) } it { is_expected.to have_many(:events) } diff --git a/spec/models/shard_spec.rb b/spec/models/shard_spec.rb new file mode 100644 index 00000000000..83104711b55 --- /dev/null +++ b/spec/models/shard_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literals: true +require 'spec_helper' + +describe Shard do + describe '.populate!' do + it 'creates shards based on the config file' do + expect(described_class.all).to be_empty + + stub_storage_settings(foo: {}, bar: {}, baz: {}) + + described_class.populate! + + expect(described_class.all.map(&:name)).to match_array(%w[default foo bar baz]) + end + end + + describe '.by_name' do + let(:default_shard) { described_class.find_by(name: 'default') } + + before do + described_class.populate! + end + + it 'returns an existing shard' do + expect(described_class.by_name('default')).to eq(default_shard) + end + + it 'creates a new shard' do + result = described_class.by_name('foo') + + expect(result).not_to eq(default_shard) + expect(result.name).to eq('foo') + end + + it 'retries if creation races' do + expect(described_class) + .to receive(:find_or_create_by) + .with(name: 'default') + .and_raise(ActiveRecord::RecordNotUnique, 'fail') + .once + + expect(described_class) + .to receive(:find_or_create_by) + .with(name: 'default') + .and_call_original + + expect(described_class.by_name('default')).to eq(default_shard) + end + end +end