Merge branch '28447-hybrid-repository-storages' into 'master'

Update storage settings to allow extra values per shard

See merge request !9597
This commit is contained in:
Sean McGivern 2017-03-07 16:55:03 +00:00
commit a5db7f5425
28 changed files with 178 additions and 65 deletions

View File

@ -81,8 +81,8 @@ module ApplicationSettingsHelper
end
def repository_storages_options_for_select
options = Gitlab.config.repositories.storages.map do |name, path|
["#{name} - #{path}", name]
options = Gitlab.config.repositories.storages.map do |name, storage|
["#{name} - #{storage['path']}", name]
end
options_for_select(options, @application_setting.repository_storages)

View File

@ -392,7 +392,7 @@ class Project < ActiveRecord::Base
end
def repository_storage_path
Gitlab.config.repositories.storages[repository_storage]
Gitlab.config.repositories.storages[repository_storage]['path']
end
def team

View File

@ -50,10 +50,6 @@ class Repository
end
end
def self.storages
Gitlab.config.repositories.storages
end
def initialize(path_with_namespace, project)
@path_with_namespace = path_with_namespace
@project = project

View File

@ -3,8 +3,8 @@ class PostReceive
include DedicatedSidekiqQueue
def perform(repo_path, identifier, changes)
if path = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1].to_s) }
repo_path.gsub!(path[1].to_s, "")
if repository_storage = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1]['path'].to_s) }
repo_path.gsub!(repository_storage[1]['path'].to_s, "")
else
log("Check gitlab.yml config for correct repositories.storages values. No repository storage path matches \"#{repo_path}\"")
end

View File

@ -0,0 +1,4 @@
---
title: Update storage settings to allow extra values per repository storage
merge_request: 9597
author:

View File

@ -461,7 +461,8 @@ production: &base
# gitlab-shell invokes Dir.pwd inside the repository path and that results
# real path not the symlink.
storages: # You must have at least a `default` storage path.
default: /home/git/repositories/
default:
path: /home/git/repositories/
## Backup settings
backup:
@ -574,7 +575,8 @@ test:
path: tmp/tests/gitlab-satellites/
repositories:
storages:
default: tmp/tests/repositories/
default:
path: tmp/tests/repositories/
backup:
path: tmp/tests/backups
gitlab_shell:

View File

@ -83,7 +83,7 @@ class Settings < Settingslogic
def base_url(config)
custom_port = on_standard_port?(config) ? nil : ":#{config.port}"
[
config.protocol,
"://",
@ -366,8 +366,13 @@ Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.send(:build_gitlab_shell_s
#
Settings['repositories'] ||= Settingslogic.new({})
Settings.repositories['storages'] ||= {}
# Setting gitlab_shell.repos_path is DEPRECATED and WILL BE REMOVED in version 9.0
Settings.repositories.storages['default'] ||= Settings.gitlab_shell['repos_path'] || Settings.gitlab['user_home'] + '/repositories/'
unless Settings.repositories.storages['default']
Settings.repositories.storages['default'] ||= {}
# We set the path only if the default storage doesn't exist, in case it exists
# but follows the pre-9.0 configuration structure. `6_validations.rb` initializer
# will validate all storages and throw a relevant error to the user if necessary.
Settings.repositories.storages['default']['path'] ||= Settings.gitlab['user_home'] + '/repositories/'
end
#
# The repository_downloads_path is used to remove outdated repository
@ -376,11 +381,11 @@ Settings.repositories.storages['default'] ||= Settings.gitlab_shell['repos_path'
# data-integrity issue. In this case, we sets it to the default
# repository_downloads_path value.
#
repositories_storages_path = Settings.repositories.storages.values
repositories_storages = Settings.repositories.storages.values
repository_downloads_path = Settings.gitlab['repository_downloads_path'].to_s.gsub(/\/$/, '')
repository_downloads_full_path = File.expand_path(repository_downloads_path, Settings.gitlab['user_home'])
if repository_downloads_path.blank? || repositories_storages_path.any? { |path| [repository_downloads_path, repository_downloads_full_path].include?(path.gsub(/\/$/, '')) }
if repository_downloads_path.blank? || repositories_storages.any? { |rs| [repository_downloads_path, repository_downloads_full_path].include?(rs['path'].gsub(/\/$/, '')) }
Settings.gitlab['repository_downloads_path'] = File.join(Settings.shared['path'], 'cache/archive')
end

View File

@ -4,8 +4,8 @@ end
def find_parent_path(name, path)
parent = Pathname.new(path).realpath.parent
Gitlab.config.repositories.storages.detect do |n, p|
name != n && Pathname.new(p).realpath == parent
Gitlab.config.repositories.storages.detect do |n, rs|
name != n && Pathname.new(rs['path']).realpath == parent
end
end
@ -16,10 +16,22 @@ end
def validate_storages
storage_validation_error('No repository storage path defined') if Gitlab.config.repositories.storages.empty?
Gitlab.config.repositories.storages.each do |name, path|
Gitlab.config.repositories.storages.each do |name, repository_storage|
storage_validation_error("\"#{name}\" is not a valid storage name") unless storage_name_valid?(name)
parent_name, _parent_path = find_parent_path(name, path)
if repository_storage.is_a?(String)
error = "#{name} is not a valid storage, because it has no `path` key. " \
"It may be configured as:\n\n#{name}:\n path: #{repository_storage}\n\n" \
"Refer to gitlab.yml.example for an updated example"
storage_validation_error(error)
end
if !repository_storage.is_a?(Hash) || repository_storage['path'].nil?
storage_validation_error("#{name} is not a valid storage, because it has no `path` key. Refer to gitlab.yml.example for an updated example")
end
parent_name, _parent_path = find_parent_path(name, repository_storage['path'])
if parent_name
storage_validation_error("#{name} is a nested path of #{parent_name}. Nested paths are not supported for repository storages")
end

View File

@ -8,7 +8,7 @@ class MigrateRepoSize < ActiveRecord::Migration
project_data.each do |project|
id = project['id']
namespace_path = project['namespace_path'] || ''
repos_path = Gitlab.config.gitlab_shell['repos_path'] || Gitlab.config.repositories.storages.default
repos_path = Gitlab.config.gitlab_shell['repos_path'] || Gitlab.config.repositories.storages.default['path']
path = File.join(repos_path, namespace_path, project['project_path'] + '.git')
begin

View File

@ -12,7 +12,7 @@ class MigrateProcessCommitWorkerJobs < ActiveRecord::Migration
end
def repository_storage_path
Gitlab.config.repositories.storages[repository_storage]
Gitlab.config.repositories.storages[repository_storage]['path']
end
def repository_path

View File

@ -60,7 +60,7 @@ class RemoveDotGitFromGroupNames < ActiveRecord::Migration
def move_namespace(group_id, path_was, path)
repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row|
Gitlab.config.repositories.storages[row['repository_storage']]
Gitlab.config.repositories.storages[row['repository_storage']]['path']
end.compact
# Move the namespace directory in all storages paths used by member projects

View File

@ -71,7 +71,7 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
route_exists = route_exists?(path)
Gitlab.config.repositories.storages.each_value do |storage|
if route_exists || path_exists?(path, storage)
if route_exists || path_exists?(path, storage['path'])
counter += 1
path = "#{base}#{counter}"
@ -84,7 +84,7 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration
def move_namespace(namespace_id, path_was, path)
repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row|
Gitlab.config.repositories.storages[row['repository_storage']]
Gitlab.config.repositories.storages[row['repository_storage']]['path']
end.compact
# Move the namespace directory in all storages paths used by member projects

View File

@ -52,9 +52,12 @@ respectively.
# Paths where repositories can be stored. Give the canonicalized absolute pathname.
# NOTE: REPOS PATHS MUST NOT CONTAIN ANY SYMLINK!!!
storages: # You must have at least a 'default' storage path.
default: /home/git/repositories
nfs: /mnt/nfs/repositories
cephfs: /mnt/cephfs/repositories
default:
path: /home/git/repositories
nfs:
path: /mnt/nfs/repositories
cephfs:
path: /mnt/cephfs/repositories
```
1. [Restart GitLab] for the changes to take effect.
@ -75,9 +78,9 @@ working, you can remove the `repos_path` line.
```ruby
git_data_dirs({
"default" => "/var/opt/gitlab/git-data",
"nfs" => "/mnt/nfs/git-data",
"cephfs" => "/mnt/cephfs/git-data"
"default" => { "path" => "/var/opt/gitlab/git-data" },
"nfs" => { "path" => "/mnt/nfs/git-data" },
"cephfs" => { "path" => "/mnt/cephfs/git-data" }
})
```

View File

@ -1,3 +1,66 @@
#### Configuration changes for repository storages
This version introduces a new configuration structure for repository storages.
Update your current configuration as follows, replacing with your storages names and paths:
**For installations from source**
1. Update your `gitlab.yml`, from
```yaml
repositories:
storages: # You must have at least a 'default' storage path.
default: /home/git/repositories
nfs: /mnt/nfs/repositories
cephfs: /mnt/cephfs/repositories
```
to
```yaml
repositories:
storages: # You must have at least a 'default' storage path.
default:
path: /home/git/repositories
nfs:
path: /mnt/nfs/repositories
cephfs:
path: /mnt/cephfs/repositories
```
**For Omnibus installations**
1. Upate your `/etc/gitlab/gitlab.rb`, from
```ruby
git_data_dirs({
"default" => "/var/opt/gitlab/git-data",
"nfs" => "/mnt/nfs/git-data",
"cephfs" => "/mnt/cephfs/git-data"
})
```
to
```ruby
git_data_dirs({
"default" => { "path" => "/var/opt/gitlab/git-data" },
"nfs" => { "path" => "/mnt/nfs/git-data" },
"cephfs" => { "path" => "/mnt/cephfs/git-data" }
})
```
#### Git configuration
Configure Git to generate packfile bitmaps (introduced in Git 2.0) on
the GitLab server during `git gc`.
```sh
cd /home/git/gitlab
sudo -u git -H git config --global repack.writeBitmaps true
```
#### Nginx configuration
Ensure you're still up-to-date with the latest NGINX configuration changes:
@ -12,7 +75,7 @@ git diff origin/8-17-stable:lib/support/nginx/gitlab-ssl origin/9-0-stable:lib/s
git diff origin/8-17-stable:lib/support/nginx/gitlab origin/9-0-stable:lib/support/nginx/gitlab
```
If you are using Strict-Transport-Security in your installation to continue using it you must enable it in your Nginx
If you are using Strict-Transport-Security in your installation to continue using it you must enable it in your Nginx
configuration as GitLab application no longer handles setting it.
If you are using Apache instead of NGINX please see the updated [Apache templates].

View File

@ -9,11 +9,11 @@ module API
# In addition, they may have a '.git' extension and multiple namespaces
#
# Transform all these cases to 'namespace/project'
def clean_project_path(project_path, storage_paths = Repository.storages.values)
def clean_project_path(project_path, storages = Gitlab.config.repositories.storages.values)
project_path = project_path.sub(/\.git\z/, '')
storage_paths.each do |storage_path|
storage_path = File.expand_path(storage_path)
storages.each do |storage|
storage_path = File.expand_path(storage['path'])
if project_path.start_with?(storage_path)
project_path = project_path.sub(storage_path, '')

View File

@ -68,7 +68,8 @@ module Backup
end
def restore
Gitlab.config.repositories.storages.each do |name, path|
Gitlab.config.repositories.storages.each do |name, repository_storage|
path = repository_storage['path']
next unless File.exist?(path)
# Move repos dir to 'repositories.old' dir
@ -199,7 +200,7 @@ module Backup
private
def repository_storage_paths_args
Gitlab.config.repositories.storages.values
Gitlab.config.repositories.storages.values.map { |rs| rs['path'] }
end
end
end

View File

@ -354,7 +354,8 @@ namespace :gitlab do
def check_repo_base_exists
puts "Repo base directory exists?"
Gitlab.config.repositories.storages.each do |name, repo_base_path|
Gitlab.config.repositories.storages.each do |name, repository_storage|
repo_base_path = repository_storage['path']
print "#{name}... "
if File.exist?(repo_base_path)
@ -378,7 +379,8 @@ namespace :gitlab do
def check_repo_base_is_not_symlink
puts "Repo storage directories are symlinks?"
Gitlab.config.repositories.storages.each do |name, repo_base_path|
Gitlab.config.repositories.storages.each do |name, repository_storage|
repo_base_path = repository_storage['path']
print "#{name}... "
unless File.exist?(repo_base_path)
@ -401,7 +403,8 @@ namespace :gitlab do
def check_repo_base_permissions
puts "Repo paths access is drwxrws---?"
Gitlab.config.repositories.storages.each do |name, repo_base_path|
Gitlab.config.repositories.storages.each do |name, repository_storage|
repo_base_path = repository_storage['path']
print "#{name}... "
unless File.exist?(repo_base_path)
@ -431,7 +434,8 @@ namespace :gitlab do
gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group
puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?"
Gitlab.config.repositories.storages.each do |name, repo_base_path|
Gitlab.config.repositories.storages.each do |name, repository_storage|
repo_base_path = repository_storage['path']
print "#{name}... "
unless File.exist?(repo_base_path)
@ -810,8 +814,8 @@ namespace :gitlab do
namespace :repo do
desc "GitLab | Check the integrity of the repositories managed by GitLab"
task check: :environment do
Gitlab.config.repositories.storages.each do |name, path|
namespace_dirs = Dir.glob(File.join(path, '*'))
Gitlab.config.repositories.storages.each do |name, repository_storage|
namespace_dirs = Dir.glob(File.join(repository_storage['path'], '*'))
namespace_dirs.each do |namespace_dir|
repo_dirs = Dir.glob(File.join(namespace_dir, '*'))

View File

@ -6,7 +6,8 @@ namespace :gitlab do
remove_flag = ENV['REMOVE']
namespaces = Namespace.pluck(:path)
Gitlab.config.repositories.storages.each do |name, git_base_path|
Gitlab.config.repositories.storages.each do |name, repository_storage|
git_base_path = repository_storage['path']
all_dirs = Dir.glob(git_base_path + '/*')
puts git_base_path.color(:yellow)
@ -47,7 +48,8 @@ namespace :gitlab do
warn_user_is_not_gitlab
move_suffix = "+orphaned+#{Time.now.to_i}"
Gitlab.config.repositories.storages.each do |name, repo_root|
Gitlab.config.repositories.storages.each do |name, repository_storage|
repo_root = repository_storage['path']
# Look for global repos (legacy, depth 1) and normal repos (depth 2)
IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
find.each_line do |path|

View File

@ -11,7 +11,8 @@ namespace :gitlab do
#
desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
task repos: :environment do
Gitlab.config.repositories.storages.each do |name, git_base_path|
Gitlab.config.repositories.storages.each_value do |repository_storage|
git_base_path = repository_storage['path']
repos_to_import = Dir.glob(git_base_path + '/**/*.git')
repos_to_import.each do |repo_path|

View File

@ -65,8 +65,8 @@ namespace :gitlab do
puts "GitLab Shell".color(:yellow)
puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}"
puts "Repository storage paths:"
Gitlab.config.repositories.storages.each do |name, path|
puts "- #{name}: \t#{path}"
Gitlab.config.repositories.storages.each do |name, repository_storage|
puts "- #{name}: \t#{repository_storage['path']}"
end
puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}"
puts "Git:\t\t#{Gitlab.config.git.bin_path}"

View File

@ -130,8 +130,8 @@ module Gitlab
end
def all_repos
Gitlab.config.repositories.storages.each do |name, path|
IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
Gitlab.config.repositories.storages.each_value do |repository_storage|
IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
find.each_line do |path|
yield path.chomp
end
@ -140,7 +140,7 @@ module Gitlab
end
def repository_storage_paths_args
Gitlab.config.repositories.storages.values
Gitlab.config.repositories.storages.values.map { |rs| rs['path'] }
end
def user_home

View File

@ -14,7 +14,7 @@ describe '6_validations', lib: true do
context 'with correct settings' do
before do
mock_storages('foo' => 'tmp/tests/paths/a/b/c', 'bar' => 'tmp/tests/paths/a/b/d')
mock_storages('foo' => { 'path' => 'tmp/tests/paths/a/b/c' }, 'bar' => { 'path' => 'tmp/tests/paths/a/b/d' })
end
it 'passes through' do
@ -24,7 +24,7 @@ describe '6_validations', lib: true do
context 'with invalid storage names' do
before do
mock_storages('name with spaces' => 'tmp/tests/paths/a/b/c')
mock_storages('name with spaces' => { 'path' => 'tmp/tests/paths/a/b/c' })
end
it 'throws an error' do
@ -34,7 +34,7 @@ describe '6_validations', lib: true do
context 'with nested storage paths' do
before do
mock_storages('foo' => 'tmp/tests/paths/a/b/c', 'bar' => 'tmp/tests/paths/a/b/c/d')
mock_storages('foo' => { 'path' => 'tmp/tests/paths/a/b/c' }, 'bar' => { 'path' => 'tmp/tests/paths/a/b/c/d' })
end
it 'throws an error' do
@ -44,7 +44,7 @@ describe '6_validations', lib: true do
context 'with similar but un-nested storage paths' do
before do
mock_storages('foo' => 'tmp/tests/paths/a/b/c', 'bar' => 'tmp/tests/paths/a/b/c2')
mock_storages('foo' => { 'path' => 'tmp/tests/paths/a/b/c' }, 'bar' => { 'path' => 'tmp/tests/paths/a/b/c2' })
end
it 'passes through' do
@ -52,6 +52,26 @@ describe '6_validations', lib: true do
end
end
context 'with incomplete settings' do
before do
mock_storages('foo' => {})
end
it 'throws an error suggesting the user to update its settings' do
expect { validate_storages }.to raise_error('foo is not a valid storage, because it has no `path` key. Refer to gitlab.yml.example for an updated example. Please fix this in your gitlab.yml before starting GitLab.')
end
end
context 'with deprecated settings structure' do
before do
mock_storages('foo' => 'tmp/tests/paths/a/b/c')
end
it 'throws an error suggesting the user to update its settings' do
expect { validate_storages }.to raise_error("foo is not a valid storage, because it has no `path` key. It may be configured as:\n\nfoo:\n path: tmp/tests/paths/a/b/c\n\nRefer to gitlab.yml.example for an updated example. Please fix this in your gitlab.yml before starting GitLab.")
end
end
def mock_storages(storages)
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end

View File

@ -165,7 +165,7 @@ describe Namespace, models: true do
describe :rm_dir do
let!(:project) { create(:empty_project, namespace: namespace) }
let!(:path) { File.join(Gitlab.config.repositories.storages.default, namespace.full_path) }
let!(:path) { File.join(Gitlab.config.repositories.storages.default['path'], namespace.full_path) }
it "removes its dirs when deleted" do
namespace.destroy

View File

@ -179,7 +179,7 @@ describe Project, models: true do
let(:project2) { build(:empty_project, repository_storage: 'missing') }
before do
storages = { 'custom' => 'tmp/tests/custom_repositories' }
storages = { 'custom' => { 'path' => 'tmp/tests/custom_repositories' } }
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
@ -381,7 +381,7 @@ describe Project, models: true do
before do
FileUtils.mkdir('tmp/tests/custom_repositories')
storages = { 'custom' => 'tmp/tests/custom_repositories' }
storages = { 'custom' => { 'path' => 'tmp/tests/custom_repositories' } }
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end
@ -947,8 +947,8 @@ describe Project, models: true do
before do
storages = {
'default' => 'tmp/tests/repositories',
'picked' => 'tmp/tests/repositories',
'default' => { 'path' => 'tmp/tests/repositories' },
'picked' => { 'path' => 'tmp/tests/repositories' },
}
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
end

View File

@ -21,7 +21,7 @@ describe ::API::Helpers::InternalHelpers do
# Relative and absolute storage paths, with and without trailing /
['.', './', Dir.pwd, Dir.pwd + '/'].each do |storage_path|
context "storage path is #{storage_path}" do
subject { clean_project_path(project_path, [storage_path]) }
subject { clean_project_path(project_path, [{ 'path' => storage_path }]) }
it { is_expected.to eq(expected) }
end

View File

@ -143,7 +143,7 @@ module TestEnv
end
def repos_path
Gitlab.config.repositories.storages.default
Gitlab.config.repositories.storages.default['path']
end
def backup_path

View File

@ -227,8 +227,8 @@ describe 'gitlab:app namespace rake task' do
FileUtils.mkdir('tmp/tests/default_storage')
FileUtils.mkdir('tmp/tests/custom_storage')
storages = {
'default' => 'tmp/tests/default_storage',
'custom' => 'tmp/tests/custom_storage'
'default' => { 'path' => 'tmp/tests/default_storage' },
'custom' => { 'path' => 'tmp/tests/custom_storage' }
}
allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)

View File

@ -105,6 +105,6 @@ describe PostReceive do
end
def pwd(project)
File.join(Gitlab.config.repositories.storages.default, project.path_with_namespace)
File.join(Gitlab.config.repositories.storages.default['path'], project.path_with_namespace)
end
end