gitlab-org--gitlab-foss/lib/gitlab/shell.rb

495 lines
15 KiB
Ruby
Raw Normal View History

2017-07-13 11:58:45 -04:00
# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository.
# SSH key operations are not part of Gitaly so will never be migrated.
require 'securerandom'
module Gitlab
2013-02-04 08:07:56 -05:00
class Shell
GITLAB_SHELL_ENV_VARS = %w(GIT_TERMINAL_PROMPT).freeze
Error = Class.new(StandardError)
2015-12-14 21:53:52 -05:00
KeyAdder = Struct.new(:io) do
def add_key(id, key)
key = Gitlab::Shell.strip_key(key)
# Newline and tab are part of the 'protocol' used to transmit id+key to the other end
if key.include?("\t") || key.include?("\n")
raise Error.new("Invalid key: #{key.inspect}")
end
io.puts("#{id}\t#{key}")
end
end
2014-11-05 11:14:22 -05:00
class << self
def secret_token
@secret_token ||= begin
File.read(Gitlab.config.gitlab_shell.secret_file).chomp
end
end
def ensure_secret_token!
return if File.exist?(File.join(Gitlab.config.gitlab_shell.path, '.gitlab_shell_secret'))
generate_and_link_secret_token
end
2014-11-05 11:14:22 -05:00
def version_required
2017-06-21 09:48:12 -04:00
@version_required ||= File.read(Rails.root
.join('GITLAB_SHELL_VERSION')).strip
2014-11-05 11:14:22 -05:00
end
def strip_key(key)
key.split(/[ ]+/)[0, 2].join(' ')
end
private
# Create (if necessary) and link the secret token file
def generate_and_link_secret_token
secret_file = Gitlab.config.gitlab_shell.secret_file
shell_path = Gitlab.config.gitlab_shell.path
unless File.size?(secret_file)
# Generate a new token of 16 random hexadecimal characters and store it in secret_file.
@secret_token = SecureRandom.hex(16)
File.write(secret_file, @secret_token)
end
link_path = File.join(shell_path, '.gitlab_shell_secret')
if File.exist?(shell_path) && !File.exist?(link_path)
FileUtils.symlink(secret_file, link_path)
end
end
2014-11-05 11:14:22 -05:00
end
2013-02-04 08:07:56 -05:00
# Init new repository
2013-01-28 14:02:10 -05:00
#
2017-09-28 13:07:22 -04:00
# storage - project's storage name
# name - project disk path
#
# Ex.
# add_repository("/path/to/storage", "gitlab/gitlab-ci")
#
def add_repository(storage, name)
2017-09-28 13:07:22 -04:00
relative_path = name.dup
relative_path << '.git' unless relative_path.end_with?('.git')
gitaly_migrate(:create_repository) do |is_enabled|
if is_enabled
repository = Gitlab::Git::Repository.new(storage, relative_path, '')
repository.gitaly_repository_client.create_repository
true
else
repo_path = File.join(Gitlab.config.repositories.storages[storage]['path'], relative_path)
Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: gitlab_shell_hooks_path)
end
end
rescue => err
Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}")
false
end
2013-02-11 12:41:02 -05:00
# Import repository
#
# storage - project's storage path
# name - project disk path
# url - URL to import from
2013-02-11 12:41:02 -05:00
#
# Ex.
# import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git")
2013-02-11 12:41:02 -05:00
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874
def import_repository(storage, name, url)
2018-01-04 12:05:49 -05:00
if url.start_with?('.', '/')
2018-01-04 10:34:37 -05:00
raise Error.new("don't use disk paths with import_repository: #{url.inspect}")
end
# The timeout ensures the subprocess won't hang forever
cmd = gitlab_projects(storage, "#{name}.git")
success = cmd.import_project(url, git_timeout)
raise Error, cmd.output unless success
success
2013-02-11 12:41:02 -05:00
end
# Fetch remote for repository
#
# repository - an instance of Git::Repository
# remote - remote name
# ssh_auth - SSH known_hosts data and a private key to use for public-key authentication
# forced - should we use --force flag?
# no_tags - should we use --no-tags flag?
#
# Ex.
# fetch_remote(my_repo, "upstream")
#
def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false)
gitaly_migrate(:fetch_remote) do |is_enabled|
if is_enabled
repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout)
else
storage_path = Gitlab.config.repositories.storages[repository.storage]["path"]
local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
end
end
end
# Move repository reroutes to mv_directory which is an alias for
# mv_namespace. Given the underlying implementation is a move action,
# indescriminate of what the folders might be.
#
# storage - project's storage path
# path - project disk path
# new_path - new project disk path
#
# Ex.
# mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873
def mv_repository(storage, path, new_path)
return false if path.empty? || new_path.empty?
!!mv_directory(storage, "#{path}.git", "#{new_path}.git")
end
# Fork repository to new path
# forked_from_storage - forked-from project's storage path
# forked_from_disk_path - project disk path
# forked_to_storage - forked-to project's storage path
# forked_to_disk_path - forked project disk path
#
# Ex.
# fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817
def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path)
gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git")
.fork_repository(forked_to_storage, "#{forked_to_disk_path}.git")
end
# Removes a repository from file system, using rm_diretory which is an alias
# for rm_namespace. Given the underlying implementation removes the name
# passed as second argument on the passed storage.
#
# storage - project's storage path
# name - project disk path
#
# Ex.
# remove_repository("/path/to/storage", "gitlab/gitlab-ci")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873
def remove_repository(storage, name)
return false if name.empty?
!!rm_directory(storage, "#{name}.git")
rescue ArgumentError => e
Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git")
false
2012-03-05 17:26:40 -05:00
end
2013-02-04 08:07:56 -05:00
# Add new key to gitlab-shell
2013-02-04 07:28:10 -05:00
#
2013-02-04 08:07:56 -05:00
# Ex.
# add_key("key-42", "sha-rsa ...")
2013-02-04 08:07:56 -05:00
#
def add_key(key_id, key_content)
return unless self.authorized_keys_enabled?
gitlab_shell_fast_execute([gitlab_shell_keys_path,
'add-key', key_id, self.class.strip_key(key_content)])
2013-02-04 08:07:56 -05:00
end
# Batch-add keys to authorized_keys
#
# Ex.
# batch_add_keys { |adder| adder.add_key("key-42", "sha-rsa ...") }
def batch_add_keys(&block)
return unless self.authorized_keys_enabled?
IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys batch-add-keys), 'w') do |io|
2017-02-21 18:59:42 -05:00
yield(KeyAdder.new(io))
end
end
2013-02-04 08:07:56 -05:00
# Remove ssh key from gitlab shell
#
# Ex.
# remove_key("key-342", "sha-rsa ...")
#
Backport authorized_keys_enabled defaults to true' Originally from branch 'fix-authorized-keys-enabled-default-2738' via merge request https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/2240 Removed background migrations which were intended to fix state after using Gitlab without a default having been set Squashed commits: Locally, if Spring was not restarted, `current_application_settings` was still cached, which prevented the migration from editing the file. This will also ensure that any app server somehow hitting old cache data will properly default this setting regardless. Retroactively fix migration This allows us to identify customers who ran the broken migration. Their `authorized_keys_enabled` column does not have a default at this point. We will fix the column after we fix the `authorized_keys` file. Fix authorized_keys file if needed Add default to authorized_keys_enabled setting Reminder: The original migration was fixed retroactively a few commits ago, so people who did not ever run GitLab 9.3.0 already have a column that defaults to true and disallows nulls. I have tested on PostgreSQL and MySQL that it is safe to run this migration regardless. Affected customers who did run 9.3.0 are the ones who need this migration to fix the authorized_keys_enabled column. The reason for the retroactive fix plus this migration is that it allows us to run a migration in between to fix the authorized_keys file only for those who ran 9.3.0. Tweaks to address feedback Extract work into background migration Move batch-add-logic to background migration Do the work synchronously to avoid multiple workers attempting to add batches of keys at the same time. Also, make the delete portion wait until after adding is done. Do read and delete work in background migration Fix Rubocop offenses Add changelog entry Inform the user of actions taken or not taken Prevent unnecessary `select`s and `remove_key`s Add logs for action taken Fix optimization Reuse `Gitlab::ShellAdapter` Guarantee the earliest key Fix migration spec for MySQL
2017-06-26 17:40:08 -04:00
def remove_key(key_id, key_content = nil)
return unless self.authorized_keys_enabled?
args = [gitlab_shell_keys_path, 'rm-key', key_id]
args << key_content if key_content
gitlab_shell_fast_execute(args)
end
# Remove all ssh keys from gitlab shell
#
# Ex.
2013-07-29 06:46:00 -04:00
# remove_all_keys
#
def remove_all_keys
return unless self.authorized_keys_enabled?
gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear'])
end
Backport authorized_keys_enabled defaults to true' Originally from branch 'fix-authorized-keys-enabled-default-2738' via merge request https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/2240 Removed background migrations which were intended to fix state after using Gitlab without a default having been set Squashed commits: Locally, if Spring was not restarted, `current_application_settings` was still cached, which prevented the migration from editing the file. This will also ensure that any app server somehow hitting old cache data will properly default this setting regardless. Retroactively fix migration This allows us to identify customers who ran the broken migration. Their `authorized_keys_enabled` column does not have a default at this point. We will fix the column after we fix the `authorized_keys` file. Fix authorized_keys file if needed Add default to authorized_keys_enabled setting Reminder: The original migration was fixed retroactively a few commits ago, so people who did not ever run GitLab 9.3.0 already have a column that defaults to true and disallows nulls. I have tested on PostgreSQL and MySQL that it is safe to run this migration regardless. Affected customers who did run 9.3.0 are the ones who need this migration to fix the authorized_keys_enabled column. The reason for the retroactive fix plus this migration is that it allows us to run a migration in between to fix the authorized_keys file only for those who ran 9.3.0. Tweaks to address feedback Extract work into background migration Move batch-add-logic to background migration Do the work synchronously to avoid multiple workers attempting to add batches of keys at the same time. Also, make the delete portion wait until after adding is done. Do read and delete work in background migration Fix Rubocop offenses Add changelog entry Inform the user of actions taken or not taken Prevent unnecessary `select`s and `remove_key`s Add logs for action taken Fix optimization Reuse `Gitlab::ShellAdapter` Guarantee the earliest key Fix migration spec for MySQL
2017-06-26 17:40:08 -04:00
# Remove ssh keys from gitlab shell that are not in the DB
#
# Ex.
# remove_keys_not_found_in_db
#
def remove_keys_not_found_in_db
return unless self.authorized_keys_enabled?
Rails.logger.info("Removing keys not found in DB")
batch_read_key_ids do |ids_in_file|
ids_in_file.uniq!
keys_in_db = Key.where(id: ids_in_file)
next unless ids_in_file.size > keys_in_db.count # optimization
ids_to_remove = ids_in_file - keys_in_db.pluck(:id)
ids_to_remove.each do |id|
Rails.logger.info("Removing key-#{id} not found in DB")
remove_key("key-#{id}")
end
end
end
# Iterate over all ssh key IDs from gitlab shell, in batches
#
# Ex.
# batch_read_key_ids { |batch| keys = Key.where(id: batch) }
#
def batch_read_key_ids(batch_size: 100, &block)
return unless self.authorized_keys_enabled?
list_key_ids do |key_id_stream|
key_id_stream.lazy.each_slice(batch_size) do |lines|
key_ids = lines.map { |l| l.chomp.to_i }
yield(key_ids)
end
end
end
# Stream all ssh key IDs from gitlab shell, separated by newlines
#
# Ex.
# list_key_ids
#
def list_key_ids(&block)
return unless self.authorized_keys_enabled?
IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys list-key-ids), &block)
end
# Add empty directory for storing repositories
#
# Ex.
# add_namespace("/path/to/storage", "gitlab")
#
def add_namespace(storage, name)
Gitlab::GitalyClient.migrate(:add_namespace) do |enabled|
if enabled
gitaly_namespace_client(storage).add(name)
else
path = full_path(storage, name)
FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name)
end
end
rescue Errno::EEXIST => e
Rails.logger.warn("Directory exists as a file: #{e} at: #{path}")
rescue GRPC::InvalidArgument => e
raise ArgumentError, e.message
end
# Remove directory from repositories storage
# Every repository inside this directory will be removed too
#
# Ex.
# rm_namespace("/path/to/storage", "gitlab")
#
def rm_namespace(storage, name)
Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled|
if enabled
gitaly_namespace_client(storage).remove(name)
else
FileUtils.rm_r(full_path(storage, name), force: true)
end
end
rescue GRPC::InvalidArgument => e
raise ArgumentError, e.message
end
alias_method :rm_directory, :rm_namespace
# Move namespace directory inside repositories storage
#
# Ex.
# mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
#
def mv_namespace(storage, old_name, new_name)
Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled|
if enabled
gitaly_namespace_client(storage).rename(old_name, new_name)
else
return false if exists?(storage, new_name) || !exists?(storage, old_name)
FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name))
end
end
2017-10-05 09:01:26 -04:00
rescue GRPC::InvalidArgument
false
end
alias_method :mv_directory, :mv_namespace
def url_to_repo(path)
2013-02-11 12:16:59 -05:00
Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git"
end
# Return GitLab shell version
def version
gitlab_shell_version_file = "#{gitlab_shell_path}/VERSION"
if File.readable?(gitlab_shell_version_file)
File.read(gitlab_shell_version_file).chomp
end
end
# Check if such directory exists in repositories.
#
# Usage:
# exists?(storage, 'gitlab')
# exists?(storage, 'gitlab/cookies.git')
#
2017-07-13 11:58:45 -04:00
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
def exists?(storage, dir_name)
Gitlab::GitalyClient.migrate(:namespace_exists) do |enabled|
if enabled
gitaly_namespace_client(storage).exists?(dir_name)
else
File.exist?(full_path(storage, dir_name))
end
end
end
protected
def gitlab_shell_path
File.expand_path(Gitlab.config.gitlab_shell.path)
end
def gitlab_shell_hooks_path
File.expand_path(Gitlab.config.gitlab_shell.hooks_path)
end
def gitlab_shell_user_home
File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}")
end
def full_path(storage, dir_name)
raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?
File.join(storage, dir_name)
end
2014-10-31 08:00:50 -04:00
def gitlab_shell_projects_path
File.join(gitlab_shell_path, 'bin', 'gitlab-projects')
end
def gitlab_shell_keys_path
File.join(gitlab_shell_path, 'bin', 'gitlab-keys')
end
def authorized_keys_enabled?
# Return true if nil to ensure the authorized_keys methods work while
# fixing the authorized_keys file during migration.
return true if Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled.nil?
Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled
end
private
def gitlab_projects(shard_path, disk_path)
Gitlab::Git::GitlabProjects.new(
shard_path,
disk_path,
global_hooks_path: Gitlab.config.gitlab_shell.hooks_path,
logger: Rails.logger
)
end
def local_fetch_remote(storage_path, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false)
vars = { force: forced, tags: !no_tags }
if ssh_auth&.ssh_import?
if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
vars[:ssh_key] = ssh_auth.ssh_private_key
end
if ssh_auth.ssh_known_hosts.present?
vars[:known_hosts] = ssh_auth.ssh_known_hosts
end
end
cmd = gitlab_projects(storage_path, repository_relative_path)
success = cmd.fetch_remote(remote, git_timeout, vars)
raise Error, cmd.output unless success
success
end
def gitlab_shell_fast_execute(cmd)
output, status = gitlab_shell_fast_execute_helper(cmd)
return true if status.zero?
Rails.logger.error("gitlab-shell failed with error #{status}: #{output}")
false
end
def gitlab_shell_fast_execute_raise_error(cmd, vars = {})
output, status = gitlab_shell_fast_execute_helper(cmd, vars)
raise Error, output unless status.zero?
true
end
def gitlab_shell_fast_execute_helper(cmd, vars = {})
vars.merge!(ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS))
# Don't pass along the entire parent environment to prevent gitlab-shell
# from wasting I/O by searching through GEM_PATH
Bundler.with_original_env { Popen.popen(cmd, nil, vars) }
end
def gitaly_namespace_client(storage_path)
storage, _value = Gitlab.config.repositories.storages.find do |storage, value|
value['path'] == storage_path
end
Gitlab::GitalyClient::NamespaceService.new(storage)
end
def git_timeout
Gitlab.config.gitlab_shell.git_timeout
end
def gitaly_migrate(method, &block)
Gitlab::GitalyClient.migrate(method, &block)
rescue GRPC::NotFound, GRPC::BadStatus => e
# Old Popen code returns [Error, output] to the caller, so we
# need to do the same here...
raise Error, e
end
2011-12-03 18:44:59 -05:00
end
end