diff --git a/qa/qa/git/repository.rb b/qa/qa/git/repository.rb index 86e00cdbb9c..ac8dcbf0d83 100644 --- a/qa/qa/git/repository.rb +++ b/qa/qa/git/repository.rb @@ -11,14 +11,15 @@ module QA class Repository include Scenario::Actable - attr_writer :password + attr_writer :password, :use_lfs attr_accessor :env_vars def initialize # We set HOME to the current working directory (which is a # temporary directory created in .perform()) so the temporarily dropped # .netrc can be utilised - self.env_vars = [%Q{HOME="#{File.dirname(netrc_file_path)}"}] + self.env_vars = [%Q{HOME="#{tmp_home_dir}"}] + @use_lfs = false end def self.perform(*args) @@ -33,17 +34,22 @@ module QA def username=(username) @username = username - @uri.user = username + # Only include the user in the URI if we're using HTTP as this breaks + # SSH authentication. + @uri.user = username unless ssh_key_set? end def use_default_credentials self.username, self.password = default_credentials - - add_credentials_to_netrc unless ssh_key_set? end def clone(opts = '') - run("git clone #{opts} #{uri} ./") + clone_result = run("git clone #{opts} #{uri} ./") + return clone_result.response unless clone_result.success + + enable_lfs_result = enable_lfs if use_lfs? + + clone_result.to_s + enable_lfs_result.to_s end def checkout(branch_name, new_branch: false) @@ -58,8 +64,6 @@ module QA def configure_identity(name, email) run(%Q{git config user.name #{name}}) run(%Q{git config user.email #{email}}) - - add_credentials_to_netrc end def commit_file(name, contents, message) @@ -70,15 +74,22 @@ module QA def add_file(name, contents) ::File.write(name, contents) - run(%Q{git add #{name}}) + if use_lfs? + git_lfs_track_result = run(%Q{git lfs track #{name} --lockable}) + return git_lfs_track_result.response unless git_lfs_track_result.success + end + + git_add_result = run(%Q{git add #{name}}) + + git_lfs_track_result.to_s + git_add_result.to_s end def commit(message) - run(%Q{git commit -m "#{message}"}) + run(%Q{git commit -m "#{message}"}).to_s end def push_changes(branch = 'master') - run("git push #{uri} #{branch}") + run("git push #{uri} #{branch}").to_s end def merge(branch) @@ -86,7 +97,7 @@ module QA end def commits - run('git log --oneline').split("\n") + run('git log --oneline').to_s.split("\n") end def use_ssh_key(key) @@ -98,7 +109,8 @@ module QA keyscan_params = ['-H'] keyscan_params << "-p #{uri.port}" if uri.port keyscan_params << uri.host - run("ssh-keyscan #{keyscan_params.join(' ')} >> #{known_hosts_file.path}") + res = run("ssh-keyscan #{keyscan_params.join(' ')} >> #{known_hosts_file.path}") + return res.response unless res.success? self.env_vars << %Q{GIT_SSH_COMMAND="ssh -i #{private_key_file.path} -o UserKnownHostsFile=#{known_hosts_file.path}"} end @@ -132,23 +144,66 @@ module QA output[/git< version (\d+)/, 1] || 'unknown' end + def try_add_credentials_to_netrc + return unless add_credentials? + return if netrc_already_contains_content? + + # Despite libcurl supporting a custom .netrc location through the + # CURLOPT_NETRC_FILE environment variable, git does not support it :( + # Info: https://curl.haxx.se/libcurl/c/CURLOPT_NETRC_FILE.html + # + # This will create a .netrc in the correct working directory, which is + # a temporary directory created in .perform() + # + FileUtils.mkdir_p(tmp_home_dir) + File.open(netrc_file_path, 'a') { |file| file.puts(netrc_content) } + File.chmod(0600, netrc_file_path) + end + private - attr_reader :uri, :username, :password, :known_hosts_file, :private_key_file + attr_reader :uri, :username, :password, :known_hosts_file, + :private_key_file, :use_lfs + + alias_method :use_lfs?, :use_lfs + + Result = Struct.new(:success, :response) do + alias_method :success?, :success + alias_method :to_s, :response + end + + def add_credentials? + return false if !username || !password + return true unless ssh_key_set? + return true if ssh_key_set? && use_lfs? + + false + end def ssh_key_set? !private_key_file.nil? end + def enable_lfs + # git lfs install *needs* a .gitconfig defined at ${HOME}/.gitconfig + FileUtils.mkdir_p(tmp_home_dir) + touch_gitconfig_result = run("touch #{tmp_home_dir}/.gitconfig") + return touch_gitconfig_result.response unless touch_gitconfig_result.success? + + git_lfs_install_result = run('git lfs install') + + touch_gitconfig_result.to_s + git_lfs_install_result.to_s + end + def run(command_str, *extra_env) command = [env_vars, *extra_env, command_str, '2>&1'].compact.join(' ') - Runtime::Logger.debug "Git: command=[#{command}]" + Runtime::Logger.debug "Git: pwd=[#{Dir.pwd}], command=[#{command}]" - output, _ = Open3.capture2(command) - output = output.chomp.gsub(/\s+$/, '') - Runtime::Logger.debug "Git: output=[#{output}]" + output, status = Open3.capture2e(command) + output.chomp! + Runtime::Logger.debug "Git: output=[#{output}], exitstatus=[#{status.exitstatus}]" - output + Result.new(status.exitstatus == 0, output) end def default_credentials @@ -159,12 +214,12 @@ module QA end end - def tmp_netrc_directory - @tmp_netrc_directory ||= File.join(Dir.tmpdir, "qa-netrc-credentials", $$.to_s) + def tmp_home_dir + @tmp_home_dir ||= File.join(Dir.tmpdir, "qa-netrc-credentials", $$.to_s) end def netrc_file_path - @netrc_file_path ||= File.join(tmp_netrc_directory, '.netrc') + @netrc_file_path ||= File.join(tmp_home_dir, '.netrc') end def netrc_content @@ -175,21 +230,6 @@ module QA File.exist?(netrc_file_path) && File.readlines(netrc_file_path).grep(/^#{netrc_content}$/).any? end - - def add_credentials_to_netrc - # Despite libcurl supporting a custom .netrc location through the - # CURLOPT_NETRC_FILE environment variable, git does not support it :( - # Info: https://curl.haxx.se/libcurl/c/CURLOPT_NETRC_FILE.html - # - # This will create a .netrc in the correct working directory, which is - # a temporary directory created in .perform() - # - return if netrc_already_contains_content? - - FileUtils.mkdir_p(tmp_netrc_directory) - File.open(netrc_file_path, 'a') { |file| file.puts(netrc_content) } - File.chmod(0600, netrc_file_path) - end end end end diff --git a/qa/qa/resource/repository/push.rb b/qa/qa/resource/repository/push.rb index f33aa0acef0..32f15547da2 100644 --- a/qa/qa/resource/repository/push.rb +++ b/qa/qa/resource/repository/push.rb @@ -8,7 +8,7 @@ module QA class Push < Base attr_accessor :file_name, :file_content, :commit_message, :branch_name, :new_branch, :output, :repository_http_uri, - :repository_ssh_uri, :ssh_key, :user + :repository_ssh_uri, :ssh_key, :user, :use_lfs attr_writer :remote_branch @@ -20,6 +20,7 @@ module QA @new_branch = true @repository_http_uri = "" @ssh_key = nil + @use_lfs = false end def remote_branch @@ -33,7 +34,9 @@ module QA end def files=(files) - if !files.is_a?(Array) || files.empty? + if !files.is_a?(Array) || + files.empty? || + files.any? { |file| !file.has_key?(:name) || !file.has_key?(:content) } raise ArgumentError, "Please provide an array of hashes e.g.: [{name: 'file1', content: 'foo'}]" end @@ -42,6 +45,8 @@ module QA def fabricate! Git::Repository.perform do |repository| + @output = '' + if ssh_key repository.uri = repository_ssh_uri repository.use_ssh_key(ssh_key) @@ -50,6 +55,8 @@ module QA repository.use_default_credentials unless user end + repository.use_lfs = use_lfs + username = 'GitLab QA' email = 'root@gitlab.com' @@ -60,29 +67,27 @@ module QA email = user.email end - repository.clone + repository.try_add_credentials_to_netrc + + @output += repository.clone repository.configure_identity(username, email) - if new_branch - repository.checkout(branch_name, new_branch: true) - else - repository.checkout(branch_name) - end + @output += repository.checkout(branch_name, new_branch: new_branch) if @directory @directory.each_child do |f| - repository.add_file(f.basename, f.read) if f.file? + @output += repository.add_file(f.basename, f.read) if f.file? end elsif @files @files.each do |f| repository.add_file(f[:name], f[:content]) end else - repository.add_file(file_name, file_content) + @output += repository.add_file(file_name, file_content) end - repository.commit(commit_message) - @output = repository.push_changes("#{branch_name}:#{remote_branch}") + @output += repository.commit(commit_message) + @output += repository.push_changes("#{branch_name}:#{remote_branch}") repository.delete_ssh_key end