Merge branch '49565-ssh-push-mirroring' into 'master'
Backport SSH host key detection code to CE See merge request gitlab-org/gitlab-ce!22577
This commit is contained in:
commit
982276c3f7
|
@ -44,6 +44,22 @@ class Projects::MirrorsController < Projects::ApplicationController
|
||||||
redirect_to_repository_settings(project, anchor: 'js-push-remote-settings')
|
redirect_to_repository_settings(project, anchor: 'js-push-remote-settings')
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def ssh_host_keys
|
||||||
|
lookup = SshHostKey.new(project: project, url: params[:ssh_url], compare_host_keys: params[:compare_host_keys])
|
||||||
|
|
||||||
|
if lookup.error.present?
|
||||||
|
# Failed to read keys
|
||||||
|
render json: { message: lookup.error }, status: :bad_request
|
||||||
|
elsif lookup.known_hosts.nil?
|
||||||
|
# Still working, come back later
|
||||||
|
render body: nil, status: :no_content
|
||||||
|
else
|
||||||
|
render json: lookup
|
||||||
|
end
|
||||||
|
rescue ArgumentError => err
|
||||||
|
render json: { message: err.message }, status: :bad_request
|
||||||
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def remote_mirror
|
def remote_mirror
|
||||||
|
|
|
@ -0,0 +1,130 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
# Detected SSH host keys are transiently stored in Redis
|
||||||
|
class SshHostKey
|
||||||
|
class Fingerprint < Gitlab::SSHPublicKey
|
||||||
|
attr_reader :index
|
||||||
|
|
||||||
|
def initialize(key, index: nil)
|
||||||
|
super(key)
|
||||||
|
|
||||||
|
@index = index
|
||||||
|
end
|
||||||
|
|
||||||
|
def as_json(*)
|
||||||
|
{ bits: bits, fingerprint: fingerprint, type: type, index: index }
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
include ReactiveCaching
|
||||||
|
|
||||||
|
self.reactive_cache_key = ->(key) { [key.class.to_s, key.id] }
|
||||||
|
|
||||||
|
# Do not refresh the data in the background - it is not expected to change.
|
||||||
|
# This is achieved by making the lifetime shorter than the refresh interval.
|
||||||
|
self.reactive_cache_refresh_interval = 15.minutes
|
||||||
|
self.reactive_cache_lifetime = 10.minutes
|
||||||
|
|
||||||
|
def self.find_by(opts = {})
|
||||||
|
return nil unless opts.key?(:id)
|
||||||
|
|
||||||
|
project_id, url = opts[:id].split(':', 2)
|
||||||
|
project = Project.find_by(id: project_id)
|
||||||
|
|
||||||
|
project.presence && new(project: project, url: url)
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.fingerprint_host_keys(data)
|
||||||
|
return [] unless data.is_a?(String)
|
||||||
|
|
||||||
|
data
|
||||||
|
.each_line
|
||||||
|
.each_with_index
|
||||||
|
.map { |line, index| Fingerprint.new(line, index: index) }
|
||||||
|
.select(&:valid?)
|
||||||
|
end
|
||||||
|
|
||||||
|
attr_reader :project, :url, :compare_host_keys
|
||||||
|
|
||||||
|
def initialize(project:, url:, compare_host_keys: nil)
|
||||||
|
@project = project
|
||||||
|
@url = normalize_url(url)
|
||||||
|
@compare_host_keys = compare_host_keys
|
||||||
|
end
|
||||||
|
|
||||||
|
def id
|
||||||
|
[project.id, url].join(':')
|
||||||
|
end
|
||||||
|
|
||||||
|
def as_json(*)
|
||||||
|
{
|
||||||
|
host_keys_changed: host_keys_changed?,
|
||||||
|
fingerprints: fingerprints,
|
||||||
|
known_hosts: known_hosts
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def known_hosts
|
||||||
|
with_reactive_cache { |data| data[:known_hosts] }
|
||||||
|
end
|
||||||
|
|
||||||
|
def fingerprints
|
||||||
|
@fingerprints ||= self.class.fingerprint_host_keys(known_hosts)
|
||||||
|
end
|
||||||
|
|
||||||
|
# Returns true if the known_hosts data differs from the version passed in at
|
||||||
|
# initialization as `compare_host_keys`. Comments, ordering, etc, is ignored
|
||||||
|
def host_keys_changed?
|
||||||
|
cleanup(known_hosts) != cleanup(compare_host_keys)
|
||||||
|
end
|
||||||
|
|
||||||
|
def error
|
||||||
|
with_reactive_cache { |data| data[:error] }
|
||||||
|
end
|
||||||
|
|
||||||
|
def calculate_reactive_cache
|
||||||
|
known_hosts, errors, status =
|
||||||
|
Open3.popen3({}, *%W[ssh-keyscan -T 5 -p #{url.port} -f-]) do |stdin, stdout, stderr, wait_thr|
|
||||||
|
stdin.puts(url.host)
|
||||||
|
stdin.close
|
||||||
|
|
||||||
|
[
|
||||||
|
cleanup(stdout.read),
|
||||||
|
cleanup(stderr.read),
|
||||||
|
wait_thr.value
|
||||||
|
]
|
||||||
|
end
|
||||||
|
|
||||||
|
# ssh-keyscan returns an exit code 0 in several error conditions, such as an
|
||||||
|
# unknown hostname, so check both STDERR and the exit code
|
||||||
|
if status.success? && !errors.present?
|
||||||
|
{ known_hosts: known_hosts }
|
||||||
|
else
|
||||||
|
Rails.logger.debug("Failed to detect SSH host keys for #{id}: #{errors}")
|
||||||
|
|
||||||
|
{ error: 'Failed to detect SSH host keys' }
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
# Remove comments and duplicate entries
|
||||||
|
def cleanup(data)
|
||||||
|
data
|
||||||
|
.to_s
|
||||||
|
.each_line
|
||||||
|
.reject { |line| line.start_with?('#') || line.chomp.empty? }
|
||||||
|
.uniq
|
||||||
|
.sort
|
||||||
|
.join
|
||||||
|
end
|
||||||
|
|
||||||
|
def normalize_url(url)
|
||||||
|
full_url = ::Addressable::URI.parse(url)
|
||||||
|
raise ArgumentError.new("Invalid URL") unless full_url&.scheme == 'ssh'
|
||||||
|
|
||||||
|
Addressable::URI.parse("ssh://#{full_url.host}:#{full_url.inferred_port}")
|
||||||
|
rescue Addressable::URI::InvalidURIError
|
||||||
|
raise ArgumentError.new("Invalid URL")
|
||||||
|
end
|
||||||
|
end
|
|
@ -178,6 +178,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
|
||||||
|
|
||||||
resource :mirror, only: [:show, :update] do
|
resource :mirror, only: [:show, :update] do
|
||||||
member do
|
member do
|
||||||
|
get :ssh_host_keys, constraints: { format: :json }
|
||||||
post :update_now
|
post :update_now
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -63,6 +63,69 @@ describe Projects::MirrorsController do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe '#ssh_host_keys', :use_clean_rails_memory_store_caching do
|
||||||
|
let(:project) { create(:project) }
|
||||||
|
let(:cache) { SshHostKey.new(project: project, url: "ssh://example.com:22") }
|
||||||
|
|
||||||
|
before do
|
||||||
|
sign_in(project.owner)
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'invalid URLs' do
|
||||||
|
%w[
|
||||||
|
INVALID
|
||||||
|
git@example.com:foo/bar.git
|
||||||
|
ssh://git@example.com:foo/bar.git
|
||||||
|
].each do |url|
|
||||||
|
it "returns an error with a 400 response for URL #{url.inspect}" do
|
||||||
|
do_get(project, url)
|
||||||
|
|
||||||
|
expect(response).to have_gitlab_http_status(400)
|
||||||
|
expect(json_response).to eq('message' => 'Invalid URL')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'no data in cache' do
|
||||||
|
it 'requests the cache to be filled and returns a 204 response' do
|
||||||
|
expect(ReactiveCachingWorker).to receive(:perform_async).with(cache.class, cache.id).at_least(:once)
|
||||||
|
|
||||||
|
do_get(project)
|
||||||
|
|
||||||
|
expect(response).to have_gitlab_http_status(204)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'error in the cache' do
|
||||||
|
it 'returns the error with a 400 response' do
|
||||||
|
stub_reactive_cache(cache, error: 'An error')
|
||||||
|
|
||||||
|
do_get(project)
|
||||||
|
|
||||||
|
expect(response).to have_gitlab_http_status(400)
|
||||||
|
expect(json_response).to eq('message' => 'An error')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'data in the cache' do
|
||||||
|
let(:ssh_key) { 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf' }
|
||||||
|
let(:ssh_fp) { { type: 'ed25519', bits: 256, fingerprint: '2e:65:6a:c8:cf:bf:b2:8b:9a:bd:6d:9f:11:5c:12:16', index: 0 } }
|
||||||
|
|
||||||
|
it 'returns the data with a 200 response' do
|
||||||
|
stub_reactive_cache(cache, known_hosts: ssh_key)
|
||||||
|
|
||||||
|
do_get(project)
|
||||||
|
|
||||||
|
expect(response).to have_gitlab_http_status(200)
|
||||||
|
expect(json_response).to eq('known_hosts' => ssh_key, 'fingerprints' => [ssh_fp.stringify_keys], 'host_keys_changed' => true)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def do_get(project, url = 'ssh://example.com')
|
||||||
|
get :ssh_host_keys, namespace_id: project.namespace, project_id: project, ssh_url: url
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
def do_put(project, options, extra_attrs = {})
|
def do_put(project, options, extra_attrs = {})
|
||||||
attrs = extra_attrs.merge(namespace_id: project.namespace.to_param, project_id: project.to_param)
|
attrs = extra_attrs.merge(namespace_id: project.namespace.to_param, project_id: project.to_param)
|
||||||
attrs[:project] = options
|
attrs[:project] = options
|
||||||
|
|
|
@ -0,0 +1,164 @@
|
||||||
|
require 'spec_helper'
|
||||||
|
|
||||||
|
describe SshHostKey do
|
||||||
|
using RSpec::Parameterized::TableSyntax
|
||||||
|
include ReactiveCachingHelpers
|
||||||
|
|
||||||
|
let(:key1) do
|
||||||
|
'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC3UpyF2iLqy1d63M6k3jH1vuEnq/NWtE+o' \
|
||||||
|
'rJe1Xn7JoRbduKd6zpsJ0JhBGWgcQK0ph0aGW5PcudzzBSc+SlYfCc4GTaxDtmj41hW0o72m' \
|
||||||
|
'NiuDW3oKXXShOiVRde2ZOquH8Z865jGiZIC8BI/bXZD29IGUih0hPu7Rjp70VYiE+35QRf/p' \
|
||||||
|
'sD0Ddrz8QUIG3A/2dMzLI5F5ZORk3BIX2F3mJwJOvZxRhR/SqyphDMZ5eZ0EzqbFBCDE6HAB' \
|
||||||
|
'Woz9ck8RBGLvCIggmDHj3FmMLcQGMDiy6wKp7QdnBtxjCP6vtE6YPUM223AqsWt+9NTtCfB8' \
|
||||||
|
'YdNAH7YcHHOR1FgtSk1x'
|
||||||
|
end
|
||||||
|
|
||||||
|
let(:key2) do
|
||||||
|
'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLIp+4ciR2YO9f9rpldc7InNQw/TBUtcNb' \
|
||||||
|
'J2XR0rr15/5ytz7YM16xXG0Qjx576PNSmqs4gbTrvTuFZak+v1Jx/9deHRq/yqp9f+tv33+i' \
|
||||||
|
'aJGCQCX/+OVY7aWgV2R9YsS7XQ4mnv4XlOTEssib/rGAIT+ATd/GcdYSEOO+dh4O09/6O/jI' \
|
||||||
|
'MGSeP+NNetgn1nPCnLOjrXFZUnUtNDi6EEKeIlrliJjSb7Jr4f7gjvZnv4RskWHHFo8FgAAq' \
|
||||||
|
't0gOMT6EmKrnypBe2vLGSAXbtkXr01q6/DNPH+n9VA1LTV6v1KN/W5CN5tQV11wRSKiM8g5O' \
|
||||||
|
'Ebi86VjJRi2sOuYoXQU1'
|
||||||
|
end
|
||||||
|
|
||||||
|
# Purposefully ordered so that `sort` will make changes
|
||||||
|
let(:known_hosts) do
|
||||||
|
<<~EOF
|
||||||
|
example.com #{key1} git@localhost
|
||||||
|
@revoked other.example.com #{key2} git@localhost
|
||||||
|
EOF
|
||||||
|
end
|
||||||
|
|
||||||
|
let(:extra) { known_hosts + "foo\nbar\n" }
|
||||||
|
let(:reversed) { known_hosts.lines.reverse.join }
|
||||||
|
|
||||||
|
let(:compare_host_keys) { nil }
|
||||||
|
|
||||||
|
def stub_ssh_keyscan(args, status: true, stdout: "", stderr: "")
|
||||||
|
stdin = StringIO.new
|
||||||
|
stdout = double(:stdout, read: stdout)
|
||||||
|
stderr = double(:stderr, read: stderr)
|
||||||
|
wait_thr = double(:wait_thr, value: double(success?: status))
|
||||||
|
|
||||||
|
expect(Open3).to receive(:popen3).with({}, 'ssh-keyscan', *args).and_yield(stdin, stdout, stderr, wait_thr)
|
||||||
|
|
||||||
|
stdin
|
||||||
|
end
|
||||||
|
|
||||||
|
let(:project) { build(:project) }
|
||||||
|
|
||||||
|
subject(:ssh_host_key) { described_class.new(project: project, url: 'ssh://example.com:2222', compare_host_keys: compare_host_keys) }
|
||||||
|
|
||||||
|
describe '#fingerprints', :use_clean_rails_memory_store_caching do
|
||||||
|
it 'returns an array of indexed fingerprints when the cache is filled' do
|
||||||
|
stub_reactive_cache(ssh_host_key, known_hosts: known_hosts)
|
||||||
|
|
||||||
|
expected = [key1, key2]
|
||||||
|
.map { |data| Gitlab::SSHPublicKey.new(data) }
|
||||||
|
.each_with_index
|
||||||
|
.map { |key, i| { bits: key.bits, fingerprint: key.fingerprint, type: key.type, index: i } }
|
||||||
|
|
||||||
|
expect(ssh_host_key.fingerprints.as_json).to eq(expected)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'returns an empty array when the cache is empty' do
|
||||||
|
expect(ssh_host_key.fingerprints).to eq([])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe '#fingerprints', :use_clean_rails_memory_store_caching do
|
||||||
|
it 'returns an array of indexed fingerprints when the cache is filled' do
|
||||||
|
stub_reactive_cache(ssh_host_key, known_hosts: known_hosts)
|
||||||
|
|
||||||
|
expect(ssh_host_key.fingerprints.as_json).to eq(
|
||||||
|
[
|
||||||
|
{ bits: 2048, fingerprint: Gitlab::SSHPublicKey.new(key1).fingerprint, type: :rsa, index: 0 },
|
||||||
|
{ bits: 2048, fingerprint: Gitlab::SSHPublicKey.new(key2).fingerprint, type: :rsa, index: 1 }
|
||||||
|
]
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'returns an empty array when the cache is empty' do
|
||||||
|
expect(ssh_host_key.fingerprints).to eq([])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe '#host_keys_changed?' do
|
||||||
|
where(:known_hosts_a, :known_hosts_b, :result) do
|
||||||
|
known_hosts | extra | true
|
||||||
|
known_hosts | "foo\n" | true
|
||||||
|
known_hosts | '' | true
|
||||||
|
known_hosts | nil | true
|
||||||
|
known_hosts | known_hosts | false
|
||||||
|
reversed | known_hosts | false
|
||||||
|
extra | "foo\n" | true
|
||||||
|
'' | '' | false
|
||||||
|
nil | nil | false
|
||||||
|
'' | nil | false
|
||||||
|
end
|
||||||
|
|
||||||
|
with_them do
|
||||||
|
let(:compare_host_keys) { known_hosts_b }
|
||||||
|
|
||||||
|
subject { ssh_host_key.host_keys_changed? }
|
||||||
|
|
||||||
|
context '(normal)' do
|
||||||
|
let(:compare_host_keys) { known_hosts_b }
|
||||||
|
|
||||||
|
before do
|
||||||
|
expect(ssh_host_key).to receive(:known_hosts).and_return(known_hosts_a)
|
||||||
|
end
|
||||||
|
|
||||||
|
it { is_expected.to eq(result) }
|
||||||
|
end
|
||||||
|
|
||||||
|
# Comparisons should be symmetrical, so test the reverse too
|
||||||
|
context '(reversed)' do
|
||||||
|
let(:compare_host_keys) { known_hosts_a }
|
||||||
|
|
||||||
|
before do
|
||||||
|
expect(ssh_host_key).to receive(:known_hosts).and_return(known_hosts_b)
|
||||||
|
end
|
||||||
|
|
||||||
|
it { is_expected.to eq(result) }
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe '#calculate_reactive_cache' do
|
||||||
|
subject(:cache) { ssh_host_key.calculate_reactive_cache }
|
||||||
|
|
||||||
|
it 'writes the hostname to STDIN' do
|
||||||
|
stdin = stub_ssh_keyscan(%w[-T 5 -p 2222 -f-])
|
||||||
|
|
||||||
|
cache
|
||||||
|
|
||||||
|
expect(stdin.string).to eq("example.com\n")
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'successful key scan' do
|
||||||
|
it 'stores the cleaned known_hosts data' do
|
||||||
|
stub_ssh_keyscan(%w[-T 5 -p 2222 -f-], stdout: "KEY 1\nKEY 1\n\n# comment\nKEY 2\n")
|
||||||
|
|
||||||
|
is_expected.to eq(known_hosts: "KEY 1\nKEY 2\n")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'failed key scan (exit code 1)' do
|
||||||
|
it 'returns a generic error' do
|
||||||
|
stub_ssh_keyscan(%w[-T 5 -p 2222 -f-], stdout: 'blarg', status: false)
|
||||||
|
|
||||||
|
is_expected.to eq(error: 'Failed to detect SSH host keys')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'failed key scan (exit code 0)' do
|
||||||
|
it 'returns a generic error' do
|
||||||
|
stub_ssh_keyscan(%w[-T 5 -p 2222 -f-], stderr: 'Unknown host')
|
||||||
|
|
||||||
|
is_expected.to eq(error: 'Failed to detect SSH host keys')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
Loading…
Reference in New Issue