From 324ff19571cada7e148c53bb70e70f823eff4335 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Wed, 24 Oct 2018 16:03:00 +0100 Subject: [PATCH] Backport SSH host key detection code to CE This functionality is needed for SSH push mirroring support, which is a CE feature. --- .../projects/mirrors_controller.rb | 16 ++ app/models/ssh_host_key.rb | 130 ++++++++++++++ config/routes/project.rb | 1 + .../projects/mirrors_controller_spec.rb | 63 +++++++ spec/models/ssh_host_key_spec.rb | 164 ++++++++++++++++++ 5 files changed, 374 insertions(+) create mode 100644 app/models/ssh_host_key.rb create mode 100644 spec/models/ssh_host_key_spec.rb diff --git a/app/controllers/projects/mirrors_controller.rb b/app/controllers/projects/mirrors_controller.rb index 78d5faf2326..53176978416 100644 --- a/app/controllers/projects/mirrors_controller.rb +++ b/app/controllers/projects/mirrors_controller.rb @@ -44,6 +44,22 @@ class Projects::MirrorsController < Projects::ApplicationController redirect_to_repository_settings(project, anchor: 'js-push-remote-settings') end + def ssh_host_keys + lookup = SshHostKey.new(project: project, url: params[:ssh_url], compare_host_keys: params[:compare_host_keys]) + + if lookup.error.present? + # Failed to read keys + render json: { message: lookup.error }, status: :bad_request + elsif lookup.known_hosts.nil? + # Still working, come back later + render body: nil, status: :no_content + else + render json: lookup + end + rescue ArgumentError => err + render json: { message: err.message }, status: :bad_request + end + private def remote_mirror diff --git a/app/models/ssh_host_key.rb b/app/models/ssh_host_key.rb new file mode 100644 index 00000000000..b6844dbe870 --- /dev/null +++ b/app/models/ssh_host_key.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +# Detected SSH host keys are transiently stored in Redis +class SshHostKey + class Fingerprint < Gitlab::SSHPublicKey + attr_reader :index + + def initialize(key, index: nil) + super(key) + + @index = index + end + + def as_json(*) + { bits: bits, fingerprint: fingerprint, type: type, index: index } + end + end + + include ReactiveCaching + + self.reactive_cache_key = ->(key) { [key.class.to_s, key.id] } + + # Do not refresh the data in the background - it is not expected to change. + # This is achieved by making the lifetime shorter than the refresh interval. + self.reactive_cache_refresh_interval = 15.minutes + self.reactive_cache_lifetime = 10.minutes + + def self.find_by(opts = {}) + return nil unless opts.key?(:id) + + project_id, url = opts[:id].split(':', 2) + project = Project.find_by(id: project_id) + + project.presence && new(project: project, url: url) + end + + def self.fingerprint_host_keys(data) + return [] unless data.is_a?(String) + + data + .each_line + .each_with_index + .map { |line, index| Fingerprint.new(line, index: index) } + .select(&:valid?) + end + + attr_reader :project, :url, :compare_host_keys + + def initialize(project:, url:, compare_host_keys: nil) + @project = project + @url = normalize_url(url) + @compare_host_keys = compare_host_keys + end + + def id + [project.id, url].join(':') + end + + def as_json(*) + { + host_keys_changed: host_keys_changed?, + fingerprints: fingerprints, + known_hosts: known_hosts + } + end + + def known_hosts + with_reactive_cache { |data| data[:known_hosts] } + end + + def fingerprints + @fingerprints ||= self.class.fingerprint_host_keys(known_hosts) + end + + # Returns true if the known_hosts data differs from the version passed in at + # initialization as `compare_host_keys`. Comments, ordering, etc, is ignored + def host_keys_changed? + cleanup(known_hosts) != cleanup(compare_host_keys) + end + + def error + with_reactive_cache { |data| data[:error] } + end + + def calculate_reactive_cache + known_hosts, errors, status = + Open3.popen3({}, *%W[ssh-keyscan -T 5 -p #{url.port} -f-]) do |stdin, stdout, stderr, wait_thr| + stdin.puts(url.host) + stdin.close + + [ + cleanup(stdout.read), + cleanup(stderr.read), + wait_thr.value + ] + end + + # ssh-keyscan returns an exit code 0 in several error conditions, such as an + # unknown hostname, so check both STDERR and the exit code + if status.success? && !errors.present? + { known_hosts: known_hosts } + else + Rails.logger.debug("Failed to detect SSH host keys for #{id}: #{errors}") + + { error: 'Failed to detect SSH host keys' } + end + end + + private + + # Remove comments and duplicate entries + def cleanup(data) + data + .to_s + .each_line + .reject { |line| line.start_with?('#') || line.chomp.empty? } + .uniq + .sort + .join + end + + def normalize_url(url) + full_url = ::Addressable::URI.parse(url) + raise ArgumentError.new("Invalid URL") unless full_url&.scheme == 'ssh' + + Addressable::URI.parse("ssh://#{full_url.host}:#{full_url.inferred_port}") + rescue Addressable::URI::InvalidURIError + raise ArgumentError.new("Invalid URL") + end +end diff --git a/config/routes/project.rb b/config/routes/project.rb index 9cbd5b644f6..85872a4122a 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -178,6 +178,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do resource :mirror, only: [:show, :update] do member do + get :ssh_host_keys, constraints: { format: :json } post :update_now end end diff --git a/spec/controllers/projects/mirrors_controller_spec.rb b/spec/controllers/projects/mirrors_controller_spec.rb index 6114eef7003..00c1e617e3a 100644 --- a/spec/controllers/projects/mirrors_controller_spec.rb +++ b/spec/controllers/projects/mirrors_controller_spec.rb @@ -63,6 +63,69 @@ describe Projects::MirrorsController do end end + describe '#ssh_host_keys', :use_clean_rails_memory_store_caching do + let(:project) { create(:project) } + let(:cache) { SshHostKey.new(project: project, url: "ssh://example.com:22") } + + before do + sign_in(project.owner) + end + + context 'invalid URLs' do + %w[ + INVALID + git@example.com:foo/bar.git + ssh://git@example.com:foo/bar.git + ].each do |url| + it "returns an error with a 400 response for URL #{url.inspect}" do + do_get(project, url) + + expect(response).to have_gitlab_http_status(400) + expect(json_response).to eq('message' => 'Invalid URL') + end + end + end + + context 'no data in cache' do + it 'requests the cache to be filled and returns a 204 response' do + expect(ReactiveCachingWorker).to receive(:perform_async).with(cache.class, cache.id).at_least(:once) + + do_get(project) + + expect(response).to have_gitlab_http_status(204) + end + end + + context 'error in the cache' do + it 'returns the error with a 400 response' do + stub_reactive_cache(cache, error: 'An error') + + do_get(project) + + expect(response).to have_gitlab_http_status(400) + expect(json_response).to eq('message' => 'An error') + end + end + + context 'data in the cache' do + let(:ssh_key) { 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf' } + let(:ssh_fp) { { type: 'ed25519', bits: 256, fingerprint: '2e:65:6a:c8:cf:bf:b2:8b:9a:bd:6d:9f:11:5c:12:16', index: 0 } } + + it 'returns the data with a 200 response' do + stub_reactive_cache(cache, known_hosts: ssh_key) + + do_get(project) + + expect(response).to have_gitlab_http_status(200) + expect(json_response).to eq('known_hosts' => ssh_key, 'fingerprints' => [ssh_fp.stringify_keys], 'host_keys_changed' => true) + end + end + + def do_get(project, url = 'ssh://example.com') + get :ssh_host_keys, namespace_id: project.namespace, project_id: project, ssh_url: url + end + end + def do_put(project, options, extra_attrs = {}) attrs = extra_attrs.merge(namespace_id: project.namespace.to_param, project_id: project.to_param) attrs[:project] = options diff --git a/spec/models/ssh_host_key_spec.rb b/spec/models/ssh_host_key_spec.rb new file mode 100644 index 00000000000..75db43b3d56 --- /dev/null +++ b/spec/models/ssh_host_key_spec.rb @@ -0,0 +1,164 @@ +require 'spec_helper' + +describe SshHostKey do + using RSpec::Parameterized::TableSyntax + include ReactiveCachingHelpers + + let(:key1) do + 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC3UpyF2iLqy1d63M6k3jH1vuEnq/NWtE+o' \ + 'rJe1Xn7JoRbduKd6zpsJ0JhBGWgcQK0ph0aGW5PcudzzBSc+SlYfCc4GTaxDtmj41hW0o72m' \ + 'NiuDW3oKXXShOiVRde2ZOquH8Z865jGiZIC8BI/bXZD29IGUih0hPu7Rjp70VYiE+35QRf/p' \ + 'sD0Ddrz8QUIG3A/2dMzLI5F5ZORk3BIX2F3mJwJOvZxRhR/SqyphDMZ5eZ0EzqbFBCDE6HAB' \ + 'Woz9ck8RBGLvCIggmDHj3FmMLcQGMDiy6wKp7QdnBtxjCP6vtE6YPUM223AqsWt+9NTtCfB8' \ + 'YdNAH7YcHHOR1FgtSk1x' + end + + let(:key2) do + 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLIp+4ciR2YO9f9rpldc7InNQw/TBUtcNb' \ + 'J2XR0rr15/5ytz7YM16xXG0Qjx576PNSmqs4gbTrvTuFZak+v1Jx/9deHRq/yqp9f+tv33+i' \ + 'aJGCQCX/+OVY7aWgV2R9YsS7XQ4mnv4XlOTEssib/rGAIT+ATd/GcdYSEOO+dh4O09/6O/jI' \ + 'MGSeP+NNetgn1nPCnLOjrXFZUnUtNDi6EEKeIlrliJjSb7Jr4f7gjvZnv4RskWHHFo8FgAAq' \ + 't0gOMT6EmKrnypBe2vLGSAXbtkXr01q6/DNPH+n9VA1LTV6v1KN/W5CN5tQV11wRSKiM8g5O' \ + 'Ebi86VjJRi2sOuYoXQU1' + end + + # Purposefully ordered so that `sort` will make changes + let(:known_hosts) do + <<~EOF + example.com #{key1} git@localhost + @revoked other.example.com #{key2} git@localhost + EOF + end + + let(:extra) { known_hosts + "foo\nbar\n" } + let(:reversed) { known_hosts.lines.reverse.join } + + let(:compare_host_keys) { nil } + + def stub_ssh_keyscan(args, status: true, stdout: "", stderr: "") + stdin = StringIO.new + stdout = double(:stdout, read: stdout) + stderr = double(:stderr, read: stderr) + wait_thr = double(:wait_thr, value: double(success?: status)) + + expect(Open3).to receive(:popen3).with({}, 'ssh-keyscan', *args).and_yield(stdin, stdout, stderr, wait_thr) + + stdin + end + + let(:project) { build(:project) } + + subject(:ssh_host_key) { described_class.new(project: project, url: 'ssh://example.com:2222', compare_host_keys: compare_host_keys) } + + describe '#fingerprints', :use_clean_rails_memory_store_caching do + it 'returns an array of indexed fingerprints when the cache is filled' do + stub_reactive_cache(ssh_host_key, known_hosts: known_hosts) + + expected = [key1, key2] + .map { |data| Gitlab::SSHPublicKey.new(data) } + .each_with_index + .map { |key, i| { bits: key.bits, fingerprint: key.fingerprint, type: key.type, index: i } } + + expect(ssh_host_key.fingerprints.as_json).to eq(expected) + end + + it 'returns an empty array when the cache is empty' do + expect(ssh_host_key.fingerprints).to eq([]) + end + end + + describe '#fingerprints', :use_clean_rails_memory_store_caching do + it 'returns an array of indexed fingerprints when the cache is filled' do + stub_reactive_cache(ssh_host_key, known_hosts: known_hosts) + + expect(ssh_host_key.fingerprints.as_json).to eq( + [ + { bits: 2048, fingerprint: Gitlab::SSHPublicKey.new(key1).fingerprint, type: :rsa, index: 0 }, + { bits: 2048, fingerprint: Gitlab::SSHPublicKey.new(key2).fingerprint, type: :rsa, index: 1 } + ] + ) + end + + it 'returns an empty array when the cache is empty' do + expect(ssh_host_key.fingerprints).to eq([]) + end + end + + describe '#host_keys_changed?' do + where(:known_hosts_a, :known_hosts_b, :result) do + known_hosts | extra | true + known_hosts | "foo\n" | true + known_hosts | '' | true + known_hosts | nil | true + known_hosts | known_hosts | false + reversed | known_hosts | false + extra | "foo\n" | true + '' | '' | false + nil | nil | false + '' | nil | false + end + + with_them do + let(:compare_host_keys) { known_hosts_b } + + subject { ssh_host_key.host_keys_changed? } + + context '(normal)' do + let(:compare_host_keys) { known_hosts_b } + + before do + expect(ssh_host_key).to receive(:known_hosts).and_return(known_hosts_a) + end + + it { is_expected.to eq(result) } + end + + # Comparisons should be symmetrical, so test the reverse too + context '(reversed)' do + let(:compare_host_keys) { known_hosts_a } + + before do + expect(ssh_host_key).to receive(:known_hosts).and_return(known_hosts_b) + end + + it { is_expected.to eq(result) } + end + end + end + + describe '#calculate_reactive_cache' do + subject(:cache) { ssh_host_key.calculate_reactive_cache } + + it 'writes the hostname to STDIN' do + stdin = stub_ssh_keyscan(%w[-T 5 -p 2222 -f-]) + + cache + + expect(stdin.string).to eq("example.com\n") + end + + context 'successful key scan' do + it 'stores the cleaned known_hosts data' do + stub_ssh_keyscan(%w[-T 5 -p 2222 -f-], stdout: "KEY 1\nKEY 1\n\n# comment\nKEY 2\n") + + is_expected.to eq(known_hosts: "KEY 1\nKEY 2\n") + end + end + + context 'failed key scan (exit code 1)' do + it 'returns a generic error' do + stub_ssh_keyscan(%w[-T 5 -p 2222 -f-], stdout: 'blarg', status: false) + + is_expected.to eq(error: 'Failed to detect SSH host keys') + end + end + + context 'failed key scan (exit code 0)' do + it 'returns a generic error' do + stub_ssh_keyscan(%w[-T 5 -p 2222 -f-], stderr: 'Unknown host') + + is_expected.to eq(error: 'Failed to detect SSH host keys') + end + end + end +end