2016-02-01 05:33:22 -05:00
|
|
|
require 'base64'
|
|
|
|
require 'json'
|
2016-08-19 13:10:41 -04:00
|
|
|
require 'securerandom'
|
2017-03-24 13:22:42 -04:00
|
|
|
require 'uri'
|
2016-02-01 05:33:22 -05:00
|
|
|
|
|
|
|
module Gitlab
|
|
|
|
class Workhorse
|
2017-02-21 18:32:18 -05:00
|
|
|
SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data'.freeze
|
|
|
|
VERSION_FILE = 'GITLAB_WORKHORSE_VERSION'.freeze
|
|
|
|
INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json'.freeze
|
|
|
|
INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request'.freeze
|
2017-02-28 06:07:04 -05:00
|
|
|
NOTIFICATION_CHANNEL = 'workhorse:notifications'.freeze
|
2016-08-19 13:10:41 -04:00
|
|
|
|
|
|
|
# Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32
|
|
|
|
# bytes https://tools.ietf.org/html/rfc4868#section-2.6
|
|
|
|
SECRET_LENGTH = 32
|
2016-02-02 08:09:55 -05:00
|
|
|
|
2016-02-11 12:10:14 -05:00
|
|
|
class << self
|
2017-10-11 20:31:59 -04:00
|
|
|
def git_http_ok(repository, is_wiki, user, action, show_all_refs: false)
|
2017-05-03 17:07:54 -04:00
|
|
|
project = repository.project
|
2017-03-30 12:48:22 -04:00
|
|
|
repo_path = repository.path_to_repo
|
2016-12-26 12:15:40 -05:00
|
|
|
params = {
|
2016-08-19 13:10:41 -04:00
|
|
|
GL_ID: Gitlab::GlId.gl_id(user),
|
2017-05-03 17:07:54 -04:00
|
|
|
GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki),
|
2017-08-03 14:38:33 -04:00
|
|
|
GL_USERNAME: user&.username,
|
2017-10-11 20:31:59 -04:00
|
|
|
RepoPath: repo_path,
|
|
|
|
ShowAllRefs: show_all_refs
|
2016-04-06 11:52:12 -04:00
|
|
|
}
|
2017-07-06 08:45:29 -04:00
|
|
|
server = {
|
|
|
|
address: Gitlab::GitalyClient.address(project.repository_storage),
|
|
|
|
token: Gitlab::GitalyClient.token(project.repository_storage)
|
|
|
|
}
|
|
|
|
params[:Repository] = repository.gitaly_repository.to_h
|
2017-03-30 11:24:36 -04:00
|
|
|
|
2017-07-06 08:45:29 -04:00
|
|
|
feature_enabled = case action.to_s
|
|
|
|
when 'git_receive_pack'
|
2018-01-22 11:51:47 -05:00
|
|
|
Gitlab::GitalyClient.feature_enabled?(
|
|
|
|
:post_receive_pack,
|
|
|
|
status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT
|
|
|
|
)
|
2017-07-06 08:45:29 -04:00
|
|
|
when 'git_upload_pack'
|
2017-09-01 04:45:19 -04:00
|
|
|
true
|
2017-07-06 08:45:29 -04:00
|
|
|
when 'info_refs'
|
|
|
|
true
|
|
|
|
else
|
|
|
|
raise "Unsupported action: #{action}"
|
|
|
|
end
|
2018-01-11 11:34:01 -05:00
|
|
|
|
2017-07-06 08:45:29 -04:00
|
|
|
if feature_enabled
|
|
|
|
params[:GitalyServer] = server
|
2017-03-24 13:22:42 -04:00
|
|
|
end
|
2016-12-26 12:15:40 -05:00
|
|
|
|
|
|
|
params
|
2016-04-06 11:52:12 -04:00
|
|
|
end
|
|
|
|
|
2016-08-19 13:10:41 -04:00
|
|
|
def lfs_upload_ok(oid, size)
|
|
|
|
{
|
2018-01-29 12:57:34 -05:00
|
|
|
StoreLFSPath: LfsObjectUploader.workhorse_upload_path,
|
2016-08-19 13:10:41 -04:00
|
|
|
LfsOid: oid,
|
2017-05-03 07:22:03 -04:00
|
|
|
LfsSize: size
|
2016-08-19 13:10:41 -04:00
|
|
|
}
|
|
|
|
end
|
|
|
|
|
|
|
|
def artifact_upload_ok
|
2018-01-29 12:57:34 -05:00
|
|
|
{ TempPath: JobArtifactUploader.workhorse_upload_path }
|
2016-08-19 13:10:41 -04:00
|
|
|
end
|
|
|
|
|
2016-02-01 05:33:22 -05:00
|
|
|
def send_git_blob(repository, blob)
|
2017-07-24 01:07:20 -04:00
|
|
|
params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show)
|
2017-07-09 23:43:20 -04:00
|
|
|
{
|
|
|
|
'GitalyServer' => gitaly_server_hash(repository),
|
|
|
|
'GetBlobRequest' => {
|
|
|
|
repository: repository.gitaly_repository.to_h,
|
|
|
|
oid: blob.id,
|
|
|
|
limit: -1
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
'RepoPath' => repository.path_to_repo,
|
|
|
|
'BlobId' => blob.id
|
|
|
|
}
|
|
|
|
end
|
2016-02-01 05:33:22 -05:00
|
|
|
|
|
|
|
[
|
2016-02-02 08:09:55 -05:00
|
|
|
SEND_DATA_HEADER,
|
2016-06-08 08:30:15 -04:00
|
|
|
"git-blob:#{encode(params)}"
|
2016-02-01 05:33:22 -05:00
|
|
|
]
|
|
|
|
end
|
2016-02-02 08:09:55 -05:00
|
|
|
|
2016-06-06 07:16:30 -04:00
|
|
|
def send_git_archive(repository, ref:, format:)
|
2016-02-02 08:09:55 -05:00
|
|
|
format ||= 'tar.gz'
|
|
|
|
format.downcase!
|
2016-06-06 07:16:30 -04:00
|
|
|
params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format)
|
2016-02-02 08:09:55 -05:00
|
|
|
raise "Repository or ref not found" if params.empty?
|
|
|
|
|
2017-09-30 20:13:23 -04:00
|
|
|
if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive)
|
|
|
|
params.merge!(
|
|
|
|
'GitalyServer' => gitaly_server_hash(repository),
|
|
|
|
'GitalyRepository' => repository.gitaly_repository.to_h
|
|
|
|
)
|
|
|
|
end
|
|
|
|
|
2018-01-09 10:59:46 -05:00
|
|
|
# If present DisableCache must be a Boolean. Otherwise workhorse ignores it.
|
|
|
|
params['DisableCache'] = true if git_archive_cache_disabled?
|
|
|
|
|
2016-02-02 08:09:55 -05:00
|
|
|
[
|
|
|
|
SEND_DATA_HEADER,
|
2016-06-08 08:30:15 -04:00
|
|
|
"git-archive:#{encode(params)}"
|
2016-02-02 08:09:55 -05:00
|
|
|
]
|
|
|
|
end
|
2016-05-12 14:50:49 -04:00
|
|
|
|
2016-06-08 08:30:15 -04:00
|
|
|
def send_git_diff(repository, diff_refs)
|
2017-10-02 14:51:20 -04:00
|
|
|
params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff)
|
|
|
|
{
|
|
|
|
'GitalyServer' => gitaly_server_hash(repository),
|
|
|
|
'RawDiffRequest' => Gitaly::RawDiffRequest.new(
|
|
|
|
gitaly_diff_or_patch_hash(repository, diff_refs)
|
|
|
|
).to_json
|
|
|
|
}
|
|
|
|
else
|
|
|
|
workhorse_diff_or_patch_hash(repository, diff_refs)
|
|
|
|
end
|
2016-05-12 14:50:49 -04:00
|
|
|
|
|
|
|
[
|
|
|
|
SEND_DATA_HEADER,
|
|
|
|
"git-diff:#{encode(params)}"
|
2016-02-02 08:09:55 -05:00
|
|
|
]
|
|
|
|
end
|
2016-06-06 07:16:30 -04:00
|
|
|
|
2016-07-03 17:01:13 -04:00
|
|
|
def send_git_patch(repository, diff_refs)
|
2017-10-02 14:51:20 -04:00
|
|
|
params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch)
|
|
|
|
{
|
|
|
|
'GitalyServer' => gitaly_server_hash(repository),
|
|
|
|
'RawPatchRequest' => Gitaly::RawPatchRequest.new(
|
|
|
|
gitaly_diff_or_patch_hash(repository, diff_refs)
|
|
|
|
).to_json
|
|
|
|
}
|
|
|
|
else
|
|
|
|
workhorse_diff_or_patch_hash(repository, diff_refs)
|
|
|
|
end
|
2016-06-10 08:57:50 -04:00
|
|
|
|
|
|
|
[
|
2016-06-28 08:59:25 -04:00
|
|
|
SEND_DATA_HEADER,
|
2016-06-10 08:57:50 -04:00
|
|
|
"git-format-patch:#{encode(params)}"
|
|
|
|
]
|
|
|
|
end
|
|
|
|
|
2017-09-08 08:04:44 -04:00
|
|
|
def send_artifacts_entry(build, entry)
|
2018-01-29 12:57:34 -05:00
|
|
|
file = build.artifacts_file
|
|
|
|
archive = file.file_storage? ? file.path : file.url
|
|
|
|
|
2016-07-05 10:58:38 -04:00
|
|
|
params = {
|
2018-01-29 12:57:34 -05:00
|
|
|
'Archive' => archive,
|
2017-09-08 08:04:44 -04:00
|
|
|
'Entry' => Base64.encode64(entry.to_s)
|
2016-07-05 10:58:38 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
[
|
|
|
|
SEND_DATA_HEADER,
|
|
|
|
"artifacts-entry:#{encode(params)}"
|
|
|
|
]
|
|
|
|
end
|
|
|
|
|
2018-02-06 09:31:39 -05:00
|
|
|
def send_url(url, allow_redirects: false)
|
|
|
|
params = {
|
|
|
|
'URL' => url,
|
|
|
|
'AllowRedirects' => allow_redirects
|
|
|
|
}
|
|
|
|
|
|
|
|
[
|
|
|
|
SEND_DATA_HEADER,
|
|
|
|
"send-url:#{encode(params)}"
|
|
|
|
]
|
|
|
|
end
|
|
|
|
|
2016-11-22 14:55:56 -05:00
|
|
|
def terminal_websocket(terminal)
|
|
|
|
details = {
|
|
|
|
'Terminal' => {
|
|
|
|
'Subprotocols' => terminal[:subprotocols],
|
|
|
|
'Url' => terminal[:url],
|
2017-01-26 13:16:50 -05:00
|
|
|
'Header' => terminal[:headers],
|
2017-05-03 07:22:03 -04:00
|
|
|
'MaxSessionTime' => terminal[:max_session_time]
|
2016-11-22 14:55:56 -05:00
|
|
|
}
|
|
|
|
}
|
2017-06-02 13:11:26 -04:00
|
|
|
details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.key?(:ca_pem)
|
2016-11-22 14:55:56 -05:00
|
|
|
|
|
|
|
details
|
|
|
|
end
|
|
|
|
|
2016-07-18 07:58:08 -04:00
|
|
|
def version
|
2016-07-21 16:04:28 -04:00
|
|
|
path = Rails.root.join(VERSION_FILE)
|
|
|
|
path.readable? ? path.read.chomp : 'unknown'
|
2016-07-18 07:58:08 -04:00
|
|
|
end
|
|
|
|
|
2016-08-19 13:10:41 -04:00
|
|
|
def secret
|
|
|
|
@secret ||= begin
|
2016-09-13 13:45:02 -04:00
|
|
|
bytes = Base64.strict_decode64(File.read(secret_path).chomp)
|
2016-08-19 13:10:41 -04:00
|
|
|
raise "#{secret_path} does not contain #{SECRET_LENGTH} bytes" if bytes.length != SECRET_LENGTH
|
2017-11-14 04:02:39 -05:00
|
|
|
|
2016-08-19 13:10:41 -04:00
|
|
|
bytes
|
|
|
|
end
|
|
|
|
end
|
2016-09-20 12:21:52 -04:00
|
|
|
|
2016-08-19 13:10:41 -04:00
|
|
|
def write_secret
|
|
|
|
bytes = SecureRandom.random_bytes(SECRET_LENGTH)
|
2016-09-20 12:21:52 -04:00
|
|
|
File.open(secret_path, 'w:BINARY', 0600) do |f|
|
2016-09-26 08:21:39 -04:00
|
|
|
f.chmod(0600) # If the file already existed, the '0600' passed to 'open' above was a no-op.
|
2016-08-19 13:10:41 -04:00
|
|
|
f.write(Base64.strict_encode64(bytes))
|
|
|
|
end
|
|
|
|
end
|
2016-09-20 12:21:52 -04:00
|
|
|
|
2016-08-19 13:10:41 -04:00
|
|
|
def verify_api_request!(request_headers)
|
2016-08-18 10:31:44 -04:00
|
|
|
decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER])
|
|
|
|
end
|
|
|
|
|
|
|
|
def decode_jwt(encoded_message)
|
2016-08-19 13:10:41 -04:00
|
|
|
JWT.decode(
|
2016-08-18 10:31:44 -04:00
|
|
|
encoded_message,
|
2016-08-19 13:10:41 -04:00
|
|
|
secret,
|
|
|
|
true,
|
2017-05-03 07:27:17 -04:00
|
|
|
{ iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' }
|
2016-08-19 13:10:41 -04:00
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
def secret_path
|
2017-03-30 20:37:45 -04:00
|
|
|
Gitlab.config.workhorse.secret_file
|
2016-08-19 13:10:41 -04:00
|
|
|
end
|
2016-09-20 12:21:52 -04:00
|
|
|
|
2017-03-06 05:44:45 -05:00
|
|
|
def set_key_and_notify(key, value, expire: nil, overwrite: true)
|
2017-07-10 23:35:47 -04:00
|
|
|
Gitlab::Redis::Queues.with do |redis|
|
2017-02-28 06:07:04 -05:00
|
|
|
result = redis.set(key, value, ex: expire, nx: !overwrite)
|
|
|
|
if result
|
2017-03-06 05:44:45 -05:00
|
|
|
redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}")
|
2017-02-28 06:07:04 -05:00
|
|
|
value
|
|
|
|
else
|
|
|
|
redis.get(key)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2016-02-02 08:09:55 -05:00
|
|
|
protected
|
2016-06-06 07:16:30 -04:00
|
|
|
|
2016-02-02 08:09:55 -05:00
|
|
|
def encode(hash)
|
|
|
|
Base64.urlsafe_encode64(JSON.dump(hash))
|
|
|
|
end
|
2017-07-09 23:43:20 -04:00
|
|
|
|
|
|
|
def gitaly_server_hash(repository)
|
|
|
|
{
|
|
|
|
address: Gitlab::GitalyClient.address(repository.project.repository_storage),
|
|
|
|
token: Gitlab::GitalyClient.token(repository.project.repository_storage)
|
|
|
|
}
|
|
|
|
end
|
2017-10-02 14:51:20 -04:00
|
|
|
|
|
|
|
def workhorse_diff_or_patch_hash(repository, diff_refs)
|
|
|
|
{
|
|
|
|
'RepoPath' => repository.path_to_repo,
|
|
|
|
'ShaFrom' => diff_refs.base_sha,
|
|
|
|
'ShaTo' => diff_refs.head_sha
|
|
|
|
}
|
|
|
|
end
|
|
|
|
|
|
|
|
def gitaly_diff_or_patch_hash(repository, diff_refs)
|
|
|
|
{
|
|
|
|
repository: repository.gitaly_repository,
|
|
|
|
left_commit_id: diff_refs.base_sha,
|
|
|
|
right_commit_id: diff_refs.head_sha
|
|
|
|
}
|
|
|
|
end
|
2018-01-09 10:59:46 -05:00
|
|
|
|
|
|
|
def git_archive_cache_disabled?
|
|
|
|
ENV['WORKHORSE_ARCHIVE_CACHE_DISABLED'].present? || Feature.enabled?(:workhorse_archive_cache_disabled)
|
|
|
|
end
|
2016-02-01 05:33:22 -05:00
|
|
|
end
|
|
|
|
end
|
2016-02-01 06:27:35 -05:00
|
|
|
end
|