kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Remove satellites

Browse source
This commit is contained in:
Dmitriy Zaporozhets 2015-07-15 15:45:57 +02:00
parent 277f311f20
commit a87989fb7d
35 changed files with 18 additions and 844 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
Gemfile.lock
View file

@ -875,6 +875,3 @@ DEPENDENCIES
virtus
webmock (~> 1.21.0)
wikicloth (= 0.8.1)
BUNDLED WITH
1.10.5

2
app/controllers/projects/merge_requests_controller.rb
View file

@ -1,5 +1,3 @@
require 'gitlab/satellite/satellite'
class Projects::MergeRequestsController < Projects::ApplicationController
before_action :module_enabled
before_action :merge_request, only: [

6
app/models/merge_request.rb
View file

@ -207,7 +207,7 @@ class MergeRequest < ActiveRecord::Base
def check_if_can_be_merged
can_be_merged =
if for_fork?
Gitlab::Satellite::MergeAction.new(self.author, self).can_be_merged?
raise 'Implement me'
else
project.repository.can_be_merged?(source_branch, target_branch)
end
@ -274,14 +274,14 @@ class MergeRequest < ActiveRecord::Base
#
# see "git diff"
def to_diff(current_user)
Gitlab::Satellite::MergeAction.new(current_user, self).diff_in_satellite
raise 'Implement me'
end
# Returns the commit as a series of email patches.
#
# see "git format-patch"
def to_patch(current_user)
Gitlab::Satellite::MergeAction.new(current_user, self).format_patch
raise 'Implement me'
end
def hook_attrs

5
app/models/namespace.rb
View file

@ -115,12 +115,11 @@ class Namespace < ActiveRecord::Base
def move_dir
if gitlab_shell.mv_namespace(path_was, path)
# If repositories moved successfully we need to remove old satellites
# and send update instructions to users.
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
# So we basically we mute exceptions in next actions
begin
gitlab_shell.rm_satellites(path_was)
send_update_instructions
rescue
# Returning false does not rollback after_* transaction but gives

14
app/models/project.rb
View file

@ -520,14 +520,6 @@ class Project < ActiveRecord::Base
!repository.exists? || repository.empty?
end
def ensure_satellite_exists
self.satellite.create unless self.satellite.exists?
end
def satellite
@satellite ||= Gitlab::Satellite::Satellite.new(self)
end
def repo
repository.raw
end
@ -597,14 +589,11 @@ class Project < ActiveRecord::Base
new_path_with_namespace = File.join(namespace_dir, path)
if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to remove old satellite
# and send update instructions to users.
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
gitlab_shell.rm_satellites(old_path_with_namespace)
ensure_satellite_exists
send_move_instructions
reset_events_cache
rescue
@ -698,7 +687,6 @@ class Project < ActiveRecord::Base
def create_repository
if forked?
if gitlab_shell.fork_repository(forked_from_project.path_with_namespace, self.namespace.path)
ensure_satellite_exists
true
else
errors.add(:base, 'Failed to fork repository')

11
app/services/compare_service.rb
View file

@ -3,9 +3,6 @@
class CompareService
def execute(current_user, source_project, source_branch, target_project, target_branch)
# Try to compare branches to get commits list and diffs
#
# Note: Use satellite only when need to compare between two repos
# because satellites are slower than operations on bare repo
if target_project == source_project
Gitlab::CompareResult.new(
Gitlab::Git::Compare.new(
@ -15,13 +12,7 @@ class CompareService
)
)
else
Gitlab::Satellite::CompareAction.new(
current_user,
target_project,
target_branch,
source_project,
source_branch
).result
raise 'Implement me'
end
end
end

10
app/services/git_push_service.rb
View file

@ -10,16 +10,14 @@ class GitPushService
#
# Next, this method:
# 1. Creates the push event
# 2. Ensures that the project satellite exists
# 3. Updates merge requests
# 4. Recognizes cross-references from commit messages
# 5. Executes the project's web hooks
# 6. Executes the project's services
# 2. Updates merge requests
# 3. Recognizes cross-references from commit messages
# 4. Executes the project's web hooks
# 5. Executes the project's services
#
def execute(project, user, oldrev, newrev, ref)
@project, @user = project, user
project.ensure_satellite_exists
project.repository.expire_cache
project.update_repository_size

5
app/services/merge_requests/auto_merge_service.rb
View file

@ -1,7 +1,7 @@
module MergeRequests
# AutoMergeService class
#
# Do git merge in satellite and in case of success
# Do git merge and in case of success
# mark merge request as merged and execute all hooks and notifications
# Called when you do merge via GitLab UI
class AutoMergeService < BaseMergeService
@ -32,9 +32,8 @@ module MergeRequests
def merge!
if merge_request.for_fork?
Gitlab::Satellite::MergeAction.new(current_user, merge_request).merge!(commit_message)
raise 'Implement me'
else
# Merge local branches using rugged instead of satellites
if sha = commit
after_commit(sha, merge_request.target_branch)
end

4
app/services/merge_requests/build_service.rb
View file

@ -40,7 +40,6 @@ module MergeRequests
merge_request.compare_diffs = diffs
elsif diffs == false
# satellite timeout return false
merge_request.can_be_created = false
merge_request.compare_failed = true
end
@ -59,9 +58,6 @@ module MergeRequests
end
merge_request
rescue Gitlab::Satellite::BranchesWithoutParent
return build_failed(merge_request, "Selected branches have no common commit so they cannot be merged.")
end
def build_failed(merge_request, message)

1
app/services/projects/destroy_service.rb
View file

@ -27,7 +27,6 @@ module Projects
end
end
project.satellite.destroy
log_info("Project \"#{project.name}\" was removed")
system_hook_service.execute_hooks_for(project, :destroy)
true

6
app/services/projects/transfer_service.rb
View file

@ -33,9 +33,6 @@ module Projects
raise TransferError.new("Project with same path in target namespace already exists")
end
# Remove old satellite
project.satellite.destroy
# Apply new namespace id
project.namespace = new_namespace
project.save!
@ -51,9 +48,6 @@ module Projects
# Move wiki repo also if present
gitlab_shell.mv_repository("#{old_path}.wiki", "#{new_path}.wiki")
# Create a new satellite (reload project from DB)
Project.find(project.id).ensure_satellite_exists
# clear project cached events
project.reset_events_cache

2
app/views/projects/merge_requests/_new_compare.html.haml
View file

@ -35,7 +35,7 @@
- if @merge_request.compare_failed
.alert.alert-danger
%h4 Compare failed
%p We can't compare selected branches. It may be because of huge diff or satellite timeout. Please try again or select different branches.
%p We can't compare selected branches. It may be because of huge diff. Please try again or select different branches.
- else
.light-well
.center

2
app/views/projects/merge_requests/widget/_open.html.haml
View file

@ -3,8 +3,6 @@
.mr-widget-body
- if @project.archived?
= render 'projects/merge_requests/widget/open/archived'
- elsif !@project.satellite.exists?
= render 'projects/merge_requests/widget/open/no_satellite'
- elsif @merge_request.commits.blank?
= render 'projects/merge_requests/widget/open/nothing'
- elsif @merge_request.branch_missing?

3
app/views/projects/merge_requests/widget/open/_no_satellite.html.haml
View file

@ -1,3 +0,0 @@
%p
%span
%strong This repository does not have a satellite. Please ask an administrator to fix this issue!

1
app/workers/repository_import_worker.rb
View file

@ -27,7 +27,6 @@ class RepositoryImportWorker
project.import_finish
project.save
project.satellite.create unless project.satellite.exists?
project.update_repository_size
Gitlab::BitbucketImport::KeyDeleter.new(project).execute if project.import_type == 'bitbucket'
end

2
doc/development/architecture.md
View file

@ -58,7 +58,7 @@ A typical install of GitLab will be on GNU/Linux. It uses Nginx or Apache as a w
The GitLab web app uses MySQL or PostgreSQL for persistent database information (e.g. users, permissions, issues, other meta data). GitLab stores the bare git repositories it serves in `/home/git/repositories` by default. It also keeps default branch and hook information with the bare repository. `/home/git/gitlab-satellites` keeps checked out repositories when performing actions such as a merge request, editing files in the web interface, etc.
The satellite repository is used by the web interface for editing repositories and the wiki which is also a git repository. When serving repositories over HTTP/HTTPS GitLab utilizes the GitLab API to resolve authorization and access as well as serving git objects.
[DEPRECATED] The satellite repository is used by the web interface for editing repositories and the wiki which is also a git repository. When serving repositories over HTTP/HTTPS GitLab utilizes the GitLab API to resolve authorization and access as well as serving git objects.
The add-on component gitlab-shell serves repositories over SSH. It manages the SSH keys within `/home/git/.ssh/authorized_keys` which should not be manually edited. gitlab-shell accesses the bare repositories directly to serve git objects and communicates with redis to submit jobs to Sidekiq for GitLab to process. gitlab-shell queries the GitLab API to determine authorization and access.

4
doc/install/installation.md
View file

@ -216,10 +216,6 @@ We recommend using a PostgreSQL database. For MySQL check [MySQL setup guide](da
sudo chmod -R u+rwX,go-w log/
sudo chmod -R u+rwX tmp/
# Create directory for satellites
sudo -u git -H mkdir /home/git/gitlab-satellites
sudo chmod u+rwx,g=rx,o-rwx /home/git/gitlab-satellites
# Make sure GitLab can write to the tmp/pids/ and tmp/sockets/ directories
sudo chmod -R u+rwX tmp/pids/
sudo chmod -R u+rwX tmp/sockets/

4
doc/install/structure.md
View file

@ -6,16 +6,14 @@ This is the directory structure you will end up with following the instructions
| |-- git
| |-- .ssh
| |-- gitlab
| |-- gitlab-satellites
| |-- gitlab-shell
| |-- repositories
* `/home/git/.ssh` - contains openssh settings. Specifically the `authorized_keys` file managed by gitlab-shell.
* `/home/git/gitlab` - GitLab core software.
* `/home/git/gitlab-satellites` - checked out repositories for merge requests and file editing from web UI. This can be treated as a temporary files directory.
* `/home/git/gitlab-shell` - Core add-on component of GitLab. Maintains SSH cloning and other functionality.
* `/home/git/repositories` - bare repositories for all projects organized by namespace. This is where the git repositories which are pushed/pulled are maintained for all projects. **This area is critical data for projects. [Keep a backup](../raketasks/backup_restore.md)**
*Note: the default locations for gitlab-satellites and repositories can be configured in `config/gitlab.yml` of GitLab and `config.yml` of gitlab-shell.*
*Note: the default locations for repositories can be configured in `config/gitlab.yml` of GitLab and `config.yml` of gitlab-shell.*
To see a more in-depth overview see the [GitLab architecture doc](../development/architecture.md).

12
doc/logs/logs.md
View file

@ -51,16 +51,6 @@ December 03, 2014 13:20 -> ERROR -> Command failed [1]: /usr/bin/git --git-dir=/
error: failed to push some refs to '/Users/vsizov/gitlab-development-kit/repositories/gitlabhq/gitlab_git.git'
```
#### satellites.log
This file lives in `/var/log/gitlab/gitlab-rails/satellites.log` for omnibus package or in `/home/git/gitlab/log/satellites.log` for installations from the source.
In some cases GitLab should perform write actions to git repository, for example when it is needed to merge the merge request or edit a file with online editor. If something went wrong you can look into this file to find out what exactly happened.
```
October 07, 2014 11:36: Failed to create satellite for Chesley Weimann III / project1817
October 07, 2014 11:36: PID: 1872: git clone /Users/vsizov/gitlab-development-kit/gitlab/tmp/tests/repositories/conrad6841/gitlabhq.git /Users/vsizov/gitlab-development-kit/gitlab/tmp/tests/gitlab-satellites/conrad6841/gitlabhq
October 07, 2014 11:36: PID: 1872: -> fatal: repository '/Users/vsizov/gitlab-development-kit/gitlab/tmp/tests/repositories/conrad6841/gitlabhq.git' does not exist
```
#### sidekiq.log
This file lives in `/var/log/gitlab/gitlab-rails/sidekiq.log` for omnibus package or in `/home/git/gitlab/log/sidekiq.log` for installations from the source.
@ -99,4 +89,4 @@ W, [2015-02-13T07:16:01.313000 #9094] WARN -- : Unicorn::WorkerKiller send SIGQ
I, [2015-02-13T07:16:01.530733 #9047] INFO -- : reaped #<Process::Status: pid 9094 exit 0> worker=1
I, [2015-02-13T07:16:01.534501 #13379] INFO -- : worker=1 spawned pid=13379
I, [2015-02-13T07:16:01.534848 #13379] INFO -- : worker=1 ready
```
```

13
doc/raketasks/maintenance.md
View file

@ -105,24 +105,11 @@ Log directory writable? ... yes
Tmp directory writable? ... yes
Init script exists? ... yes
Init script up-to-date? ... yes
Projects have satellites? ... yes
Redis version >= 2.0.0? ... yes
Checking GitLab ... Finished
```
## (Re-)Create satellite repositories
This will create satellite repositories for all your projects.
If necessary, remove the `repo_satellites` directory and rerun the commands below.
```
sudo -u git -H mkdir -p /home/git/gitlab-satellites
sudo -u git -H bundle exec rake gitlab:satellites:create RAILS_ENV=production
sudo chmod u+rwx,g=rx,o-rwx /home/git/gitlab-satellites
```
## Rebuild authorized_keys file
In some case it is necessary to rebuild the `authorized_keys` file.

1
features/steps/project/forked_merge_requests.rb
View file

@ -9,7 +9,6 @@ class Spinach::Features::ProjectForkedMergeRequests < Spinach::FeatureSteps
@project = Project.find_by(name: "Shop")
@project ||= create(:project, name: "Shop")
@project.team << [@user, :reporter]
@project.ensure_satellite_exists
end
step 'I have a project forked off of "Shop" called "Forked Shop"' do

5
features/steps/project/merge_requests.rb
View file

@ -198,15 +198,10 @@ class Spinach::Features::ProjectMergeRequests < Spinach::FeatureSteps
end
step 'merge request "Bug NS-05" is mergeable' do
merge_request.project.satellite.create
merge_request.mark_as_mergeable
end
step 'I accept this merge request' do
Gitlab::Satellite::MergeAction.any_instance.stub(
merge!: true,
)
page.within '.mr-state-widget' do
click_button "Accept Merge Request"
end

5
lib/gitlab.rb
View file

@ -1,5 +0,0 @@
require 'gitlab/git'
module Gitlab
autoload :Satellite, 'gitlab/satellite/satellite'
end

14
lib/gitlab/backend/shell.rb
View file

@ -217,20 +217,6 @@ module Gitlab
FileUtils.mv(full_path(old_name), full_path(new_name))
end
# Remove GitLab Satellites for provided path (namespace or repo dir)
#
# Ex.
# rm_satellites("gitlab")
#
# rm_satellites("gitlab/gitlab-ci.git")
#
def rm_satellites(path)
raise ArgumentError.new("Path can't be blank") if path.blank?
satellites_path = File.join(Gitlab.config.satellites.path, path)
FileUtils.rm_r(satellites_path, force: true)
end
def url_to_repo(path)
Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git"
end

58
lib/gitlab/satellite/action.rb
View file

@ -1,58 +0,0 @@
module Gitlab
module Satellite
class Action
DEFAULT_OPTIONS = { git_timeout: Gitlab.config.satellites.timeout.seconds }
attr_accessor :options, :project, :user
def initialize(user, project, options = {})
@options = DEFAULT_OPTIONS.merge(options)
@project = project
@user = user
end
protected
# * Sets a 30s timeout for Git
# * Locks the satellite repo
# * Yields the prepared satellite repo
def in_locked_and_timed_satellite
Gitlab::ShellEnv.set_env(user)
Grit::Git.with_timeout(options[:git_timeout]) do
project.satellite.lock do
return yield project.satellite.repo
end
end
rescue Errno::ENOMEM => ex
return handle_exception(ex)
rescue Grit::Git::GitTimeout => ex
return handle_exception(ex)
ensure
Gitlab::ShellEnv.reset_env
end
# * Recreates the satellite
# * Sets up Git variables for the user
#
# Note: use this within #in_locked_and_timed_satellite
def prepare_satellite!(repo)
project.satellite.clear_and_update!
if user
repo.config['user.name'] = user.name
repo.config['user.email'] = user.email
end
end
def default_options(options = {})
{ raise: true, timeout: true }.merge(options)
end
def handle_exception(exception)
Gitlab::GitLogger.error(exception.message)
false
end
end
end
end

44
lib/gitlab/satellite/compare_action.rb
View file

@ -1,44 +0,0 @@
module Gitlab
module Satellite
class BranchesWithoutParent < StandardError; end
class CompareAction < Action
def initialize(user, target_project, target_branch, source_project, source_branch)
super user, target_project
@target_project, @target_branch = target_project, target_branch
@source_project, @source_branch = source_project, source_branch
end
# Compare 2 repositories and return Gitlab::CompareResult object
def result
in_locked_and_timed_satellite do |target_repo|
prepare_satellite!(target_repo)
update_satellite_source_and_target!(target_repo)
Gitlab::CompareResult.new(compare(target_repo))
end
rescue Grit::Git::CommandFailed => ex
raise BranchesWithoutParent
end
private
# Assumes a satellite exists that is a fresh clone of the projects repo, prepares satellite for diffs
def update_satellite_source_and_target!(target_repo)
target_repo.remote_add('source', @source_project.repository.path_to_repo)
target_repo.remote_fetch('source')
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
def compare(repo)
@compare ||= Gitlab::Git::Compare.new(
Gitlab::Git::Repository.new(repo.path),
"origin/#{@target_branch}",
"source/#{@source_branch}"
)
end
end
end
end

13
lib/gitlab/satellite/logger.rb
View file

@ -1,13 +0,0 @@
module Gitlab
module Satellite
class Logger < Gitlab::Logger
def self.file_name
'satellites.log'
end
def format_message(severity, timestamp, progname, msg)
"#{timestamp.to_s(:long)}: #{msg}\n"
end
end
end
end

146
lib/gitlab/satellite/merge_action.rb
View file

@ -1,146 +0,0 @@
module Gitlab
module Satellite
# GitLab server-side merge
class MergeAction < Action
attr_accessor :merge_request
def initialize(user, merge_request)
super user, merge_request.target_project
@merge_request = merge_request
end
# Checks if a merge request can be executed without user interaction
def can_be_merged?
in_locked_and_timed_satellite do |merge_repo|
prepare_satellite!(merge_repo)
merge_in_satellite!(merge_repo)
end
end
# Merges the source branch into the target branch in the satellite and
# pushes it back to the repository.
# It also removes the source branch if requested in the merge request (and this is permitted by the merge request).
#
# Returns false if the merge produced conflicts
# Returns false if pushing from the satellite to the repository failed or was rejected
# Returns true otherwise
def merge!(merge_commit_message = nil)
in_locked_and_timed_satellite do |merge_repo|
prepare_satellite!(merge_repo)
if merge_in_satellite!(merge_repo, merge_commit_message)
# push merge back to bare repo
# will raise CommandFailed when push fails
merge_repo.git.push(default_options, :origin, merge_request.target_branch)
# remove source branch
if merge_request.should_remove_source_branch && !project.root_ref?(merge_request.source_branch) && !merge_request.for_fork?
# will raise CommandFailed when push fails
merge_repo.git.push(default_options, :origin, ":#{merge_request.source_branch}")
end
# merge, push and branch removal successful
true
end
end
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
def diff_in_satellite
in_locked_and_timed_satellite do |merge_repo|
prepare_satellite!(merge_repo)
update_satellite_source_and_target!(merge_repo)
# Only show what is new in the source branch compared to the target branch, not the other way around.
# The line below with merge_base is equivalent to diff with three dots (git diff branch1...branch2)
# From the git documentation: "git diff A...B" is equivalent to "git diff $(git-merge-base A B) B"
common_commit = merge_repo.git.native(:merge_base, default_options, ["origin/#{merge_request.target_branch}", "source/#{merge_request.source_branch}"]).strip
merge_repo.git.native(:diff, default_options, common_commit, "source/#{merge_request.source_branch}")
end
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
def diffs_between_satellite
in_locked_and_timed_satellite do |merge_repo|
prepare_satellite!(merge_repo)
update_satellite_source_and_target!(merge_repo)
if merge_request.for_fork?
repository = Gitlab::Git::Repository.new(merge_repo.path)
diffs = Gitlab::Git::Diff.between(
repository,
"source/#{merge_request.source_branch}",
"origin/#{merge_request.target_branch}"
)
else
raise "Attempt to determine diffs between for a non forked merge request in satellite MergeRequest.id:[#{merge_request.id}]"
end
return diffs
end
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
# Get commit as an email patch
def format_patch
in_locked_and_timed_satellite do |merge_repo|
prepare_satellite!(merge_repo)
update_satellite_source_and_target!(merge_repo)
patch = merge_repo.git.format_patch(default_options({ stdout: true }), "origin/#{merge_request.target_branch}..source/#{merge_request.source_branch}")
end
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
# Retrieve an array of commits between the source and the target
def commits_between
in_locked_and_timed_satellite do |merge_repo|
prepare_satellite!(merge_repo)
update_satellite_source_and_target!(merge_repo)
if merge_request.for_fork?
repository = Gitlab::Git::Repository.new(merge_repo.path)
commits = Gitlab::Git::Commit.between(
repository,
"origin/#{merge_request.target_branch}",
"source/#{merge_request.source_branch}"
)
else
raise "Attempt to determine commits between for a non forked merge request in satellite MergeRequest.id:[#{merge_request.id}]"
end
return commits
end
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
private
# Merges the source_branch into the target_branch in the satellite.
#
# Note: it will clear out the satellite before doing anything
#
# Returns false if the merge produced conflicts
# Returns true otherwise
def merge_in_satellite!(repo, message = nil)
update_satellite_source_and_target!(repo)
message ||= "Merge branch '#{merge_request.source_branch}' into '#{merge_request.target_branch}'"
# merge the source branch into the satellite
# will raise CommandFailed when merge fails
repo.git.merge(default_options({ no_ff: true }), "-m#{message}", "source/#{merge_request.source_branch}")
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
# Assumes a satellite exists that is a fresh clone of the projects repo, prepares satellite for merges, diffs etc
def update_satellite_source_and_target!(repo)
repo.remote_add('source', merge_request.source_project.repository.path_to_repo)
repo.remote_fetch('source')
repo.git.checkout(default_options({ b: true }), merge_request.target_branch, "origin/#{merge_request.target_branch}")
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
end
end
end

148
lib/gitlab/satellite/satellite.rb
View file

@ -1,148 +0,0 @@
module Gitlab
module Satellite
autoload :DeleteFileAction, 'gitlab/satellite/files/delete_file_action'
autoload :EditFileAction, 'gitlab/satellite/files/edit_file_action'
autoload :FileAction, 'gitlab/satellite/files/file_action'
autoload :NewFileAction, 'gitlab/satellite/files/new_file_action'
class CheckoutFailed < StandardError; end
class CommitFailed < StandardError; end
class PushFailed < StandardError; end
class Satellite
include Gitlab::Popen
PARKING_BRANCH = "__parking_branch"
attr_accessor :project
def initialize(project)
@project = project
end
def log(message)
Gitlab::Satellite::Logger.error(message)
end
def clear_and_update!
project.ensure_satellite_exists
@repo = nil
clear_working_dir!
delete_heads!
remove_remotes!
update_from_source!
end
def create
output, status = popen(%W(git clone -- #{project.repository.path_to_repo} #{path}),
Gitlab.config.satellites.path)
log("PID: #{project.id}: git clone #{project.repository.path_to_repo} #{path}")
log("PID: #{project.id}: -> #{output}")
if status.zero?
true
else
log("Failed to create satellite for #{project.name_with_namespace}")
false
end
end
def exists?
File.exists? path
end
# * Locks the satellite
# * Changes the current directory to the satellite's working dir
# * Yields
def lock
project.ensure_satellite_exists
File.open(lock_file, "w+") do |f|
begin
f.flock File::LOCK_EX
yield
ensure
f.flock File::LOCK_UN
end
end
end
def lock_file
create_locks_dir unless File.exists?(lock_files_dir)
File.join(lock_files_dir, "satellite_#{project.id}.lock")
end
def path
File.join(Gitlab.config.satellites.path, project.path_with_namespace)
end
def repo
project.ensure_satellite_exists
@repo ||= Grit::Repo.new(path)
end
def destroy
FileUtils.rm_rf(path)
end
private
# Clear the working directory
def clear_working_dir!
repo.git.reset(hard: true)
repo.git.clean(f: true, d: true, x: true)
end
# Deletes all branches except the parking branch
#
# This ensures we have no name clashes or issues updating branches when
# working with the satellite.
def delete_heads!
heads = repo.heads.map(&:name)
# update or create the parking branch
repo.git.checkout(default_options({ B: true }), PARKING_BRANCH)
# remove the parking branch from the list of heads ...
heads.delete(PARKING_BRANCH)
# ... and delete all others
heads.each { |head| repo.git.branch(default_options({ D: true }), head) }
end
# Deletes all remotes except origin
#
# This ensures we have no remote name clashes or issues updating branches when
# working with the satellite.
def remove_remotes!
remotes = repo.git.remote.split(' ')
remotes.delete('origin')
remotes.each { |name| repo.git.remote(default_options,'rm', name)}
end
# Updates the satellite from bare repo
#
# Note: this will only update remote branches (i.e. origin/*)
def update_from_source!
repo.git.remote(default_options, 'set-url', :origin, project.repository.path_to_repo)
repo.git.fetch(default_options, :origin)
end
def default_options(options = {})
{ raise: true, timeout: true }.merge(options)
end
# Create directory for storing
# satellites lock files
def create_locks_dir
FileUtils.mkdir_p(lock_files_dir)
end
def lock_files_dir
@lock_files_dir ||= File.join(Gitlab.config.satellites.path, "tmp")
end
end
end
end

56
lib/tasks/gitlab/check.rake
View file

@ -25,7 +25,6 @@ namespace :gitlab do
check_init_script_exists
check_init_script_up_to_date
check_projects_have_namespace
check_satellites_exist
check_redis_version
check_ruby_version
check_git_version
@ -238,37 +237,6 @@ namespace :gitlab do
end
end
def check_satellites_exist
print "Projects have satellites? ... "
unless Project.count > 0
puts "can't check, you have no projects".magenta
return
end
puts ""
Project.find_each(batch_size: 100) do |project|
print sanitized_message(project)
if project.satellite.exists?
puts "yes".green
elsif project.empty_repo?
puts "can't create, repository is empty".magenta
else
puts "no".red
try_fixing_it(
sudo_gitlab("bundle exec rake gitlab:satellites:create RAILS_ENV=production"),
"If necessary, remove the tmp/repo_satellites directory ...",
"... and rerun the above command"
)
for_more_information(
"doc/raketasks/maintenance.md "
)
fix_and_rerun
end
end
end
def check_log_writable
print "Log directory writable? ... "
@ -339,7 +307,6 @@ namespace :gitlab do
check_repo_base_is_not_symlink
check_repo_base_user_and_group
check_repo_base_permissions
check_satellites_permissions
check_repos_hooks_directory_is_link
check_gitlab_shell_self_test
@ -417,29 +384,6 @@ namespace :gitlab do
end
end
def check_satellites_permissions
print "Satellites access is drwxr-x---? ... "
satellites_path = Gitlab.config.satellites.path
unless File.exists?(satellites_path)
puts "can't check because of previous errors".magenta
return
end
if File.stat(satellites_path).mode.to_s(8).ends_with?("0750")
puts "yes".green
else
puts "no".red
try_fixing_it(
"sudo chmod u+rwx,g=rx,o-rwx #{satellites_path}",
)
for_more_information(
see_installation_guide_section "GitLab"
)
fix_and_rerun
end
end
def check_repo_base_user_and_group
gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user
gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group

39
lib/tasks/gitlab/enable_automerge.rake
View file

@ -1,39 +0,0 @@
namespace :gitlab do
namespace :satellites do
desc "GitLab | Create satellite repos"
task create: :environment do
create_satellites
end
end
def create_satellites
warn_user_is_not_gitlab
print "Creating satellites for ..."
unless Project.count > 0
puts "skipping, because you have no projects".magenta
return
end
puts ""
Project.find_each(batch_size: 100) do |project|
print "#{project.name_with_namespace.yellow} ... "
unless project.repo_exists?
puts "skipping, because the repo is empty".magenta
next
end
if project.satellite.exists?
puts "exists already".green
else
print "\n... "
if project.satellite.create
puts "created".green
else
puts "error".red
end
end
end
end
end

116
spec/lib/gitlab/satellite/action_spec.rb
View file

@ -1,116 +0,0 @@
require 'spec_helper'
describe 'Gitlab::Satellite::Action' do
let(:project) { create(:project) }
let(:user) { create(:user) }
describe '#prepare_satellite!' do
it 'should be able to fetch timeout from conf' do
expect(Gitlab::Satellite::Action::DEFAULT_OPTIONS[:git_timeout]).to eq(30.seconds)
end
it 'create a repository with a parking branch and one remote: origin' do
repo = project.satellite.repo
#now lets dirty it up
starting_remote_count = repo.git.list_remotes.size
expect(starting_remote_count).to be >= 1
#kind of hookey way to add a second remote
origin_uri = repo.git.remote({ v: true }).split(" ")[1]
repo.git.remote({ raise: true }, 'add', 'another-remote', origin_uri)
repo.git.branch({ raise: true }, 'a-new-branch')
expect(repo.heads.size).to be > (starting_remote_count)
expect(repo.git.remote().split(" ").size).to be > (starting_remote_count)
repo.git.config({}, "user.name", "#{user.name} -- foo")
repo.git.config({}, "user.email", "#{user.email} -- foo")
expect(repo.config['user.name']).to eq("#{user.name} -- foo")
expect(repo.config['user.email']).to eq("#{user.email} -- foo")
#These must happen in the context of the satellite directory...
satellite_action = Gitlab::Satellite::Action.new(user, project)
project.satellite.lock do
#Now clean it up, use send to get around prepare_satellite! being protected
satellite_action.send(:prepare_satellite!, repo)
end
#verify it's clean
heads = repo.heads.map(&:name)
expect(heads.size).to eq(1)
expect(heads.include?(Gitlab::Satellite::Satellite::PARKING_BRANCH)).to eq(true)
remotes = repo.git.remote().split(' ')
expect(remotes.size).to eq(1)
expect(remotes.include?('origin')).to eq(true)
expect(repo.config['user.name']).to eq(user.name)
expect(repo.config['user.email']).to eq(user.email)
end
end
describe '#in_locked_and_timed_satellite' do
it 'should make use of a lockfile' do
repo = project.satellite.repo
called = false
#set assumptions
FileUtils.rm_f(project.satellite.lock_file)
expect(File.exists?(project.satellite.lock_file)).to be_falsey
satellite_action = Gitlab::Satellite::Action.new(user, project)
satellite_action.send(:in_locked_and_timed_satellite) do |sat_repo|
expect(repo).to eq(sat_repo)
expect(File.exists? project.satellite.lock_file).to be_truthy
called = true
end
expect(called).to be_truthy
end
it 'should be able to use the satellite after locking' do
repo = project.satellite.repo
called = false
# Set base assumptions
if File.exists? project.satellite.lock_file
expect(FileLockStatusChecker.new(project.satellite.lock_file).flocked?).to be_falsey
end
satellite_action = Gitlab::Satellite::Action.new(user, project)
satellite_action.send(:in_locked_and_timed_satellite) do |sat_repo|
called = true
expect(repo).to eq(sat_repo)
expect(File.exists? project.satellite.lock_file).to be_truthy
expect(FileLockStatusChecker.new(project.satellite.lock_file).flocked?).to be_truthy
end
expect(called).to be_truthy
expect(FileLockStatusChecker.new(project.satellite.lock_file).flocked?).to be_falsey
end
class FileLockStatusChecker < File
def flocked?(&block)
status = flock LOCK_EX|LOCK_NB
case status
when false
return true
when 0
begin
block ? block.call : false
ensure
flock LOCK_UN
end
else
raise SystemCallError, status
end
end
end
end
end

104
spec/lib/gitlab/satellite/merge_action_spec.rb
View file

@ -1,104 +0,0 @@
require 'spec_helper'
describe 'Gitlab::Satellite::MergeAction' do
include RepoHelpers
let(:project) { create(:project, namespace: create(:group)) }
let(:fork_project) { create(:project, namespace: create(:group), forked_from_project: project) }
let(:merge_request) { create(:merge_request, source_project: project, target_project: project) }
let(:merge_request_fork) { create(:merge_request, source_project: fork_project, target_project: project) }
let(:merge_request_with_conflict) { create(:merge_request, :conflict, source_project: project, target_project: project) }
let(:merge_request_fork_with_conflict) { create(:merge_request, :conflict, source_project: project, target_project: project) }
describe '#commits_between' do
def verify_commits(commits, first_commit_sha, last_commit_sha)
commits.each { |commit| expect(commit.class).to eq(Gitlab::Git::Commit) }
expect(commits.first.id).to eq(first_commit_sha)
expect(commits.last.id).to eq(last_commit_sha)
end
context 'on fork' do
it 'should get proper commits between' do
commits = Gitlab::Satellite::MergeAction.new(merge_request_fork.author, merge_request_fork).commits_between
verify_commits(commits, sample_compare.commits.first, sample_compare.commits.last)
end
end
context 'between branches' do
it 'should raise exception -- not expected to be used by non forks' do
expect { Gitlab::Satellite::MergeAction.new(merge_request.author, merge_request).commits_between }.to raise_error(RuntimeError)
end
end
end
describe '#format_patch' do
def verify_content(patch)
sample_compare.commits.each do |commit|
expect(patch.include?(commit)).to be_truthy
end
end
context 'on fork' do
it 'should build a format patch' do
patch = Gitlab::Satellite::MergeAction.new(merge_request_fork.author, merge_request_fork).format_patch
verify_content(patch)
end
end
context 'between branches' do
it 'should build a format patch' do
patch = Gitlab::Satellite::MergeAction.new(merge_request_fork.author, merge_request).format_patch
verify_content(patch)
end
end
end
describe '#diffs_between_satellite tested against diff_in_satellite' do
def is_a_matching_diff(diff, diffs)
diff_count = diff.scan('diff --git').size
expect(diff_count).to be >= 1
expect(diffs.size).to eq(diff_count)
diffs.each do |a_diff|
expect(a_diff.class).to eq(Gitlab::Git::Diff)
expect(diff.include? a_diff.diff).to be_truthy
end
end
context 'on fork' do
it 'should get proper diffs' do
diffs = Gitlab::Satellite::MergeAction.new(merge_request_fork.author, merge_request_fork).diffs_between_satellite
diff = Gitlab::Satellite::MergeAction.new(merge_request.author, merge_request_fork).diff_in_satellite
is_a_matching_diff(diff, diffs)
end
end
context 'between branches' do
it 'should get proper diffs' do
expect{ Gitlab::Satellite::MergeAction.new(merge_request.author, merge_request).diffs_between_satellite }.to raise_error(RuntimeError)
end
end
end
describe '#can_be_merged?' do
context 'on fork' do
it do
expect(Gitlab::Satellite::MergeAction.new(merge_request_fork.author, merge_request_fork).can_be_merged?).to be_truthy
end
it do
expect(Gitlab::Satellite::MergeAction.new(merge_request_fork_with_conflict.author, merge_request_fork_with_conflict).can_be_merged?).to be_falsey
end
end
context 'between branches' do
it do
expect(Gitlab::Satellite::MergeAction.new(merge_request.author, merge_request).can_be_merged?).to be_truthy
end
it do
expect(Gitlab::Satellite::MergeAction.new(merge_request_with_conflict.author, merge_request_with_conflict).can_be_merged?).to be_falsey
end
end
end
end

1
spec/models/project_spec.rb
View file

@ -90,7 +90,6 @@ describe Project do
describe 'Respond to' do
it { is_expected.to respond_to(:url_to_repo) }
it { is_expected.to respond_to(:repo_exists?) }
it { is_expected.to respond_to(:satellite) }
it { is_expected.to respond_to(:update_merge_requests) }
it { is_expected.to respond_to(:execute_hooks) }
it { is_expected.to respond_to(:name_with_namespace) }

2
spec/requests/api/files_spec.rb
View file

@ -110,7 +110,7 @@ describe API::API, api: true do
expect(response.status).to eq(400)
end
it "should return a 400 if satellite fails to create file" do
it "should return a 400 if fails to create file" do
allow_any_instance_of(Repository).to receive(:remove_file).and_return(false)
delete api("/projects/#{project.id}/repository/files", user), valid_params