From a87989fb7d0c4f05658ba29dec48ecefedf32334 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Wed, 15 Jul 2015 15:45:57 +0200 Subject: [PATCH] Remove satellites --- Gemfile.lock | 3 - .../projects/merge_requests_controller.rb | 2 - app/models/merge_request.rb | 6 +- app/models/namespace.rb | 5 +- app/models/project.rb | 14 +- app/services/compare_service.rb | 11 +- app/services/git_push_service.rb | 10 +- .../merge_requests/auto_merge_service.rb | 5 +- app/services/merge_requests/build_service.rb | 4 - app/services/projects/destroy_service.rb | 1 - app/services/projects/transfer_service.rb | 6 - .../merge_requests/_new_compare.html.haml | 2 +- .../merge_requests/widget/_open.html.haml | 2 - .../widget/open/_no_satellite.html.haml | 3 - app/workers/repository_import_worker.rb | 1 - doc/development/architecture.md | 2 +- doc/install/installation.md | 4 - doc/install/structure.md | 4 +- doc/logs/logs.md | 12 +- doc/raketasks/maintenance.md | 13 -- .../steps/project/forked_merge_requests.rb | 1 - features/steps/project/merge_requests.rb | 5 - lib/gitlab.rb | 5 - lib/gitlab/backend/shell.rb | 14 -- lib/gitlab/satellite/action.rb | 58 ------- lib/gitlab/satellite/compare_action.rb | 44 ------ lib/gitlab/satellite/logger.rb | 13 -- lib/gitlab/satellite/merge_action.rb | 146 ----------------- lib/gitlab/satellite/satellite.rb | 148 ------------------ lib/tasks/gitlab/check.rake | 56 ------- lib/tasks/gitlab/enable_automerge.rake | 39 ----- spec/lib/gitlab/satellite/action_spec.rb | 116 -------------- .../lib/gitlab/satellite/merge_action_spec.rb | 104 ------------ spec/models/project_spec.rb | 1 - spec/requests/api/files_spec.rb | 2 +- 35 files changed, 18 insertions(+), 844 deletions(-) delete mode 100644 app/views/projects/merge_requests/widget/open/_no_satellite.html.haml delete mode 100644 lib/gitlab.rb delete mode 100644 lib/gitlab/satellite/action.rb delete mode 100644 lib/gitlab/satellite/compare_action.rb delete mode 100644 lib/gitlab/satellite/logger.rb delete mode 100644 lib/gitlab/satellite/merge_action.rb delete mode 100644 lib/gitlab/satellite/satellite.rb delete mode 100644 lib/tasks/gitlab/enable_automerge.rake delete mode 100644 spec/lib/gitlab/satellite/action_spec.rb delete mode 100644 spec/lib/gitlab/satellite/merge_action_spec.rb diff --git a/Gemfile.lock b/Gemfile.lock index 6e571072a4c..12ba2fa295a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -875,6 +875,3 @@ DEPENDENCIES virtus webmock (~> 1.21.0) wikicloth (= 0.8.1) - -BUNDLED WITH - 1.10.5 diff --git a/app/controllers/projects/merge_requests_controller.rb b/app/controllers/projects/merge_requests_controller.rb index d1265198318..0ef9e99123d 100644 --- a/app/controllers/projects/merge_requests_controller.rb +++ b/app/controllers/projects/merge_requests_controller.rb @@ -1,5 +1,3 @@ -require 'gitlab/satellite/satellite' - class Projects::MergeRequestsController < Projects::ApplicationController before_action :module_enabled before_action :merge_request, only: [ diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 53b3fc10ccb..aa2137e7469 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -207,7 +207,7 @@ class MergeRequest < ActiveRecord::Base def check_if_can_be_merged can_be_merged = if for_fork? - Gitlab::Satellite::MergeAction.new(self.author, self).can_be_merged? + raise 'Implement me' else project.repository.can_be_merged?(source_branch, target_branch) end @@ -274,14 +274,14 @@ class MergeRequest < ActiveRecord::Base # # see "git diff" def to_diff(current_user) - Gitlab::Satellite::MergeAction.new(current_user, self).diff_in_satellite + raise 'Implement me' end # Returns the commit as a series of email patches. # # see "git format-patch" def to_patch(current_user) - Gitlab::Satellite::MergeAction.new(current_user, self).format_patch + raise 'Implement me' end def hook_attrs diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 03d2ab165ea..815672a1bf7 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -115,12 +115,11 @@ class Namespace < ActiveRecord::Base def move_dir if gitlab_shell.mv_namespace(path_was, path) - # If repositories moved successfully we need to remove old satellites - # and send update instructions to users. + # If repositories moved successfully we need to + # send update instructions to users. # However we cannot allow rollback since we moved namespace dir # So we basically we mute exceptions in next actions begin - gitlab_shell.rm_satellites(path_was) send_update_instructions rescue # Returning false does not rollback after_* transaction but gives diff --git a/app/models/project.rb b/app/models/project.rb index b161cbe86b9..8481a149e21 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -520,14 +520,6 @@ class Project < ActiveRecord::Base !repository.exists? || repository.empty? end - def ensure_satellite_exists - self.satellite.create unless self.satellite.exists? - end - - def satellite - @satellite ||= Gitlab::Satellite::Satellite.new(self) - end - def repo repository.raw end @@ -597,14 +589,11 @@ class Project < ActiveRecord::Base new_path_with_namespace = File.join(namespace_dir, path) if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace) - # If repository moved successfully we need to remove old satellite - # and send update instructions to users. + # If repository moved successfully we need to send update instructions to users. # However we cannot allow rollback since we moved repository # So we basically we mute exceptions in next actions begin gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") - gitlab_shell.rm_satellites(old_path_with_namespace) - ensure_satellite_exists send_move_instructions reset_events_cache rescue @@ -698,7 +687,6 @@ class Project < ActiveRecord::Base def create_repository if forked? if gitlab_shell.fork_repository(forked_from_project.path_with_namespace, self.namespace.path) - ensure_satellite_exists true else errors.add(:base, 'Failed to fork repository') diff --git a/app/services/compare_service.rb b/app/services/compare_service.rb index 6aa9df4b194..48ad7fab4e6 100644 --- a/app/services/compare_service.rb +++ b/app/services/compare_service.rb @@ -3,9 +3,6 @@ class CompareService def execute(current_user, source_project, source_branch, target_project, target_branch) # Try to compare branches to get commits list and diffs - # - # Note: Use satellite only when need to compare between two repos - # because satellites are slower than operations on bare repo if target_project == source_project Gitlab::CompareResult.new( Gitlab::Git::Compare.new( @@ -15,13 +12,7 @@ class CompareService ) ) else - Gitlab::Satellite::CompareAction.new( - current_user, - target_project, - target_branch, - source_project, - source_branch - ).result + raise 'Implement me' end end end diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb index 6135ae65007..3a5a52e6f55 100644 --- a/app/services/git_push_service.rb +++ b/app/services/git_push_service.rb @@ -10,16 +10,14 @@ class GitPushService # # Next, this method: # 1. Creates the push event - # 2. Ensures that the project satellite exists - # 3. Updates merge requests - # 4. Recognizes cross-references from commit messages - # 5. Executes the project's web hooks - # 6. Executes the project's services + # 2. Updates merge requests + # 3. Recognizes cross-references from commit messages + # 4. Executes the project's web hooks + # 5. Executes the project's services # def execute(project, user, oldrev, newrev, ref) @project, @user = project, user - project.ensure_satellite_exists project.repository.expire_cache project.update_repository_size diff --git a/app/services/merge_requests/auto_merge_service.rb b/app/services/merge_requests/auto_merge_service.rb index df793fc997d..1c41148d41d 100644 --- a/app/services/merge_requests/auto_merge_service.rb +++ b/app/services/merge_requests/auto_merge_service.rb @@ -1,7 +1,7 @@ module MergeRequests # AutoMergeService class # - # Do git merge in satellite and in case of success + # Do git merge and in case of success # mark merge request as merged and execute all hooks and notifications # Called when you do merge via GitLab UI class AutoMergeService < BaseMergeService @@ -32,9 +32,8 @@ module MergeRequests def merge! if merge_request.for_fork? - Gitlab::Satellite::MergeAction.new(current_user, merge_request).merge!(commit_message) + raise 'Implement me' else - # Merge local branches using rugged instead of satellites if sha = commit after_commit(sha, merge_request.target_branch) end diff --git a/app/services/merge_requests/build_service.rb b/app/services/merge_requests/build_service.rb index 956480938c3..5ef1cd2caa0 100644 --- a/app/services/merge_requests/build_service.rb +++ b/app/services/merge_requests/build_service.rb @@ -40,7 +40,6 @@ module MergeRequests merge_request.compare_diffs = diffs elsif diffs == false - # satellite timeout return false merge_request.can_be_created = false merge_request.compare_failed = true end @@ -59,9 +58,6 @@ module MergeRequests end merge_request - - rescue Gitlab::Satellite::BranchesWithoutParent - return build_failed(merge_request, "Selected branches have no common commit so they cannot be merged.") end def build_failed(merge_request, message) diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index 403f419ec50..28872c89259 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -27,7 +27,6 @@ module Projects end end - project.satellite.destroy log_info("Project \"#{project.name}\" was removed") system_hook_service.execute_hooks_for(project, :destroy) true diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index f43c0ef70e9..550ed6897dd 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -33,9 +33,6 @@ module Projects raise TransferError.new("Project with same path in target namespace already exists") end - # Remove old satellite - project.satellite.destroy - # Apply new namespace id project.namespace = new_namespace project.save! @@ -51,9 +48,6 @@ module Projects # Move wiki repo also if present gitlab_shell.mv_repository("#{old_path}.wiki", "#{new_path}.wiki") - # Create a new satellite (reload project from DB) - Project.find(project.id).ensure_satellite_exists - # clear project cached events project.reset_events_cache diff --git a/app/views/projects/merge_requests/_new_compare.html.haml b/app/views/projects/merge_requests/_new_compare.html.haml index ff9c0cdb283..7709330611a 100644 --- a/app/views/projects/merge_requests/_new_compare.html.haml +++ b/app/views/projects/merge_requests/_new_compare.html.haml @@ -35,7 +35,7 @@ - if @merge_request.compare_failed .alert.alert-danger %h4 Compare failed - %p We can't compare selected branches. It may be because of huge diff or satellite timeout. Please try again or select different branches. + %p We can't compare selected branches. It may be because of huge diff. Please try again or select different branches. - else .light-well .center diff --git a/app/views/projects/merge_requests/widget/_open.html.haml b/app/views/projects/merge_requests/widget/_open.html.haml index bb794912f8f..f420cdcab49 100644 --- a/app/views/projects/merge_requests/widget/_open.html.haml +++ b/app/views/projects/merge_requests/widget/_open.html.haml @@ -3,8 +3,6 @@ .mr-widget-body - if @project.archived? = render 'projects/merge_requests/widget/open/archived' - - elsif !@project.satellite.exists? - = render 'projects/merge_requests/widget/open/no_satellite' - elsif @merge_request.commits.blank? = render 'projects/merge_requests/widget/open/nothing' - elsif @merge_request.branch_missing? diff --git a/app/views/projects/merge_requests/widget/open/_no_satellite.html.haml b/app/views/projects/merge_requests/widget/open/_no_satellite.html.haml deleted file mode 100644 index 3718cfd8333..00000000000 --- a/app/views/projects/merge_requests/widget/open/_no_satellite.html.haml +++ /dev/null @@ -1,3 +0,0 @@ -%p - %span - %strong This repository does not have a satellite. Please ask an administrator to fix this issue! diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index e6a50afedb1..4e14e4883bb 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -27,7 +27,6 @@ class RepositoryImportWorker project.import_finish project.save - project.satellite.create unless project.satellite.exists? project.update_repository_size Gitlab::BitbucketImport::KeyDeleter.new(project).execute if project.import_type == 'bitbucket' end diff --git a/doc/development/architecture.md b/doc/development/architecture.md index 541af487bb1..99f56016120 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -58,7 +58,7 @@ A typical install of GitLab will be on GNU/Linux. It uses Nginx or Apache as a w The GitLab web app uses MySQL or PostgreSQL for persistent database information (e.g. users, permissions, issues, other meta data). GitLab stores the bare git repositories it serves in `/home/git/repositories` by default. It also keeps default branch and hook information with the bare repository. `/home/git/gitlab-satellites` keeps checked out repositories when performing actions such as a merge request, editing files in the web interface, etc. -The satellite repository is used by the web interface for editing repositories and the wiki which is also a git repository. When serving repositories over HTTP/HTTPS GitLab utilizes the GitLab API to resolve authorization and access as well as serving git objects. +[DEPRECATED] The satellite repository is used by the web interface for editing repositories and the wiki which is also a git repository. When serving repositories over HTTP/HTTPS GitLab utilizes the GitLab API to resolve authorization and access as well as serving git objects. The add-on component gitlab-shell serves repositories over SSH. It manages the SSH keys within `/home/git/.ssh/authorized_keys` which should not be manually edited. gitlab-shell accesses the bare repositories directly to serve git objects and communicates with redis to submit jobs to Sidekiq for GitLab to process. gitlab-shell queries the GitLab API to determine authorization and access. diff --git a/doc/install/installation.md b/doc/install/installation.md index 8b918cba133..5bd68ab6a35 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -216,10 +216,6 @@ We recommend using a PostgreSQL database. For MySQL check [MySQL setup guide](da sudo chmod -R u+rwX,go-w log/ sudo chmod -R u+rwX tmp/ - # Create directory for satellites - sudo -u git -H mkdir /home/git/gitlab-satellites - sudo chmod u+rwx,g=rx,o-rwx /home/git/gitlab-satellites - # Make sure GitLab can write to the tmp/pids/ and tmp/sockets/ directories sudo chmod -R u+rwX tmp/pids/ sudo chmod -R u+rwX tmp/sockets/ diff --git a/doc/install/structure.md b/doc/install/structure.md index 5c03f073c18..d58b0040eef 100644 --- a/doc/install/structure.md +++ b/doc/install/structure.md @@ -6,16 +6,14 @@ This is the directory structure you will end up with following the instructions | |-- git | |-- .ssh | |-- gitlab - | |-- gitlab-satellites | |-- gitlab-shell | |-- repositories * `/home/git/.ssh` - contains openssh settings. Specifically the `authorized_keys` file managed by gitlab-shell. * `/home/git/gitlab` - GitLab core software. -* `/home/git/gitlab-satellites` - checked out repositories for merge requests and file editing from web UI. This can be treated as a temporary files directory. * `/home/git/gitlab-shell` - Core add-on component of GitLab. Maintains SSH cloning and other functionality. * `/home/git/repositories` - bare repositories for all projects organized by namespace. This is where the git repositories which are pushed/pulled are maintained for all projects. **This area is critical data for projects. [Keep a backup](../raketasks/backup_restore.md)** -*Note: the default locations for gitlab-satellites and repositories can be configured in `config/gitlab.yml` of GitLab and `config.yml` of gitlab-shell.* +*Note: the default locations for repositories can be configured in `config/gitlab.yml` of GitLab and `config.yml` of gitlab-shell.* To see a more in-depth overview see the [GitLab architecture doc](../development/architecture.md). diff --git a/doc/logs/logs.md b/doc/logs/logs.md index 83c32b09253..27937e51764 100644 --- a/doc/logs/logs.md +++ b/doc/logs/logs.md @@ -51,16 +51,6 @@ December 03, 2014 13:20 -> ERROR -> Command failed [1]: /usr/bin/git --git-dir=/ error: failed to push some refs to '/Users/vsizov/gitlab-development-kit/repositories/gitlabhq/gitlab_git.git' ``` -#### satellites.log -This file lives in `/var/log/gitlab/gitlab-rails/satellites.log` for omnibus package or in `/home/git/gitlab/log/satellites.log` for installations from the source. - -In some cases GitLab should perform write actions to git repository, for example when it is needed to merge the merge request or edit a file with online editor. If something went wrong you can look into this file to find out what exactly happened. -``` -October 07, 2014 11:36: Failed to create satellite for Chesley Weimann III / project1817 -October 07, 2014 11:36: PID: 1872: git clone /Users/vsizov/gitlab-development-kit/gitlab/tmp/tests/repositories/conrad6841/gitlabhq.git /Users/vsizov/gitlab-development-kit/gitlab/tmp/tests/gitlab-satellites/conrad6841/gitlabhq -October 07, 2014 11:36: PID: 1872: -> fatal: repository '/Users/vsizov/gitlab-development-kit/gitlab/tmp/tests/repositories/conrad6841/gitlabhq.git' does not exist -``` - #### sidekiq.log This file lives in `/var/log/gitlab/gitlab-rails/sidekiq.log` for omnibus package or in `/home/git/gitlab/log/sidekiq.log` for installations from the source. @@ -99,4 +89,4 @@ W, [2015-02-13T07:16:01.313000 #9094] WARN -- : Unicorn::WorkerKiller send SIGQ I, [2015-02-13T07:16:01.530733 #9047] INFO -- : reaped # worker=1 I, [2015-02-13T07:16:01.534501 #13379] INFO -- : worker=1 spawned pid=13379 I, [2015-02-13T07:16:01.534848 #13379] INFO -- : worker=1 ready -``` \ No newline at end of file +``` diff --git a/doc/raketasks/maintenance.md b/doc/raketasks/maintenance.md index 69171cd1765..d9dce2af480 100644 --- a/doc/raketasks/maintenance.md +++ b/doc/raketasks/maintenance.md @@ -105,24 +105,11 @@ Log directory writable? ... yes Tmp directory writable? ... yes Init script exists? ... yes Init script up-to-date? ... yes -Projects have satellites? ... yes Redis version >= 2.0.0? ... yes Checking GitLab ... Finished ``` -## (Re-)Create satellite repositories - -This will create satellite repositories for all your projects. - -If necessary, remove the `repo_satellites` directory and rerun the commands below. - -``` -sudo -u git -H mkdir -p /home/git/gitlab-satellites -sudo -u git -H bundle exec rake gitlab:satellites:create RAILS_ENV=production -sudo chmod u+rwx,g=rx,o-rwx /home/git/gitlab-satellites -``` - ## Rebuild authorized_keys file In some case it is necessary to rebuild the `authorized_keys` file. diff --git a/features/steps/project/forked_merge_requests.rb b/features/steps/project/forked_merge_requests.rb index 58c16d59d05..3e6beb20e78 100644 --- a/features/steps/project/forked_merge_requests.rb +++ b/features/steps/project/forked_merge_requests.rb @@ -9,7 +9,6 @@ class Spinach::Features::ProjectForkedMergeRequests < Spinach::FeatureSteps @project = Project.find_by(name: "Shop") @project ||= create(:project, name: "Shop") @project.team << [@user, :reporter] - @project.ensure_satellite_exists end step 'I have a project forked off of "Shop" called "Forked Shop"' do diff --git a/features/steps/project/merge_requests.rb b/features/steps/project/merge_requests.rb index a1a26abd8ca..f2198f58c13 100644 --- a/features/steps/project/merge_requests.rb +++ b/features/steps/project/merge_requests.rb @@ -198,15 +198,10 @@ class Spinach::Features::ProjectMergeRequests < Spinach::FeatureSteps end step 'merge request "Bug NS-05" is mergeable' do - merge_request.project.satellite.create merge_request.mark_as_mergeable end step 'I accept this merge request' do - Gitlab::Satellite::MergeAction.any_instance.stub( - merge!: true, - ) - page.within '.mr-state-widget' do click_button "Accept Merge Request" end diff --git a/lib/gitlab.rb b/lib/gitlab.rb deleted file mode 100644 index 5fc1862c3e9..00000000000 --- a/lib/gitlab.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'gitlab/git' - -module Gitlab - autoload :Satellite, 'gitlab/satellite/satellite' -end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 172d4902add..14ee4701e7b 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -217,20 +217,6 @@ module Gitlab FileUtils.mv(full_path(old_name), full_path(new_name)) end - # Remove GitLab Satellites for provided path (namespace or repo dir) - # - # Ex. - # rm_satellites("gitlab") - # - # rm_satellites("gitlab/gitlab-ci.git") - # - def rm_satellites(path) - raise ArgumentError.new("Path can't be blank") if path.blank? - - satellites_path = File.join(Gitlab.config.satellites.path, path) - FileUtils.rm_r(satellites_path, force: true) - end - def url_to_repo(path) Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git" end diff --git a/lib/gitlab/satellite/action.rb b/lib/gitlab/satellite/action.rb deleted file mode 100644 index 489070f1a3f..00000000000 --- a/lib/gitlab/satellite/action.rb +++ /dev/null @@ -1,58 +0,0 @@ -module Gitlab - module Satellite - class Action - DEFAULT_OPTIONS = { git_timeout: Gitlab.config.satellites.timeout.seconds } - - attr_accessor :options, :project, :user - - def initialize(user, project, options = {}) - @options = DEFAULT_OPTIONS.merge(options) - @project = project - @user = user - end - - protected - - # * Sets a 30s timeout for Git - # * Locks the satellite repo - # * Yields the prepared satellite repo - def in_locked_and_timed_satellite - Gitlab::ShellEnv.set_env(user) - - Grit::Git.with_timeout(options[:git_timeout]) do - project.satellite.lock do - return yield project.satellite.repo - end - end - rescue Errno::ENOMEM => ex - return handle_exception(ex) - rescue Grit::Git::GitTimeout => ex - return handle_exception(ex) - ensure - Gitlab::ShellEnv.reset_env - end - - # * Recreates the satellite - # * Sets up Git variables for the user - # - # Note: use this within #in_locked_and_timed_satellite - def prepare_satellite!(repo) - project.satellite.clear_and_update! - - if user - repo.config['user.name'] = user.name - repo.config['user.email'] = user.email - end - end - - def default_options(options = {}) - { raise: true, timeout: true }.merge(options) - end - - def handle_exception(exception) - Gitlab::GitLogger.error(exception.message) - false - end - end - end -end diff --git a/lib/gitlab/satellite/compare_action.rb b/lib/gitlab/satellite/compare_action.rb deleted file mode 100644 index 46c98a8f4ca..00000000000 --- a/lib/gitlab/satellite/compare_action.rb +++ /dev/null @@ -1,44 +0,0 @@ -module Gitlab - module Satellite - class BranchesWithoutParent < StandardError; end - - class CompareAction < Action - def initialize(user, target_project, target_branch, source_project, source_branch) - super user, target_project - - @target_project, @target_branch = target_project, target_branch - @source_project, @source_branch = source_project, source_branch - end - - # Compare 2 repositories and return Gitlab::CompareResult object - def result - in_locked_and_timed_satellite do |target_repo| - prepare_satellite!(target_repo) - update_satellite_source_and_target!(target_repo) - - Gitlab::CompareResult.new(compare(target_repo)) - end - rescue Grit::Git::CommandFailed => ex - raise BranchesWithoutParent - end - - private - - # Assumes a satellite exists that is a fresh clone of the projects repo, prepares satellite for diffs - def update_satellite_source_and_target!(target_repo) - target_repo.remote_add('source', @source_project.repository.path_to_repo) - target_repo.remote_fetch('source') - rescue Grit::Git::CommandFailed => ex - handle_exception(ex) - end - - def compare(repo) - @compare ||= Gitlab::Git::Compare.new( - Gitlab::Git::Repository.new(repo.path), - "origin/#{@target_branch}", - "source/#{@source_branch}" - ) - end - end - end -end diff --git a/lib/gitlab/satellite/logger.rb b/lib/gitlab/satellite/logger.rb deleted file mode 100644 index 6f3f8255aca..00000000000 --- a/lib/gitlab/satellite/logger.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Gitlab - module Satellite - class Logger < Gitlab::Logger - def self.file_name - 'satellites.log' - end - - def format_message(severity, timestamp, progname, msg) - "#{timestamp.to_s(:long)}: #{msg}\n" - end - end - end -end diff --git a/lib/gitlab/satellite/merge_action.rb b/lib/gitlab/satellite/merge_action.rb deleted file mode 100644 index 1f2e5f82dd5..00000000000 --- a/lib/gitlab/satellite/merge_action.rb +++ /dev/null @@ -1,146 +0,0 @@ -module Gitlab - module Satellite - # GitLab server-side merge - class MergeAction < Action - attr_accessor :merge_request - - def initialize(user, merge_request) - super user, merge_request.target_project - @merge_request = merge_request - end - - # Checks if a merge request can be executed without user interaction - def can_be_merged? - in_locked_and_timed_satellite do |merge_repo| - prepare_satellite!(merge_repo) - merge_in_satellite!(merge_repo) - end - end - - # Merges the source branch into the target branch in the satellite and - # pushes it back to the repository. - # It also removes the source branch if requested in the merge request (and this is permitted by the merge request). - # - # Returns false if the merge produced conflicts - # Returns false if pushing from the satellite to the repository failed or was rejected - # Returns true otherwise - def merge!(merge_commit_message = nil) - in_locked_and_timed_satellite do |merge_repo| - prepare_satellite!(merge_repo) - if merge_in_satellite!(merge_repo, merge_commit_message) - # push merge back to bare repo - # will raise CommandFailed when push fails - merge_repo.git.push(default_options, :origin, merge_request.target_branch) - - # remove source branch - if merge_request.should_remove_source_branch && !project.root_ref?(merge_request.source_branch) && !merge_request.for_fork? - # will raise CommandFailed when push fails - merge_repo.git.push(default_options, :origin, ":#{merge_request.source_branch}") - end - # merge, push and branch removal successful - true - end - end - rescue Grit::Git::CommandFailed => ex - handle_exception(ex) - end - - def diff_in_satellite - in_locked_and_timed_satellite do |merge_repo| - prepare_satellite!(merge_repo) - update_satellite_source_and_target!(merge_repo) - - # Only show what is new in the source branch compared to the target branch, not the other way around. - # The line below with merge_base is equivalent to diff with three dots (git diff branch1...branch2) - # From the git documentation: "git diff A...B" is equivalent to "git diff $(git-merge-base A B) B" - common_commit = merge_repo.git.native(:merge_base, default_options, ["origin/#{merge_request.target_branch}", "source/#{merge_request.source_branch}"]).strip - merge_repo.git.native(:diff, default_options, common_commit, "source/#{merge_request.source_branch}") - end - rescue Grit::Git::CommandFailed => ex - handle_exception(ex) - end - - def diffs_between_satellite - in_locked_and_timed_satellite do |merge_repo| - prepare_satellite!(merge_repo) - update_satellite_source_and_target!(merge_repo) - if merge_request.for_fork? - repository = Gitlab::Git::Repository.new(merge_repo.path) - diffs = Gitlab::Git::Diff.between( - repository, - "source/#{merge_request.source_branch}", - "origin/#{merge_request.target_branch}" - ) - else - raise "Attempt to determine diffs between for a non forked merge request in satellite MergeRequest.id:[#{merge_request.id}]" - end - - return diffs - end - rescue Grit::Git::CommandFailed => ex - handle_exception(ex) - end - - # Get commit as an email patch - def format_patch - in_locked_and_timed_satellite do |merge_repo| - prepare_satellite!(merge_repo) - update_satellite_source_and_target!(merge_repo) - patch = merge_repo.git.format_patch(default_options({ stdout: true }), "origin/#{merge_request.target_branch}..source/#{merge_request.source_branch}") - end - rescue Grit::Git::CommandFailed => ex - handle_exception(ex) - end - - # Retrieve an array of commits between the source and the target - def commits_between - in_locked_and_timed_satellite do |merge_repo| - prepare_satellite!(merge_repo) - update_satellite_source_and_target!(merge_repo) - if merge_request.for_fork? - repository = Gitlab::Git::Repository.new(merge_repo.path) - commits = Gitlab::Git::Commit.between( - repository, - "origin/#{merge_request.target_branch}", - "source/#{merge_request.source_branch}" - ) - else - raise "Attempt to determine commits between for a non forked merge request in satellite MergeRequest.id:[#{merge_request.id}]" - end - - return commits - end - rescue Grit::Git::CommandFailed => ex - handle_exception(ex) - end - - private - # Merges the source_branch into the target_branch in the satellite. - # - # Note: it will clear out the satellite before doing anything - # - # Returns false if the merge produced conflicts - # Returns true otherwise - def merge_in_satellite!(repo, message = nil) - update_satellite_source_and_target!(repo) - - message ||= "Merge branch '#{merge_request.source_branch}' into '#{merge_request.target_branch}'" - - # merge the source branch into the satellite - # will raise CommandFailed when merge fails - repo.git.merge(default_options({ no_ff: true }), "-m#{message}", "source/#{merge_request.source_branch}") - rescue Grit::Git::CommandFailed => ex - handle_exception(ex) - end - - # Assumes a satellite exists that is a fresh clone of the projects repo, prepares satellite for merges, diffs etc - def update_satellite_source_and_target!(repo) - repo.remote_add('source', merge_request.source_project.repository.path_to_repo) - repo.remote_fetch('source') - repo.git.checkout(default_options({ b: true }), merge_request.target_branch, "origin/#{merge_request.target_branch}") - rescue Grit::Git::CommandFailed => ex - handle_exception(ex) - end - end - end -end diff --git a/lib/gitlab/satellite/satellite.rb b/lib/gitlab/satellite/satellite.rb deleted file mode 100644 index 398643d68de..00000000000 --- a/lib/gitlab/satellite/satellite.rb +++ /dev/null @@ -1,148 +0,0 @@ -module Gitlab - module Satellite - autoload :DeleteFileAction, 'gitlab/satellite/files/delete_file_action' - autoload :EditFileAction, 'gitlab/satellite/files/edit_file_action' - autoload :FileAction, 'gitlab/satellite/files/file_action' - autoload :NewFileAction, 'gitlab/satellite/files/new_file_action' - - class CheckoutFailed < StandardError; end - class CommitFailed < StandardError; end - class PushFailed < StandardError; end - - class Satellite - include Gitlab::Popen - - PARKING_BRANCH = "__parking_branch" - - attr_accessor :project - - def initialize(project) - @project = project - end - - def log(message) - Gitlab::Satellite::Logger.error(message) - end - - def clear_and_update! - project.ensure_satellite_exists - - @repo = nil - clear_working_dir! - delete_heads! - remove_remotes! - update_from_source! - end - - def create - output, status = popen(%W(git clone -- #{project.repository.path_to_repo} #{path}), - Gitlab.config.satellites.path) - - log("PID: #{project.id}: git clone #{project.repository.path_to_repo} #{path}") - log("PID: #{project.id}: -> #{output}") - - if status.zero? - true - else - log("Failed to create satellite for #{project.name_with_namespace}") - false - end - end - - def exists? - File.exists? path - end - - # * Locks the satellite - # * Changes the current directory to the satellite's working dir - # * Yields - def lock - project.ensure_satellite_exists - - File.open(lock_file, "w+") do |f| - begin - f.flock File::LOCK_EX - yield - ensure - f.flock File::LOCK_UN - end - end - end - - def lock_file - create_locks_dir unless File.exists?(lock_files_dir) - File.join(lock_files_dir, "satellite_#{project.id}.lock") - end - - def path - File.join(Gitlab.config.satellites.path, project.path_with_namespace) - end - - def repo - project.ensure_satellite_exists - - @repo ||= Grit::Repo.new(path) - end - - def destroy - FileUtils.rm_rf(path) - end - - private - - # Clear the working directory - def clear_working_dir! - repo.git.reset(hard: true) - repo.git.clean(f: true, d: true, x: true) - end - - # Deletes all branches except the parking branch - # - # This ensures we have no name clashes or issues updating branches when - # working with the satellite. - def delete_heads! - heads = repo.heads.map(&:name) - - # update or create the parking branch - repo.git.checkout(default_options({ B: true }), PARKING_BRANCH) - - # remove the parking branch from the list of heads ... - heads.delete(PARKING_BRANCH) - # ... and delete all others - heads.each { |head| repo.git.branch(default_options({ D: true }), head) } - end - - # Deletes all remotes except origin - # - # This ensures we have no remote name clashes or issues updating branches when - # working with the satellite. - def remove_remotes! - remotes = repo.git.remote.split(' ') - remotes.delete('origin') - remotes.each { |name| repo.git.remote(default_options,'rm', name)} - end - - # Updates the satellite from bare repo - # - # Note: this will only update remote branches (i.e. origin/*) - def update_from_source! - repo.git.remote(default_options, 'set-url', :origin, project.repository.path_to_repo) - repo.git.fetch(default_options, :origin) - end - - def default_options(options = {}) - { raise: true, timeout: true }.merge(options) - end - - # Create directory for storing - # satellites lock files - def create_locks_dir - FileUtils.mkdir_p(lock_files_dir) - end - - def lock_files_dir - @lock_files_dir ||= File.join(Gitlab.config.satellites.path, "tmp") - end - end - end -end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index aed84226a2f..21f159e1bfe 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -25,7 +25,6 @@ namespace :gitlab do check_init_script_exists check_init_script_up_to_date check_projects_have_namespace - check_satellites_exist check_redis_version check_ruby_version check_git_version @@ -238,37 +237,6 @@ namespace :gitlab do end end - def check_satellites_exist - print "Projects have satellites? ... " - - unless Project.count > 0 - puts "can't check, you have no projects".magenta - return - end - puts "" - - Project.find_each(batch_size: 100) do |project| - print sanitized_message(project) - - if project.satellite.exists? - puts "yes".green - elsif project.empty_repo? - puts "can't create, repository is empty".magenta - else - puts "no".red - try_fixing_it( - sudo_gitlab("bundle exec rake gitlab:satellites:create RAILS_ENV=production"), - "If necessary, remove the tmp/repo_satellites directory ...", - "... and rerun the above command" - ) - for_more_information( - "doc/raketasks/maintenance.md " - ) - fix_and_rerun - end - end - end - def check_log_writable print "Log directory writable? ... " @@ -339,7 +307,6 @@ namespace :gitlab do check_repo_base_is_not_symlink check_repo_base_user_and_group check_repo_base_permissions - check_satellites_permissions check_repos_hooks_directory_is_link check_gitlab_shell_self_test @@ -417,29 +384,6 @@ namespace :gitlab do end end - def check_satellites_permissions - print "Satellites access is drwxr-x---? ... " - - satellites_path = Gitlab.config.satellites.path - unless File.exists?(satellites_path) - puts "can't check because of previous errors".magenta - return - end - - if File.stat(satellites_path).mode.to_s(8).ends_with?("0750") - puts "yes".green - else - puts "no".red - try_fixing_it( - "sudo chmod u+rwx,g=rx,o-rwx #{satellites_path}", - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - def check_repo_base_user_and_group gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group diff --git a/lib/tasks/gitlab/enable_automerge.rake b/lib/tasks/gitlab/enable_automerge.rake deleted file mode 100644 index 3dade9d75b8..00000000000 --- a/lib/tasks/gitlab/enable_automerge.rake +++ /dev/null @@ -1,39 +0,0 @@ -namespace :gitlab do - namespace :satellites do - desc "GitLab | Create satellite repos" - task create: :environment do - create_satellites - end - end - - def create_satellites - warn_user_is_not_gitlab - - print "Creating satellites for ..." - unless Project.count > 0 - puts "skipping, because you have no projects".magenta - return - end - puts "" - - Project.find_each(batch_size: 100) do |project| - print "#{project.name_with_namespace.yellow} ... " - - unless project.repo_exists? - puts "skipping, because the repo is empty".magenta - next - end - - if project.satellite.exists? - puts "exists already".green - else - print "\n... " - if project.satellite.create - puts "created".green - else - puts "error".red - end - end - end - end -end diff --git a/spec/lib/gitlab/satellite/action_spec.rb b/spec/lib/gitlab/satellite/action_spec.rb deleted file mode 100644 index 0a93676edc3..00000000000 --- a/spec/lib/gitlab/satellite/action_spec.rb +++ /dev/null @@ -1,116 +0,0 @@ -require 'spec_helper' - -describe 'Gitlab::Satellite::Action' do - let(:project) { create(:project) } - let(:user) { create(:user) } - - describe '#prepare_satellite!' do - it 'should be able to fetch timeout from conf' do - expect(Gitlab::Satellite::Action::DEFAULT_OPTIONS[:git_timeout]).to eq(30.seconds) - end - - it 'create a repository with a parking branch and one remote: origin' do - repo = project.satellite.repo - - #now lets dirty it up - - starting_remote_count = repo.git.list_remotes.size - expect(starting_remote_count).to be >= 1 - #kind of hookey way to add a second remote - origin_uri = repo.git.remote({ v: true }).split(" ")[1] - - repo.git.remote({ raise: true }, 'add', 'another-remote', origin_uri) - repo.git.branch({ raise: true }, 'a-new-branch') - - expect(repo.heads.size).to be > (starting_remote_count) - expect(repo.git.remote().split(" ").size).to be > (starting_remote_count) - - repo.git.config({}, "user.name", "#{user.name} -- foo") - repo.git.config({}, "user.email", "#{user.email} -- foo") - expect(repo.config['user.name']).to eq("#{user.name} -- foo") - expect(repo.config['user.email']).to eq("#{user.email} -- foo") - - - #These must happen in the context of the satellite directory... - satellite_action = Gitlab::Satellite::Action.new(user, project) - project.satellite.lock do - #Now clean it up, use send to get around prepare_satellite! being protected - satellite_action.send(:prepare_satellite!, repo) - end - - #verify it's clean - heads = repo.heads.map(&:name) - expect(heads.size).to eq(1) - expect(heads.include?(Gitlab::Satellite::Satellite::PARKING_BRANCH)).to eq(true) - remotes = repo.git.remote().split(' ') - expect(remotes.size).to eq(1) - expect(remotes.include?('origin')).to eq(true) - expect(repo.config['user.name']).to eq(user.name) - expect(repo.config['user.email']).to eq(user.email) - end - end - - describe '#in_locked_and_timed_satellite' do - - it 'should make use of a lockfile' do - repo = project.satellite.repo - called = false - - #set assumptions - FileUtils.rm_f(project.satellite.lock_file) - - expect(File.exists?(project.satellite.lock_file)).to be_falsey - - satellite_action = Gitlab::Satellite::Action.new(user, project) - satellite_action.send(:in_locked_and_timed_satellite) do |sat_repo| - expect(repo).to eq(sat_repo) - expect(File.exists? project.satellite.lock_file).to be_truthy - called = true - end - - expect(called).to be_truthy - - end - - it 'should be able to use the satellite after locking' do - repo = project.satellite.repo - called = false - - # Set base assumptions - if File.exists? project.satellite.lock_file - expect(FileLockStatusChecker.new(project.satellite.lock_file).flocked?).to be_falsey - end - - satellite_action = Gitlab::Satellite::Action.new(user, project) - satellite_action.send(:in_locked_and_timed_satellite) do |sat_repo| - called = true - expect(repo).to eq(sat_repo) - expect(File.exists? project.satellite.lock_file).to be_truthy - expect(FileLockStatusChecker.new(project.satellite.lock_file).flocked?).to be_truthy - end - - expect(called).to be_truthy - expect(FileLockStatusChecker.new(project.satellite.lock_file).flocked?).to be_falsey - - end - - class FileLockStatusChecker < File - def flocked?(&block) - status = flock LOCK_EX|LOCK_NB - case status - when false - return true - when 0 - begin - block ? block.call : false - ensure - flock LOCK_UN - end - else - raise SystemCallError, status - end - end - end - - end -end diff --git a/spec/lib/gitlab/satellite/merge_action_spec.rb b/spec/lib/gitlab/satellite/merge_action_spec.rb deleted file mode 100644 index 9b1c9a34e29..00000000000 --- a/spec/lib/gitlab/satellite/merge_action_spec.rb +++ /dev/null @@ -1,104 +0,0 @@ -require 'spec_helper' - -describe 'Gitlab::Satellite::MergeAction' do - include RepoHelpers - - let(:project) { create(:project, namespace: create(:group)) } - let(:fork_project) { create(:project, namespace: create(:group), forked_from_project: project) } - let(:merge_request) { create(:merge_request, source_project: project, target_project: project) } - let(:merge_request_fork) { create(:merge_request, source_project: fork_project, target_project: project) } - - let(:merge_request_with_conflict) { create(:merge_request, :conflict, source_project: project, target_project: project) } - let(:merge_request_fork_with_conflict) { create(:merge_request, :conflict, source_project: project, target_project: project) } - - describe '#commits_between' do - def verify_commits(commits, first_commit_sha, last_commit_sha) - commits.each { |commit| expect(commit.class).to eq(Gitlab::Git::Commit) } - expect(commits.first.id).to eq(first_commit_sha) - expect(commits.last.id).to eq(last_commit_sha) - end - - context 'on fork' do - it 'should get proper commits between' do - commits = Gitlab::Satellite::MergeAction.new(merge_request_fork.author, merge_request_fork).commits_between - verify_commits(commits, sample_compare.commits.first, sample_compare.commits.last) - end - end - - context 'between branches' do - it 'should raise exception -- not expected to be used by non forks' do - expect { Gitlab::Satellite::MergeAction.new(merge_request.author, merge_request).commits_between }.to raise_error(RuntimeError) - end - end - end - - describe '#format_patch' do - def verify_content(patch) - sample_compare.commits.each do |commit| - expect(patch.include?(commit)).to be_truthy - end - end - - context 'on fork' do - it 'should build a format patch' do - patch = Gitlab::Satellite::MergeAction.new(merge_request_fork.author, merge_request_fork).format_patch - verify_content(patch) - end - end - - context 'between branches' do - it 'should build a format patch' do - patch = Gitlab::Satellite::MergeAction.new(merge_request_fork.author, merge_request).format_patch - verify_content(patch) - end - end - end - - describe '#diffs_between_satellite tested against diff_in_satellite' do - def is_a_matching_diff(diff, diffs) - diff_count = diff.scan('diff --git').size - expect(diff_count).to be >= 1 - expect(diffs.size).to eq(diff_count) - diffs.each do |a_diff| - expect(a_diff.class).to eq(Gitlab::Git::Diff) - expect(diff.include? a_diff.diff).to be_truthy - end - end - - context 'on fork' do - it 'should get proper diffs' do - diffs = Gitlab::Satellite::MergeAction.new(merge_request_fork.author, merge_request_fork).diffs_between_satellite - diff = Gitlab::Satellite::MergeAction.new(merge_request.author, merge_request_fork).diff_in_satellite - is_a_matching_diff(diff, diffs) - end - end - - context 'between branches' do - it 'should get proper diffs' do - expect{ Gitlab::Satellite::MergeAction.new(merge_request.author, merge_request).diffs_between_satellite }.to raise_error(RuntimeError) - end - end - end - - describe '#can_be_merged?' do - context 'on fork' do - it do - expect(Gitlab::Satellite::MergeAction.new(merge_request_fork.author, merge_request_fork).can_be_merged?).to be_truthy - end - - it do - expect(Gitlab::Satellite::MergeAction.new(merge_request_fork_with_conflict.author, merge_request_fork_with_conflict).can_be_merged?).to be_falsey - end - end - - context 'between branches' do - it do - expect(Gitlab::Satellite::MergeAction.new(merge_request.author, merge_request).can_be_merged?).to be_truthy - end - - it do - expect(Gitlab::Satellite::MergeAction.new(merge_request_with_conflict.author, merge_request_with_conflict).can_be_merged?).to be_falsey - end - end - end -end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 63091e913ff..721ae88c120 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -90,7 +90,6 @@ describe Project do describe 'Respond to' do it { is_expected.to respond_to(:url_to_repo) } it { is_expected.to respond_to(:repo_exists?) } - it { is_expected.to respond_to(:satellite) } it { is_expected.to respond_to(:update_merge_requests) } it { is_expected.to respond_to(:execute_hooks) } it { is_expected.to respond_to(:name_with_namespace) } diff --git a/spec/requests/api/files_spec.rb b/spec/requests/api/files_spec.rb index 6c7860511e8..8cb8790c339 100644 --- a/spec/requests/api/files_spec.rb +++ b/spec/requests/api/files_spec.rb @@ -110,7 +110,7 @@ describe API::API, api: true do expect(response.status).to eq(400) end - it "should return a 400 if satellite fails to create file" do + it "should return a 400 if fails to create file" do allow_any_instance_of(Repository).to receive(:remove_file).and_return(false) delete api("/projects/#{project.id}/repository/files", user), valid_params