Adds remote mirror table migration

app/controllers/projects/mirrors_controller.rb

@@ -2,8 +2,9 @@ class Projects::MirrorsController < Projects::ApplicationController
   include RepositorySettingsRedirect
 
   # Authorize
-  before_action :authorize_admin_mirror!
   before_action :remote_mirror, only: [:update]
+  before_action :check_mirror_available!
+  before_action :authorize_admin_project!
 
   layout "project_settings"
 
@@ -45,6 +46,10 @@ class Projects::MirrorsController < Projects::ApplicationController
     @remote_mirror = project.remote_mirrors.first_or_initialize
   end
 
+  def check_mirror_available!
+    Gitlab::CurrentSettings.current_application_settings.mirror_available || current_user&.admin?
+  end
+
   def mirror_params_attributes
     [
       remote_mirrors_attributes: %i[

app/controllers/projects/settings/repository_controller.rb

@@ -44,8 +44,6 @@ module Projects
       end
 
       def remote_mirror
-        return unless project.feature_available?(:repository_mirrors)
-
         @remote_mirror = project.remote_mirrors.first_or_initialize
       end
 

app/helpers/application_settings_helper.rb

@@ -250,7 +250,8 @@ module ApplicationSettingsHelper
       :version_check_enabled,
       :allow_local_requests_from_hooks_and_services,
       :enforce_terms,
-      :terms
+      :terms,
+      :mirror_available
     ]
   end
 end

app/models/application_setting.rb

@@ -334,7 +334,8 @@ class ApplicationSetting < ActiveRecord::Base
       gitaly_timeout_fast: 10,
       gitaly_timeout_medium: 30,
       gitaly_timeout_default: 55,
-      allow_local_requests_from_hooks_and_services: false
+      allow_local_requests_from_hooks_and_services: false,
+      mirror_available: true
     }
   end
 

app/models/project.rb

@@ -65,7 +65,7 @@ class Project < ActiveRecord::Base
 
   add_authentication_token_field :runners_token
 
-  before_validation :mark_remote_mirrors_for_removal
+  before_validation :mark_remote_mirrors_for_removal, if: -> { ActiveRecord::Base.connection.table_exists?(:remote_mirrors) }
 
   before_save :ensure_runners_token
 

app/models/remote_mirror.rb

@@ -86,9 +86,12 @@ class RemoteMirror < ActiveRecord::Base
     raw.update(options)
   end
 
+  def sync?
+    !enabled?
+  end
+
   def sync
-    return unless enabled?
+    return if sync?
-    return if Gitlab::Geo.secondary?
 
     if recently_scheduled?
       RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now)

app/models/repository.rb

@@ -854,7 +854,7 @@ class Repository
     add_remote(remote_name, url, mirror_refmap: refmap)
     fetch_remote(remote_name, forced: forced, prune: prune)
   ensure
-    remove_remote(remote_name) if tmp_remote_name
+    async_remove_remote(remote_name) if tmp_remote_name
   end
 
   def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false, prune: true)

app/policies/project_policy.rb

@@ -80,6 +80,11 @@ class ProjectPolicy < BasePolicy
     project.merge_requests_allowing_push_to_user(user).any?
   end
 
+  with_scope :global
+  condition(:mirror_available, score: 0) do
+    ::Gitlab::CurrentSettings.current_application_settings.mirror_available
+  end
+
   # We aren't checking `:read_issue` or `:read_merge_request` in this case
   # because it could be possible for a user to see an issuable-iid
   # (`:read_issue_iid` or `:read_merge_request_iid`) but then wouldn't be
@@ -246,6 +251,8 @@ class ProjectPolicy < BasePolicy
     enable :create_cluster
   end
 
+  rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror
+
   rule { archived }.policy do
     prevent :push_code
     prevent :push_to_delete_protected_branch

app/serializers/project_mirror_entity.rb

@@ -1,6 +1,4 @@
 class ProjectMirrorEntity < Grape::Entity
-  prepend ::EE::ProjectMirrorEntity
-
   expose :id
 
   expose :remote_mirrors_attributes do |project|

app/services/concerns/exclusive_lease_guard.rb

@@ -0,0 +1,52 @@
+#
+# Concern that helps with getting an exclusive lease for running a block
+# of code.
+#
+# `#try_obtain_lease` takes a block which will be run if it was able to
+# obtain  the lease. Implement `#lease_timeout` to configure the timeout
+# for the exclusive lease. Optionally override `#lease_key` to set the
+# lease key, it defaults to the class name with underscores.
+#
+module ExclusiveLeaseGuard
+  extend ActiveSupport::Concern
+
+  def try_obtain_lease
+    lease = exclusive_lease.try_obtain
+
+    unless lease
+      log_error('Cannot obtain an exclusive lease. There must be another instance already in execution.')
+      return
+    end
+
+    begin
+      yield lease
+    ensure
+      release_lease(lease)
+    end
+  end
+
+  def exclusive_lease
+    @lease ||= Gitlab::ExclusiveLease.new(lease_key, timeout: lease_timeout)
+  end
+
+  def lease_key
+    @lease_key ||= self.class.name.underscore
+  end
+
+  def lease_timeout
+    raise NotImplementedError,
+          "#{self.class.name} does not implement #{__method__}"
+  end
+
+  def release_lease(uuid)
+    Gitlab::ExclusiveLease.cancel(lease_key, uuid)
+  end
+
+  def renew_lease!
+    exclusive_lease.renew
+  end
+
+  def log_error(message, extra_args = {})
+    logger.error(message)
+  end
+end

app/views/admin/application_settings/_repository_mirrors_form.html.haml

@@ -0,0 +1,16 @@
+= form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f|
+  = form_errors(@application_setting)
+
+  %fieldset
+    .form-group
+      = f.label :mirror_available, 'Enable mirror configuration', class: 'control-label col-sm-2'
+      .col-sm-10
+        .checkbox
+          = f.label :mirror_available do
+            = f.check_box :mirror_available
+            Allow mirrors to be setup for projects
+        %span.help-block
+          If disabled, only admins will be able to setup mirrors in projects.
+          = link_to icon('question-circle'), help_page_path('workflow/repository_mirroring')
+
+  = f.submit 'Save changes', class: "btn btn-success"

app/views/admin/application_settings/show.html.haml

@@ -313,3 +313,14 @@
       = _('Allow requests to the local network from hooks and services.')
   .settings-content
     = render 'outbound'
+
+%section.settings.as-mirror.no-animate#js-mirror-settings{ class: ('expanded' if expanded) }
+  .settings-header
+    %h4
+      = _('Repository mirror settings')
+    %button.btn.js-settings-toggle{ type: 'button' }
+      = expanded ? 'Collapse' : 'Expand'
+    %p
+      = _('Configure push and pull mirrors.')
+  .settings-content
+    = render partial: 'repository_mirrors_form'

app/views/projects/mirrors/_instructions.html.haml

@@ -0,0 +1,10 @@
+.account-well.prepend-top-default.append-bottom-default
+  %ul
+    %li
+      The repository must be accessible over <code>http://</code>, <code>https://</code>, <code>ssh://</code> or <code>git://</code>.
+    %li
+      Include the username in the URL if required: <code>https://username@gitlab.company.com/group/project.git</code>.
+    %li
+      The update action will time out after 10 minutes. For big repositories, use a clone/push combination.
+    %li
+      The Git LFS objects will <strong>not</strong> be synced.

app/views/projects/mirrors/_show.html.haml

@@ -1,3 +1,3 @@
-- if can?(current_user, :admin_mirror, @project)
+- if can?(current_user, :admin_remote_mirror, @project)
-	= render 'projects/mirrors/push'
+  = render 'projects/mirrors/push'
 

app/views/projects/settings/repository/show.html.haml

@@ -2,6 +2,8 @@
 - page_title "Repository"
 - @content_class = "limit-container-width" unless fluid_layout
 
+= render "projects/mirrors/show"
+
 -# Protected branches & tags use a lot of nested partials.
 -# The shared parts of the views can be found in the `shared` directory.
 -# Those are used throughout the actual views. These `shared` views are then

app/workers/all_queues.yml

@@ -106,6 +106,7 @@
 - rebase
 - repository_fork
 - repository_import
+- repository_remove_remote
 - storage_migrator
 - system_hook_push
 - update_merge_requests

config/routes/project.rb

@@ -174,6 +174,12 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
         end
       end
 
+      resource :mirror, only: [:show, :update] do
+        member do
+          post :update_now
+        end
+      end
+
       resources :pipelines, only: [:index, :new, :create, :show] do
         collection do
           resource :pipelines_settings, path: 'settings', only: [:show, :update]

db/migrate/20180503131624_create_remote_mirrors.rb

@@ -0,0 +1,33 @@
+class CreateRemoteMirrors < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    return if table_exists?(:remote_mirrors)
+
+    create_table :remote_mirrors do |t|
+      t.references :project, index: true, foreign_key: { on_delete: :cascade }
+      t.string :url
+      t.boolean :enabled, default: true
+      t.string :update_status
+      t.datetime :last_update_at
+      t.datetime :last_successful_update_at
+      t.datetime :last_update_started_at
+      t.string :last_error
+      t.boolean :only_protected_branches, default: false, null: false
+      t.string :remote_name
+      t.text :encrypted_credentials
+      t.string :encrypted_credentials_iv
+      t.string :encrypted_credentials_salt
+
+      t.timestamps null: false
+    end
+  end
+
+  def down
+    drop_table(:remote_mirrors) if table_exists?(:remote_mirrors)
+  end
+end

db/migrate/20180503141722_add_remote_mirror_available_overridden_to_projects.rb

@@ -0,0 +1,15 @@
+class AddRemoteMirrorAvailableOverriddenToProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column(:projects, :remote_mirror_available_overridden, :boolean) unless column_exists?(:projects, :remote_mirror_available_overridden)
+  end
+
+  def down
+    remove_column(:projects, :remote_mirror_available_overridden) if column_exists?(:projects, :remote_mirror_available_overridden)
+  end
+end

db/migrate/20180503193542_add_indexes_to_remote_mirror.rb

@@ -0,0 +1,15 @@
+class AddIndexesToRemoteMirror < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :remote_mirrors, :last_successful_update_at unless index_exists?(:remote_mirrors, :last_successful_update_at)
+  end
+
+  def down
+    remove_index :remote_mirrors, :last_successful_update_at if index_exists? :remote_mirrors, :last_successful_update_at
+  end
+end

db/migrate/20180503193953_add_mirror_available_to_application_settings.rb

@@ -0,0 +1,15 @@
+class AddMirrorAvailableToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:application_settings, :mirror_available, :boolean, default: true, allow_null: false) unless column_exists?(:application_settings, :mirror_available)
+  end
+
+  def down
+    remove_column(:application_settings, :mirror_available) if column_exists?(:application_settings, :mirror_available)
+  end
+end

db/schema.rb

@@ -165,6 +165,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do
     t.boolean "pages_domain_verification_enabled", default: true, null: false
     t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false
     t.boolean "enforce_terms", default: false
+    t.boolean "mirror_available", default: true, null: false
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -314,10 +315,10 @@ ActiveRecord::Schema.define(version: 20180503200320) do
     t.integer "auto_canceled_by_id"
     t.boolean "retried"
     t.integer "stage_id"
-    t.boolean "protected"
-    t.integer "failure_reason"
     t.integer "artifacts_file_store"
     t.integer "artifacts_metadata_store"
+    t.boolean "protected"
+    t.integer "failure_reason"
   end
 
   add_index "ci_builds", ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree
@@ -365,13 +366,13 @@ ActiveRecord::Schema.define(version: 20180503200320) do
     t.integer "project_id", null: false
     t.integer "job_id", null: false
     t.integer "file_type", null: false
+    t.integer "file_store"
     t.integer "size", limit: 8
     t.datetime_with_timezone "created_at", null: false
     t.datetime_with_timezone "updated_at", null: false
     t.datetime_with_timezone "expire_at"
     t.string "file"
     t.binary "file_sha256"
-    t.integer "file_store"
   end
 
   add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree
@@ -1593,6 +1594,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do
     t.boolean "merge_requests_rebase_enabled", default: false, null: false
     t.integer "jobs_cache_index"
     t.boolean "pages_https_only", default: true
+    t.boolean "remote_mirror_available_overridden"
   end
 
   add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
@@ -1698,6 +1700,27 @@ ActiveRecord::Schema.define(version: 20180503200320) do
   add_index "releases", ["project_id", "tag"], name: "index_releases_on_project_id_and_tag", using: :btree
   add_index "releases", ["project_id"], name: "index_releases_on_project_id", using: :btree
 
+  create_table "remote_mirrors", force: :cascade do |t|
+    t.integer "project_id"
+    t.string "url"
+    t.boolean "enabled", default: true
+    t.string "update_status"
+    t.datetime "last_update_at"
+    t.datetime "last_successful_update_at"
+    t.datetime "last_update_started_at"
+    t.string "last_error"
+    t.boolean "only_protected_branches", default: false, null: false
+    t.string "remote_name"
+    t.text "encrypted_credentials"
+    t.string "encrypted_credentials_iv"
+    t.string "encrypted_credentials_salt"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "remote_mirrors", ["last_successful_update_at"], name: "index_remote_mirrors_on_last_successful_update_at", using: :btree
+  add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree
+
   create_table "routes", force: :cascade do |t|
     t.integer "source_id", null: false
     t.string "source_type", null: false
@@ -2233,6 +2256,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do
   add_foreign_key "protected_tags", "projects", name: "fk_8e4af87648", on_delete: :cascade
   add_foreign_key "push_event_payloads", "events", name: "fk_36c74129da", on_delete: :cascade
   add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade
+  add_foreign_key "remote_mirrors", "projects", on_delete: :cascade
   add_foreign_key "services", "projects", name: "fk_71cce407f9", on_delete: :cascade
   add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade
   add_foreign_key "subscriptions", "projects", on_delete: :cascade

doc/workflow/repository_mirroring.md

@@ -0,0 +1,111 @@
+# Repository mirroring
+
+Repository Mirroring is a way to mirror repositories from external sources.
+It can be used to mirror all branches, tags, and commits that you have
+in your repository.
+
+Your mirror at GitLab will be updated automatically. You can
+also manually trigger an update at most once every 5 minutes.
+
+## Overview
+
+Repository mirroring is very useful when, for some reason, you must use a
+project from another source.
+
+There are two kinds of repository mirroring features supported by GitLab:
+**push** and **pull**, the latter being only available in GitLab Enterprise Edition.
+The **push** method mirrors the repository in GitLab to another location.
+
+Once the mirror repository is updated, all new branches,
+tags, and commits will be visible in the project's activity feed.
+Users with at least [developer access][perms] to the project can also force an
+immediate update with the click of a button. This button will not be available if
+the mirror is already being updated or 5 minutes still haven't passed since its last update.
+
+A few things/limitations to consider:
+
+- The repository must be accessible over `http://`, `https://`, `ssh://` or `git://`.
+- If your HTTP repository is not publicly accessible, add authentication
+  information to the URL, like: `https://username@gitlab.company.com/group/project.git`.
+  In some cases, you might need to use a personal access token instead of a
+  password, e.g., you want to mirror to GitHub and have 2FA enabled.
+- The import will time out after 15 minutes. For repositories that take longer
+  use a clone/push combination.
+- The Git LFS objects will not be synced. You'll need to push/pull them
+  manually.
+
+## Use-case
+
+- You have old projects in another source that you don't use actively anymore,
+  but don't want to remove for archiving purposes. In that case, you can create
+  a push mirror so that your active GitLab repository can push its changes to the
+  old location.
+
+## Pushing to a remote repository **[STARTER]**
+
+>[Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/249) in
+GitLab Enterprise Edition 8.7. [Moved to GitLab Community Edition][ce-18715] in 10.8.
+
+For an existing project, you can set up push mirror from your project's
+**Settings ➔ Repository** and searching for the "Push to a remote repository"
+section. Check the "Remote mirror repository" box and fill in the Git URL of
+the repository to push to. Click **Save changes** for the changes to take
+effect.
+
+![Push settings](repository_mirroring/repository_mirroring_push_settings.png)
+
+When push mirroring is enabled, you are advised not to push commits directly
+to the mirrored repository to prevent the mirror diverging.
+All changes will end up in the mirrored repository whenever commits
+are pushed to GitLab, or when a [forced update](#forcing-an-update) is
+initiated.
+
+Pushes into GitLab are automatically pushed to the remote mirror at least once
+every 5 minutes after they are received or once every minute if **push only
+protected branches** is enabled.
+
+In case of a diverged branch, you will see an error indicated at the **Mirror
+repository** settings.
+
+![Diverged branch](
+repository_mirroring/repository_mirroring_diverged_branch_push.png)
+
+### Push only protected branches
+
+>[Introduced][ee-3350] in GitLab Enterprise Edition 10.3. [Moved to GitLab Community Edition][ce-18715] in 10.8.
+
+You can choose to only push your protected branches from GitLab to your remote repository.
+
+To use this option go to your project's repository settings page under push mirror.
+
+## Setting up a push mirror from GitLab to GitHub
+
+To set up a mirror from GitLab to GitHub, you need to follow these steps:
+
+1. Create a [GitHub personal access token](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/) with the "public_repo" box checked:
+    
+    ![edit personal access token GitHub](repository_mirroring/repository_mirroring_github_edit_personal_access_token.png)
+
+1. Fill in the "Git repository URL" with the personal access token replacing the password `https://GitHubUsername:GitHubPersonalAccessToken@github.com/group/project.git`:
+
+    ![push to remote repo](repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository.png)
+
+1. Save
+1. And either wait or trigger the "Update Now" button:
+
+    ![update now](repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository_update_now.png)
+    
+## Forcing an update
+
+While mirrors are scheduled to update automatically, you can always force an update
+by using the **Update now** button which is exposed in various places:
+
+- in the commits page
+- in the branches page
+- in the tags page
+- in the **Mirror repository** settings page
+
+[ee-3350]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3350
+[ce-18715]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18715
+[perms]: ../user/permissions.md
+

spec/models/project_spec.rb

@@ -1852,7 +1852,7 @@ describe Project do
     it { expect(project.gitea_import?).to be true }
   end
 
-    describe '#has_remote_mirror?' do
+  describe '#has_remote_mirror?' do
     let(:project) { create(:project, :remote_mirror, :import_started) }
     subject { project.has_remote_mirror? }
 

spec/models/remote_mirror_spec.rb

@@ -164,14 +164,6 @@ describe RemoteMirror do
       end
     end
 
-    context 'as a Geo secondary' do
-      it 'returns nil' do
-        allow(Gitlab::Geo).to receive(:secondary?).and_return(true)
-
-        expect(remote_mirror.sync).to be_nil
-      end
-    end
-
     context 'with remote mirroring enabled' do
       context 'with only protected branches enabled' do
         context 'when it did not update in the last minute' do

spec/models/repository_spec.rb

@@ -2370,6 +2370,11 @@ describe Repository do
     end
   end
 
+  def create_remote_branch(remote_name, branch_name, target)
+    rugged = repository.rugged
+    rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", target.id)
+  end
+
   describe '#ancestor?' do
     let(:commit) { repository.commit }
     let(:ancestor) { commit.parents.first }