Merge branch 'feature/system_hook_push' into 'master'

Added System Hooks for push and tag_push

Implements `push` and `tag_push` events for System Hooks.

They are based on Webhooks Events but without any deprecated item. We don't send commits too.
We are implementing this, to be used by Geo repository syncing (gitlab-org/gitlab-ee#76).

These hook events will be sent only when the respective configuration flags are set to true.
UI changes to admin screen will be made in another MR.

See merge request !3744

CHANGELOG

@@ -87,6 +87,8 @@ v 8.7.0 (unreleased)
   - Use GitHub Issue/PR number as iid to keep references
   - Import GitHub labels
   - Import GitHub milestones
+  - Fix emoji catgories in the emoji picker
+  - Execute system web hooks on push to the project
 
 v 8.6.6
   - Expire the exists cache before deletion to ensure project dir actually exists (Stan Hu). !3413

app/models/hooks/project_hook.rb

@@ -21,8 +21,6 @@
 class ProjectHook < WebHook
   belongs_to :project
 
-  scope :push_hooks, -> { where(push_events: true) }
-  scope :tag_push_hooks, -> { where(tag_push_events: true) }
   scope :issue_hooks, -> { where(issues_events: true) }
   scope :note_hooks, -> { where(note_events: true) }
   scope :merge_request_hooks, -> { where(merge_requests_events: true) }

app/models/hooks/system_hook.rb

@@ -19,4 +19,7 @@
 #
 
 class SystemHook < WebHook
+  def async_execute(data, hook_name)
+    Sidekiq::Client.enqueue(SystemHookWorker, id, data, hook_name)
+  end
 end

app/models/hooks/web_hook.rb

@@ -30,6 +30,9 @@ class WebHook < ActiveRecord::Base
   default_value_for :build_events, false
   default_value_for :enable_ssl_verification, true
 
+  scope :push_hooks, -> { where(push_events: true) }
+  scope :tag_push_hooks, -> { where(tag_push_events: true) }
+
   # HTTParty timeout
   default_timeout Gitlab.config.gitlab.webhook_timeout
 

app/models/project.rb

@@ -831,8 +831,8 @@ class Project < ActiveRecord::Base
     end
   end
 
-  def hook_attrs
-    {
+  def hook_attrs(backward: true)
+    attrs = {
       name: name,
       description: description,
       web_url: web_url,
@@ -843,12 +843,19 @@ class Project < ActiveRecord::Base
       visibility_level: visibility_level,
       path_with_namespace: path_with_namespace,
       default_branch: default_branch,
-      # Backward compatibility
-      homepage: web_url,
-      url: url_to_repo,
-      ssh_url: ssh_url_to_repo,
-      http_url: http_url_to_repo
     }
+
+    # Backward compatibility
+    if backward
+      attrs.merge!({
+                    homepage: web_url,
+                    url: url_to_repo,
+                    ssh_url: ssh_url_to_repo,
+                    http_url: http_url_to_repo
+                  })
+    end
+
+    attrs
   end
 
   # Reset events cache related to this project

app/services/git_push_service.rb

@@ -73,6 +73,7 @@ class GitPushService < BaseService
     @project.update_merge_requests(params[:oldrev], params[:newrev], params[:ref], current_user)
 
     EventCreateService.new.push(@project, current_user, build_push_data)
+    SystemHooksService.new.execute_hooks(build_push_data_system_hook.dup, :push_hooks)
     @project.execute_hooks(build_push_data.dup, :push_hooks)
     @project.execute_services(build_push_data.dup, :push_hooks)
     CreateCommitBuildsService.new.execute(@project, current_user, build_push_data)
@@ -138,6 +139,11 @@ class GitPushService < BaseService
       build(@project, current_user, params[:oldrev], params[:newrev], params[:ref], push_commits)
   end
 
+  def build_push_data_system_hook
+    @push_data_system ||= Gitlab::PushDataBuilder.
+      build(@project, current_user, params[:oldrev], params[:newrev], params[:ref], [])
+  end
+
   def push_to_existing_branch?
     # Return if this is not a push to a branch (e.g. new commits)
     Gitlab::Git.branch_ref?(params[:ref]) && !Gitlab::Git.blank_ref?(params[:oldrev])

app/services/git_tag_push_service.rb

@@ -1,16 +1,16 @@
-class GitTagPushService
-  attr_accessor :project, :user, :push_data
+class GitTagPushService < BaseService
+  attr_accessor :push_data
 
-  def execute(project, user, oldrev, newrev, ref)
+  def execute
     project.repository.before_push_tag
 
-    @project, @user = project, user
-    @push_data = build_push_data(oldrev, newrev, ref)
+    @push_data = build_push_data
 
-    EventCreateService.new.push(project, user, @push_data)
+    EventCreateService.new.push(project, current_user, @push_data)
+    SystemHooksService.new.execute_hooks(build_system_push_data.dup, :tag_push_hooks)
     project.execute_hooks(@push_data.dup, :tag_push_hooks)
     project.execute_services(@push_data.dup, :tag_push_hooks)
-    CreateCommitBuildsService.new.execute(project, @user, @push_data)
+    CreateCommitBuildsService.new.execute(project, current_user, @push_data)
     ProjectCacheWorker.perform_async(project.id)
 
     true
@@ -18,14 +18,14 @@ class GitTagPushService
 
   private
 
-  def build_push_data(oldrev, newrev, ref)
+  def build_push_data
     commits = []
     message = nil
 
-    if !Gitlab::Git.blank_ref?(newrev)
-      tag_name = Gitlab::Git.ref_name(ref)
+    if !Gitlab::Git.blank_ref?(params[:newrev])
+      tag_name = Gitlab::Git.ref_name(params[:ref])
       tag = project.repository.find_tag(tag_name)
-      if tag && tag.target == newrev
+      if tag && tag.target == params[:newrev]
         commit = project.commit(tag.target)
         commits = [commit].compact
         message = tag.message
@@ -33,6 +33,11 @@ class GitTagPushService
     end
 
     Gitlab::PushDataBuilder.
-      build(project, user, oldrev, newrev, ref, commits, message)
+      build(project, current_user, params[:oldrev], params[:newrev], params[:ref], commits, message)
+  end
+
+  def build_system_push_data
+    Gitlab::PushDataBuilder.
+      build(project, current_user, params[:oldrev], params[:newrev], params[:ref], [], '')
   end
 end

app/services/system_hooks_service.rb

@@ -3,17 +3,13 @@ class SystemHooksService
     execute_hooks(build_event_data(model, event))
   end
 
-  private
-
-  def execute_hooks(data)
-    SystemHook.all.each do |sh|
-      async_execute_hook(sh, data, 'system_hooks')
+  def execute_hooks(data, hooks_scope = :all)
+    SystemHook.send(hooks_scope).each do |hook|
+      hook.async_execute(data, 'system_hooks')
     end
   end
 
-  def async_execute_hook(hook, data, hook_name)
-    Sidekiq::Client.enqueue(SystemHookWorker, hook.id, data, hook_name)
-  end
+  private
 
   def build_event_data(model, event)
     data = {

app/workers/post_receive.rb

@@ -39,7 +39,7 @@ class PostReceive
       end
 
       if Gitlab::Git.tag_ref?(ref)
-        GitTagPushService.new.execute(post_received.project, @user, oldrev, newrev, ref)
+        GitTagPushService.new(post_received.project, @user, oldrev: oldrev, newrev: newrev, ref: ref).execute
       elsif Gitlab::Git.branch_ref?(ref)
         GitPushService.new(post_received.project, @user, oldrev: oldrev, newrev: newrev, ref: ref).execute
       end
@@ -47,7 +47,7 @@ class PostReceive
   end
 
   private
-  
+
   def log(message)
     Gitlab::GitLogger.error("POST-RECEIVE: #{message}")
   end

doc/system_hooks/system_hooks.md

@@ -4,6 +4,12 @@ Your GitLab instance can perform HTTP POST requests on the following events: `pr
 
 System hooks can be used, e.g. for logging or changing information in a LDAP server.
 
+> **Note:**
+>
+> We follow the same structure from Webhooks for Push and Tag events, but we never display commits.
+>
+> Same deprecations from Webhooks are valid here.
+
 ## Hooks request example
 
 **Request header**:
@@ -240,3 +246,110 @@ X-Gitlab-Event: System Hook
        "user_id": 41
 }
 ```
+
+## Push events
+
+Triggered when you push to the repository except when pushing tags.
+
+**Request header**:
+
+```
+X-Gitlab-Event: System Hook
+```
+
+**Request body:**
+
+```json
+{
+  "event_name": "push",
+  "before": "95790bf891e76fee5e1747ab589903a6a1f80f22",
+  "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
+  "ref": "refs/heads/master",
+  "checkout_sha": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
+  "user_id": 4,
+  "user_name": "John Smith",
+  "user_email": "john@example.com",
+  "user_avatar": "https://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=8://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=80",
+  "project_id": 15,
+  "project":{
+    "name":"Diaspora",
+    "description":"",
+    "web_url":"http://example.com/mike/diaspora",
+    "avatar_url":null,
+    "git_ssh_url":"git@example.com:mike/diaspora.git",
+    "git_http_url":"http://example.com/mike/diaspora.git",
+    "namespace":"Mike",
+    "visibility_level":0,
+    "path_with_namespace":"mike/diaspora",
+    "default_branch":"master",
+    "homepage":"http://example.com/mike/diaspora",
+    "url":"git@example.com:mike/diaspora.git",
+    "ssh_url":"git@example.com:mike/diaspora.git",
+    "http_url":"http://example.com/mike/diaspora.git"
+  },
+  "repository":{
+    "name": "Diaspora",
+    "url": "git@example.com:mike/diaspora.git",
+    "description": "",
+    "homepage": "http://example.com/mike/diaspora",
+    "git_http_url":"http://example.com/mike/diaspora.git",
+    "git_ssh_url":"git@example.com:mike/diaspora.git",
+    "visibility_level":0
+  },
+  "commits": [],
+  "total_commits_count": 0
+}
+```
+
+## Tag events
+
+Triggered when you create (or delete) tags to the repository.
+
+**Request header**:
+
+```
+X-Gitlab-Event: System Hook
+```
+
+**Request body:**
+
+```json
+{
+  "event_name": "tag_push",
+  "before": "0000000000000000000000000000000000000000",
+  "after": "82b3d5ae55f7080f1e6022629cdb57bfae7cccc7",
+  "ref": "refs/tags/v1.0.0",
+  "checkout_sha": "5937ac0a7beb003549fc5fd26fc247adbce4a52e",
+  "user_id": 1,
+  "user_name": "John Smith",
+  "user_avatar": "https://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=8://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=80",
+  "project_id": 1,
+  "project":{
+    "name":"Example",
+    "description":"",
+    "web_url":"http://example.com/jsmith/example",
+    "avatar_url":null,
+    "git_ssh_url":"git@example.com:jsmith/example.git",
+    "git_http_url":"http://example.com/jsmith/example.git",
+    "namespace":"Jsmith",
+    "visibility_level":0,
+    "path_with_namespace":"jsmith/example",
+    "default_branch":"master",
+    "homepage":"http://example.com/jsmith/example",
+    "url":"git@example.com:jsmith/example.git",
+    "ssh_url":"git@example.com:jsmith/example.git",
+    "http_url":"http://example.com/jsmith/example.git"
+  },
+  "repository":{
+    "name": "Example",
+    "url": "ssh://git@example.com/jsmith/example.git",
+    "description": "",
+    "homepage": "http://example.com/jsmith/example",
+    "git_http_url":"http://example.com/jsmith/example.git",
+    "git_ssh_url":"git@example.com:jsmith/example.git",
+    "visibility_level":0
+  },
+  "commits": [],
+  "total_commits_count": 0
+}
+```

doc/web_hooks/web_hooks.md

@@ -41,6 +41,7 @@ X-Gitlab-Event: Push Hook
   "before": "95790bf891e76fee5e1747ab589903a6a1f80f22",
   "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
   "ref": "refs/heads/master",
+  "checkout_sha": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
   "user_id": 4,
   "user_name": "John Smith",
   "user_email": "john@example.com",
@@ -118,9 +119,10 @@ X-Gitlab-Event: Tag Push Hook
 ```json
 {
   "object_kind": "tag_push",
-  "ref": "refs/tags/v1.0.0",
   "before": "0000000000000000000000000000000000000000",
   "after": "82b3d5ae55f7080f1e6022629cdb57bfae7cccc7",
+  "ref": "refs/tags/v1.0.0",
+  "checkout_sha": "82b3d5ae55f7080f1e6022629cdb57bfae7cccc7",
   "user_id": 1,
   "user_name": "John Smith",
   "user_avatar": "https://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=8://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=80",

lib/gitlab/push_data_builder.rb

@@ -36,11 +36,12 @@ module Gitlab
           commit.hook_attrs(with_changed_files: true)
         end
 
-        type = Gitlab::Git.tag_ref?(ref) ? "tag_push" : "push"
+        type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push'
 
         # Hash to be passed as post_receive_data
         data = {
           object_kind: type,
+          event_name: type,
           before: oldrev,
           after: newrev,
           ref: ref,

spec/lib/gitlab/push_data_builder_spec.rb

@@ -14,11 +14,11 @@ describe Gitlab::PushDataBuilder, lib: true do
     it { expect(data[:ref]).to eq('refs/heads/master') }
     it { expect(data[:commits].size).to eq(3) }
     it { expect(data[:total_commits_count]).to eq(3) }
-    it { expect(data[:commits].first[:added]).to eq(["gitlab-grack"]) }
-    it { expect(data[:commits].first[:modified]).to eq([".gitmodules"]) }
+    it { expect(data[:commits].first[:added]).to eq(['gitlab-grack']) }
+    it { expect(data[:commits].first[:modified]).to eq(['.gitmodules']) }
     it { expect(data[:commits].first[:removed]).to eq([]) }
 
-    include_examples 'project hook data'
+    include_examples 'project hook data with deprecateds'
     include_examples 'deprecated repository hook data'
   end
 
@@ -34,9 +34,18 @@ describe Gitlab::PushDataBuilder, lib: true do
     it { expect(data[:checkout_sha]).to eq('5937ac0a7beb003549fc5fd26fc247adbce4a52e') }
     it { expect(data[:after]).to eq('8a2a6eb295bb170b34c24c76c49ed0e9b2eaf34b') }
     it { expect(data[:ref]).to eq('refs/tags/v1.1.0') }
+    it { expect(data[:user_id]).to eq(user.id) }
+    it { expect(data[:user_name]).to eq(user.name) }
+    it { expect(data[:user_email]).to eq(user.email) }
+    it { expect(data[:user_avatar]).to eq(user.avatar_url) }
+    it { expect(data[:project_id]).to eq(project.id) }
+    it { expect(data[:project]).to be_a(Hash) }
     it { expect(data[:commits]).to be_empty }
     it { expect(data[:total_commits_count]).to be_zero }
 
+    include_examples 'project hook data with deprecateds'
+    include_examples 'deprecated repository hook data'
+
     it 'does not raise an error when given nil commits' do
       expect { described_class.build(spy, spy, spy, spy, spy, nil) }.
         not_to raise_error

spec/services/git_tag_push_service_spec.rb

@@ -5,19 +5,17 @@ describe GitTagPushService, services: true do
 
   let(:user) { create :user }
   let(:project) { create :project }
-  let(:service) { GitTagPushService.new }
+  let(:service) { GitTagPushService.new(project, user, oldrev: oldrev, newrev: newrev, ref: ref) }
 
-  before do
-    @oldrev = Gitlab::Git::BLANK_SHA
-    @newrev = "8a2a6eb295bb170b34c24c76c49ed0e9b2eaf34b" # gitlab-test: git rev-parse refs/tags/v1.1.0
-    @ref = 'refs/tags/v1.1.0'
-  end
+  let(:oldrev) { Gitlab::Git::BLANK_SHA }
+  let(:newrev) { "8a2a6eb295bb170b34c24c76c49ed0e9b2eaf34b" } # gitlab-test: git rev-parse refs/tags/v1.1.0
+  let(:ref) { 'refs/tags/v1.1.0' }
 
   describe "Git Tag Push Data" do
     before do
-      service.execute(project, user, @oldrev, @newrev, @ref)
+      service.execute
       @push_data = service.push_data
-      @tag_name = Gitlab::Git.ref_name(@ref)
+      @tag_name = Gitlab::Git.ref_name(ref)
       @tag = project.repository.find_tag(@tag_name)
       @commit = project.commit(@tag.target)
     end
@@ -25,9 +23,9 @@ describe GitTagPushService, services: true do
     subject { @push_data }
 
     it { is_expected.to include(object_kind: 'tag_push') }
-    it { is_expected.to include(ref: @ref) }
-    it { is_expected.to include(before: @oldrev) }
-    it { is_expected.to include(after: @newrev) }
+    it { is_expected.to include(ref: ref) }
+    it { is_expected.to include(before: oldrev) }
+    it { is_expected.to include(after: newrev) }
     it { is_expected.to include(message: @tag.message) }
     it { is_expected.to include(user_id: user.id) }
     it { is_expected.to include(user_name: user.name) }
@@ -80,9 +78,11 @@ describe GitTagPushService, services: true do
 
   describe "Webhooks" do
     context "execute webhooks" do
+      let(:service) { GitTagPushService.new(project, user, oldrev: 'oldrev', newrev: 'newrev', ref: 'refs/tags/v1.0.0') }
+
       it "when pushing tags" do
         expect(project).to receive(:execute_hooks)
-        service.execute(project, user, 'oldrev', 'newrev', 'refs/tags/v1.0.0')
+        service.execute
       end
     end
   end

spec/support/project_hook_data_shared_example.rb

@@ -1,4 +1,4 @@
-RSpec.shared_examples 'project hook data' do |project_key: :project|
+RSpec.shared_examples 'project hook data with deprecateds' do |project_key: :project|
   it 'contains project data' do
     expect(data[project_key][:name]).to eq(project.name)
     expect(data[project_key][:description]).to eq(project.description)
@@ -17,6 +17,21 @@ RSpec.shared_examples 'project hook data' do |project_key: :project|
   end
 end
 
+RSpec.shared_examples 'project hook data' do |project_key: :project|
+  it 'contains project data' do
+    expect(data[project_key][:name]).to eq(project.name)
+    expect(data[project_key][:description]).to eq(project.description)
+    expect(data[project_key][:web_url]).to eq(project.web_url)
+    expect(data[project_key][:avatar_url]).to eq(project.avatar_url)
+    expect(data[project_key][:git_http_url]).to eq(project.http_url_to_repo)
+    expect(data[project_key][:git_ssh_url]).to eq(project.ssh_url_to_repo)
+    expect(data[project_key][:namespace]).to eq(project.namespace.name)
+    expect(data[project_key][:visibility_level]).to eq(project.visibility_level)
+    expect(data[project_key][:path_with_namespace]).to eq(project.path_with_namespace)
+    expect(data[project_key][:default_branch]).to eq(project.default_branch)
+  end
+end
+
 RSpec.shared_examples 'deprecated repository hook data' do |project_key: :project|
   it 'contains deprecated repository data' do
     expect(data[:repository][:name]).to eq(project.name)