Merge branch 'fix-email-threading' into 'master'

Fix broken email threading

The email threading support introduced in GitLab CE 6.9 is broken on several popular email clients (including Mail.app and Airmail on Mac OS X).

This MR makes the following changes to improve email threading compatibility:

* Subject of answers to an existing thread begins with `Re: ` (required by Mail.app)
* The recipient of every email in a thread is stable (required by Mail.app ; otherwise it groups emails by sender)
* Send a ‘In-Reply-To’ header along the ‘References’ header (for compatibility with the spec)

In order to do this, these commits:

* Change the `To:` field to `namespace/project` ; the actual receiver is now in the `Cc:` field.
* Introduce the `mail_new_thread` and `mail_answer_thread` methods ; they format the message correctly for threading, and can generate the `Message-ID` automatically from a model instance.
* Refactor the tests to shared behaviors for email threading.

We've been using these patches at @capitainetrain for a few months now ; I just ported them to work nicely with the recent threading commits.

app/mailers/emails/groups.rb

@@ -4,7 +4,7 @@ module Emails
       @membership = UsersGroup.find(user_group_id)
       @group = @membership.group
       @target_url = group_url(@group)
-      mail(to: @membership.user.email,
+      mail(cc: @membership.user.email,
            subject: subject("Access to group was granted"))
     end
   end

app/mailers/emails/issues.rb

@@ -4,10 +4,10 @@ module Emails
       @issue = Issue.find(issue_id)
       @project = @issue.project
       @target_url = project_issue_url(@project, @issue)
-      set_message_id("issue_#{issue_id}")
+      mail_new_thread(@issue,
-      mail(from: sender(@issue.author_id),
+                      from: sender(@issue.author_id),
-           to: recipient(recipient_id),
+                      cc: recipient(recipient_id),
-           subject: subject("#{@issue.title} (##{@issue.iid})"))
+                      subject: subject("#{@issue.title} (##{@issue.iid})"))
     end
 
     def reassigned_issue_email(recipient_id, issue_id, previous_assignee_id, updated_by_user_id)
@@ -15,10 +15,10 @@ module Emails
       @previous_assignee = User.find_by(id: previous_assignee_id) if previous_assignee_id
       @project = @issue.project
       @target_url = project_issue_url(@project, @issue)
-      set_reference("issue_#{issue_id}")
+      mail_answer_thread(@issue,
-      mail(from: sender(updated_by_user_id),
+                         from: sender(updated_by_user_id),
-           to: recipient(recipient_id),
+                         cc: recipient(recipient_id),
-           subject: subject("#{@issue.title} (##{@issue.iid})"))
+                         subject: subject("#{@issue.title} (##{@issue.iid})"))
     end
 
     def closed_issue_email(recipient_id, issue_id, updated_by_user_id)
@@ -26,10 +26,10 @@ module Emails
       @project = @issue.project
       @updated_by = User.find updated_by_user_id
       @target_url = project_issue_url(@project, @issue)
-      set_reference("issue_#{issue_id}")
+      mail_answer_thread(@issue,
-      mail(from: sender(updated_by_user_id),
+                         from: sender(updated_by_user_id),
-           to: recipient(recipient_id),
+                         cc: recipient(recipient_id),
-           subject: subject("#{@issue.title} (##{@issue.iid})"))
+                         subject: subject("#{@issue.title} (##{@issue.iid})"))
     end
 
     def issue_status_changed_email(recipient_id, issue_id, status, updated_by_user_id)
@@ -38,10 +38,10 @@ module Emails
       @project = @issue.project
       @updated_by = User.find updated_by_user_id
       @target_url = project_issue_url(@project, @issue)
-      set_reference("issue_#{issue_id}")
+      mail_answer_thread(@issue,
-      mail(from: sender(updated_by_user_id),
+                         from: sender(updated_by_user_id),
-           to: recipient(recipient_id),
+                         cc: recipient(recipient_id),
-           subject: subject("#{@issue.title} (##{@issue.iid})"))
+                         subject: subject("#{@issue.title} (##{@issue.iid})"))
     end
   end
 end

app/mailers/emails/merge_requests.rb

@@ -4,10 +4,10 @@ module Emails
       @merge_request = MergeRequest.find(merge_request_id)
       @project = @merge_request.project
       @target_url = project_merge_request_url(@project, @merge_request)
-      set_message_id("merge_request_#{merge_request_id}")
+      mail_new_thread(@merge_request,
-      mail(from: sender(@merge_request.author_id),
+                      from: sender(@merge_request.author_id),
-           to: recipient(recipient_id),
+                      cc: recipient(recipient_id),
-           subject: subject("#{@merge_request.title} (##{@merge_request.iid})"))
+                      subject: subject("#{@merge_request.title} (##{@merge_request.iid})"))
     end
 
     def reassigned_merge_request_email(recipient_id, merge_request_id, previous_assignee_id, updated_by_user_id)
@@ -15,10 +15,10 @@ module Emails
       @previous_assignee = User.find_by(id: previous_assignee_id) if previous_assignee_id
       @project = @merge_request.project
       @target_url = project_merge_request_url(@project, @merge_request)
-      set_reference("merge_request_#{merge_request_id}")
+      mail_answer_thread(@merge_request,
-      mail(from: sender(updated_by_user_id),
+                         from: sender(updated_by_user_id),
-           to: recipient(recipient_id),
+                         cc: recipient(recipient_id),
-           subject: subject("#{@merge_request.title} (##{@merge_request.iid})"))
+                         subject: subject("#{@merge_request.title} (##{@merge_request.iid})"))
     end
 
     def closed_merge_request_email(recipient_id, merge_request_id, updated_by_user_id)
@@ -26,20 +26,20 @@ module Emails
       @updated_by = User.find updated_by_user_id
       @project = @merge_request.project
       @target_url = project_merge_request_url(@project, @merge_request)
-      set_reference("merge_request_#{merge_request_id}")
+      mail_answer_thread(@merge_request,
-      mail(from: sender(updated_by_user_id),
+                         from: sender(updated_by_user_id),
-           to: recipient(recipient_id),
+                         cc: recipient(recipient_id),
-           subject: subject("#{@merge_request.title} (##{@merge_request.iid})"))
+                         subject: subject("#{@merge_request.title} (##{@merge_request.iid})"))
     end
 
     def merged_merge_request_email(recipient_id, merge_request_id, updated_by_user_id)
       @merge_request = MergeRequest.find(merge_request_id)
       @project = @merge_request.project
       @target_url = project_merge_request_url(@project, @merge_request)
-      set_reference("merge_request_#{merge_request_id}")
+      mail_answer_thread(@merge_request,
-      mail(from: sender(updated_by_user_id),
+                         from: sender(updated_by_user_id),
-           to: recipient(recipient_id),
+                         cc: recipient(recipient_id),
-           subject: subject("#{@merge_request.title} (##{@merge_request.iid})"))
+                         subject: subject("#{@merge_request.title} (##{@merge_request.iid})"))
     end
   end
 

app/mailers/emails/notes.rb

@@ -5,9 +5,10 @@ module Emails
       @commit = @note.noteable
       @project = @note.project
       @target_url = project_commit_url(@project, @commit, anchor: "note_#{@note.id}")
-      mail(from: sender(@note.author_id),
+      mail_answer_thread(@commit,
-           to: recipient(recipient_id),
+                         from: sender(@note.author_id),
-           subject: subject("#{@commit.title} (#{@commit.short_id})"))
+                         cc: recipient(recipient_id),
+                         subject: subject("#{@commit.title} (#{@commit.short_id})"))
     end
 
     def note_issue_email(recipient_id, note_id)
@@ -15,10 +16,10 @@ module Emails
       @issue = @note.noteable
       @project = @note.project
       @target_url = project_issue_url(@project, @issue, anchor: "note_#{@note.id}")
-      set_reference("issue_#{@issue.id}")
+      mail_answer_thread(@issue,
-      mail(from: sender(@note.author_id),
+                         from: sender(@note.author_id),
-           to: recipient(recipient_id),
+                         cc: recipient(recipient_id),
-           subject: subject("#{@issue.title} (##{@issue.iid})"))
+                         subject: subject("#{@issue.title} (##{@issue.iid})"))
     end
 
     def note_merge_request_email(recipient_id, note_id)
@@ -26,10 +27,10 @@ module Emails
       @merge_request = @note.noteable
       @project = @note.project
       @target_url = project_merge_request_url(@project, @merge_request, anchor: "note_#{@note.id}")
-      set_reference("merge_request_#{@merge_request.id}")
+      mail_answer_thread(@merge_request,
-      mail(from: sender(@note.author_id),
+                         from: sender(@note.author_id),
-           to: recipient(recipient_id),
+                         cc: recipient(recipient_id),
-           subject: subject("#{@merge_request.title} (##{@merge_request.iid})"))
+                         subject: subject("#{@merge_request.title} (##{@merge_request.iid})"))
     end
 
     def note_wall_email(recipient_id, note_id)
@@ -37,7 +38,7 @@ module Emails
       @project = @note.project
       @target_url = project_wall_url(@note.project, anchor: "note_#{@note.id}")
       mail(from: sender(@note.author_id),
-           to: recipient(recipient_id),
+           cc: recipient(recipient_id),
            subject: subject("Note on wall"))
     end
   end

app/mailers/emails/projects.rb

@@ -4,7 +4,7 @@ module Emails
       @users_project = UsersProject.find user_project_id
       @project = @users_project.project
       @target_url = project_url(@project)
-      mail(to: @users_project.user.email,
+      mail(cc: @users_project.user.email,
            subject: subject("Access to project was granted"))
     end
 
@@ -12,7 +12,7 @@ module Emails
       @user = User.find user_id
       @project = Project.find project_id
       @target_url = project_url(@project)
-      mail(to: @user.email,
+      mail(cc: @user.email,
            subject: subject("Project was moved"))
     end
 
@@ -30,7 +30,7 @@ module Emails
       end
 
       mail(from: sender(author_id),
-           to: recipient,
+           cc: recipient,
            subject: subject("New push to repository"))
     end
   end

app/mailers/notify.rb

@@ -1,4 +1,6 @@
 class Notify < ActionMailer::Base
+  include ActionDispatch::Routing::PolymorphicRoutes
+
   include Emails::Issues
   include Emails::MergeRequests
   include Emails::Notes
@@ -16,6 +18,7 @@ class Notify < ActionMailer::Base
   default_url_options[:script_name] = Gitlab.config.gitlab.relative_url_root
 
   default from: Proc.new { default_sender_address.format }
+  default to:   Proc.new { project_sender_address.format }
   default reply_to: "noreply@#{Gitlab.config.gitlab.host}"
 
   # Just send email with 2 seconds delay
@@ -32,6 +35,17 @@ class Notify < ActionMailer::Base
     address
   end
 
+  # The default email address to send emails to. Includes the project name if possible.
+  def project_sender_address
+    if @project
+      address = default_sender_address
+      address.display_name = @project.name_with_namespace
+      address
+    else
+      default_sender_address
+    end
+  end
+
   # Return an email address that displays the name of the sender.
   # Only the displayed name changes; the actual email address is always the same.
   def sender(sender_id)
@@ -53,14 +67,6 @@ class Notify < ActionMailer::Base
     end
   end
 
-  # Set the Message-ID header field
-  #
-  # local_part - The local part of the message ID
-  #
-  def set_message_id(local_part)
-    headers["Message-ID"] = "<#{local_part}@#{Gitlab.config.gitlab.host}>"
-  end
-
   # Set the References header field
   #
   # local_part - The local part of the referenced message ID
@@ -93,4 +99,48 @@ class Notify < ActionMailer::Base
     subject << extra.join(' | ') if extra.present?
     subject
   end
+
+  # Return a string suitable for inclusion in the 'Message-Id' mail header.
+  #
+  # The message-id is generated from the unique URL to a model object.
+  def message_id(model)
+    model_name = model.class.model_name.singular_route_key
+    "<#{model_name}_#{model.id}@#{Gitlab.config.gitlab.host}>"
+  end
+
+  # Send an email that starts a new conversation thread,
+  # with headers suitable for grouping by thread in email clients.
+  #
+  # See: mail_answer_thread
+  def mail_new_thread(model, headers = {}, &block)
+    raise ArgumentError, '"To:" header will be overwritten; use "Cc:" or "Bcc:"' unless headers[:to].nil?
+    headers[:to] = project_sender_address.format
+
+    headers['Message-ID'] = message_id(model)
+
+    mail(headers, &block)
+  end
+
+  # Send an email that responds to an existing conversation thread,
+  # with headers suitable for grouping by thread in email clients.
+  #
+  # For grouping emails by thread, email clients heuristics require the answers to:
+  #
+  #  * have a subject that begin by 'Re: '
+  #  * have a 'In-Reply-To' or 'References' header that references the original 'Message-ID'
+  #  * have stable 'From' and 'To' headers between messages of the same thread
+  #
+  def mail_answer_thread(model, headers = {}, &block)
+    raise ArgumentError, '"To:" header will be overwritten; use "Cc:" or "Bcc:"' unless headers[:to].nil?
+    headers[:to] = project_sender_address.format
+
+    headers['In-Reply-To'] = message_id(model)
+    headers['References'] = message_id(model)
+
+    if (headers[:subject])
+      headers[:subject].prepend('Re: ')
+    end
+
+    mail(headers, &block)
+  end
 end

spec/mailers/notify_spec.rb

@@ -10,7 +10,7 @@ describe Notify do
 
   shared_examples 'a multiple recipients email' do
     it 'is sent to the given recipient' do
-      should deliver_to recipient.email
+      should cc_to recipient.email
     end
   end
 
@@ -22,6 +22,23 @@ describe Notify do
     end
   end
 
+  shared_examples 'an email starting a new thread' do |message_id_prefix|
+    it 'has a discussion identifier' do
+      should have_header 'Message-ID',  /<#{message_id_prefix}(.*)@#{Gitlab.config.gitlab.host}>/
+    end
+  end
+
+  shared_examples 'an answer to an existing thread' do |thread_id_prefix|
+    it 'has a subject that begins with Re: ' do
+      should have_subject /^Re: /
+    end
+
+    it 'has headers that reference an existing thread' do
+      should have_header 'References',  /<#{thread_id_prefix}(.*)@#{Gitlab.config.gitlab.host}>/
+      should have_header 'In-Reply-To', /<#{thread_id_prefix}(.*)@#{Gitlab.config.gitlab.host}>/
+    end
+  end
+
   describe 'for new users, the email' do
     let(:example_site_path) { root_path }
     let(:new_user) { create(:user, email: 'newguy@example.com', created_by_id: 1) }
@@ -141,7 +158,7 @@ describe Notify do
         end
 
         it 'is sent to the assignee' do
-          should deliver_to assignee.email
+          should cc_to assignee.email
         end
       end
 
@@ -153,6 +170,7 @@ describe Notify do
           subject { Notify.new_issue_email(issue.assignee_id, issue.id) }
 
           it_behaves_like 'an assignee email'
+          it_behaves_like 'an email starting a new thread', 'issue'
 
           it 'has the correct subject' do
             should have_subject /#{project.name} \| #{issue.title} \(##{issue.iid}\)/
@@ -161,10 +179,6 @@ describe Notify do
           it 'contains a link to the new issue' do
             should have_body_text /#{project_issue_path project, issue}/
           end
-
-          it 'has the correct message-id set' do
-            should have_header 'Message-ID', "<issue_#{issue.id}@#{Gitlab.config.gitlab.host}>"
-          end
         end
 
         describe 'that are new with a description' do
@@ -179,6 +193,7 @@ describe Notify do
           subject { Notify.reassigned_issue_email(recipient.id, issue.id, previous_assignee.id, current_user) }
 
           it_behaves_like 'a multiple recipients email'
+          it_behaves_like 'an answer to an existing thread', 'issue'
 
           it 'is sent as the author' do
             sender = subject.header[:from].addrs[0]
@@ -201,16 +216,14 @@ describe Notify do
           it 'contains a link to the issue' do
             should have_body_text /#{project_issue_path project, issue}/
           end
-
-          it 'has the correct reference set' do
-            should have_header 'References', "<issue_#{issue.id}@#{Gitlab.config.gitlab.host}>"
-          end
         end
 
         describe 'status changed' do
           let(:status) { 'closed' }
           subject { Notify.issue_status_changed_email(recipient.id, issue.id, status, current_user) }
 
+          it_behaves_like 'an answer to an existing thread', 'issue'
+
           it 'is sent as the author' do
             sender = subject.header[:from].addrs[0]
             sender.display_name.should eq(current_user.name)
@@ -232,10 +245,6 @@ describe Notify do
           it 'contains a link to the issue' do
             should have_body_text /#{project_issue_path project, issue}/
           end
-
-          it 'has the correct reference set' do
-            should have_header 'References', "<issue_#{issue.id}@#{Gitlab.config.gitlab.host}>"
-          end
         end
 
       end
@@ -249,6 +258,7 @@ describe Notify do
           subject { Notify.new_merge_request_email(merge_request.assignee_id, merge_request.id) }
 
           it_behaves_like 'an assignee email'
+          it_behaves_like 'an email starting a new thread', 'merge_request'
 
           it 'has the correct subject' do
             should have_subject /#{merge_request.title} \(##{merge_request.iid}\)/
@@ -283,6 +293,7 @@ describe Notify do
           subject { Notify.reassigned_merge_request_email(recipient.id, merge_request.id, previous_assignee.id, current_user.id) }
 
           it_behaves_like 'a multiple recipients email'
+          it_behaves_like 'an answer to an existing thread', 'merge_request'
 
           it 'is sent as the author' do
             sender = subject.header[:from].addrs[0]
@@ -311,6 +322,7 @@ describe Notify do
           subject { Notify.merged_merge_request_email(recipient.id, merge_request.id, merge_author.id) }
 
           it_behaves_like 'a multiple recipients email'
+          it_behaves_like 'an answer to an existing thread', 'merge_request'
 
           it 'is sent as the merge author' do
             sender = subject.header[:from].addrs[0]
@@ -329,10 +341,6 @@ describe Notify do
           it 'contains a link to the merge request' do
             should have_body_text /#{project_merge_request_path project, merge_request}/
           end
-
-          it 'has the correct reference set' do
-            should have_header 'References', "<merge_request_#{merge_request.id}@#{Gitlab.config.gitlab.host}>"
-          end
         end
       end
     end
@@ -394,7 +402,7 @@ describe Notify do
         end
 
         it 'is sent to the given recipient' do
-          should deliver_to recipient.email
+          should cc_to recipient.email
         end
 
         it 'contains the message from the note' do
@@ -426,6 +434,7 @@ describe Notify do
         subject { Notify.note_commit_email(recipient.id, note.id) }
 
         it_behaves_like 'a note email'
+        it_behaves_like 'an answer to an existing thread', 'commits'
 
         it 'has the correct subject' do
           should have_subject /#{commit.title} \(#{commit.short_id}\)/
@@ -444,6 +453,7 @@ describe Notify do
         subject { Notify.note_merge_request_email(recipient.id, note.id) }
 
         it_behaves_like 'a note email'
+        it_behaves_like 'an answer to an existing thread', 'merge_request'
 
         it 'has the correct subject' do
           should have_subject /#{merge_request.title} \(##{merge_request.iid}\)/
@@ -462,6 +472,7 @@ describe Notify do
         subject { Notify.note_issue_email(recipient.id, note.id) }
 
         it_behaves_like 'a note email'
+        it_behaves_like 'an answer to an existing thread', 'issue'
 
         it 'has the correct subject' do
           should have_subject /#{issue.title} \(##{issue.iid}\)/
@@ -538,7 +549,7 @@ describe Notify do
     end
 
     it 'is sent to recipient' do
-      should deliver_to 'devs@company.name'
+      should cc_to 'devs@company.name'
     end
 
     it 'has the correct subject' do
@@ -574,7 +585,7 @@ describe Notify do
     end
 
     it 'is sent to recipient' do
-      should deliver_to 'devs@company.name'
+      should cc_to 'devs@company.name'
     end
 
     it 'has the correct subject' do

spec/services/issues/close_service_spec.rb

@@ -22,7 +22,7 @@ describe Issues::CloseService do
 
       it 'should send email to user2 about assign of new issue' do
         email = ActionMailer::Base.deliveries.last
-        email.to.first.should == user2.email
+        email.cc.first.should == user2.email
         email.subject.should include(issue.title)
       end
 

spec/services/issues/update_service_spec.rb

@@ -31,7 +31,7 @@ describe Issues::UpdateService do
 
       it 'should send email to user2 about assign of new issue' do
         email = ActionMailer::Base.deliveries.last
-        email.to.first.should == user2.email
+        email.cc.first.should == user2.email
         email.subject.should include(issue.title)
       end
 

spec/services/merge_requests/close_service_spec.rb

@@ -22,7 +22,7 @@ describe MergeRequests::CloseService do
 
       it 'should send email to user2 about assign of new merge_request' do
         email = ActionMailer::Base.deliveries.last
-        email.to.first.should == user2.email
+        email.cc.first.should == user2.email
         email.subject.should include(merge_request.title)
       end
 

spec/services/merge_requests/update_service_spec.rb

@@ -31,7 +31,7 @@ describe MergeRequests::UpdateService do
 
       it 'should send email to user2 about assign of new merge_request' do
         email = ActionMailer::Base.deliveries.last
-        email.to.first.should == user2.email
+        email.cc.first.should == user2.email
         email.subject.should include(merge_request.title)
       end