diff --git a/app/mailers/email_rejection_mailer.rb b/app/mailers/email_rejection_mailer.rb
new file mode 100644
index 00000000000..f29c4e052fc
--- /dev/null
+++ b/app/mailers/email_rejection_mailer.rb
@@ -0,0 +1,35 @@
+class EmailRejectionMailer < ActionMailer::Base
+  add_template_helper ApplicationHelper
+  add_template_helper GitlabMarkdownHelper
+
+  helper_method :current_user, :can?
+
+  default from:     "#{Gitlab.config.gitlab.email_display_name} <#{Gitlab.config.gitlab.email_from}>"
+  default reply_to: "#{Gitlab.config.gitlab.email_display_name} <#{Gitlab.config.gitlab.email_reply_to}>"
+
+  def rejection(reason, original_raw, can_retry = false)
+    @reason = reason
+    @original_message = Mail::Message.new(original_raw)
+
+    headers = {
+      to: @original_message.from,
+      subject: "[Rejected] #{@original_message.subject}"
+    }
+
+    headers['Message-ID'] = SecureRandom.hex
+    headers['In-Reply-To'] = @original_message.message_id
+    headers['References'] = @original_message.message_id
+
+    headers['Reply-To'] = @original_message.to.first if can_retry
+
+    mail(headers)
+  end
+
+  def current_user
+    nil
+  end
+
+  def can?
+    false
+  end
+end
diff --git a/app/views/email_rejection_mailer/rejection.html.haml b/app/views/email_rejection_mailer/rejection.html.haml
new file mode 100644
index 00000000000..de29d399d32
--- /dev/null
+++ b/app/views/email_rejection_mailer/rejection.html.haml
@@ -0,0 +1,5 @@
+%p
+  Unfortunately, your email message to GitLab could not be processed.
+
+%p
+  = gfm @reason
diff --git a/app/views/email_rejection_mailer/rejection.text.haml b/app/views/email_rejection_mailer/rejection.text.haml
new file mode 100644
index 00000000000..6693e6f90e8
--- /dev/null
+++ b/app/views/email_rejection_mailer/rejection.text.haml
@@ -0,0 +1,4 @@
+Unfortunately, your email message to GitLab could not be processed.
+
+
+= @reason
diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb
index 94e346b5a51..2cfd64cefad 100644
--- a/app/workers/email_receiver_worker.rb
+++ b/app/workers/email_receiver_worker.rb
@@ -6,17 +6,42 @@ class EmailReceiverWorker
   def perform(raw)
     return unless Gitlab::ReplyByEmail.enabled?
 
-    # begin
-      Gitlab::EmailReceiver.new(raw).process
-    # rescue => e
-    #   handle_failure(raw, e)
-    # end
+    begin
+      Gitlab::EmailReceiver.new(raw).execute
+    rescue => e
+      handle_failure(raw, e)
+    end
   end
 
   private
 
   def handle_failure(raw, e)
-    # TODO: Handle better.
     Rails.logger.warn("Email can not be processed: #{e}\n\n#{raw}")
+
+    can_retry = false
+    reason = nil
+
+    case e
+    when Gitlab::EmailReceiver::SentNotificationNotFound
+      reason = "We couldn't figure out what the email is in reply to. Please create your comment through the web interface."
+    when Gitlab::EmailReceiver::EmptyEmailError
+      can_retry = true
+      reason = "It appears that the email is blank. Make sure your reply is at the top of the email, we can't process inline replies."
+    when Gitlab::EmailReceiver::AutoGeneratedEmailError
+      reason = "The email was marked as 'auto generated', which we can't accept. Please create your comment through the web interface."
+    when Gitlab::EmailReceiver::UserNotFoundError
+      reason = "We couldn't figure out what user corresponds to the email. Please create your comment through the web interface."
+    when Gitlab::EmailReceiver::UserNotAuthorizedError
+      reason = "You are not allowed to respond to the thread you are replying to. If you believe this is in error, contact a staff member."
+    when Gitlab::EmailReceiver::NoteableNotFoundError
+      reason = "The thread you are replying to no longer exists, perhaps it was deleted? If you believe this is in error, contact a staff member."
+    when Gitlab::EmailReceiver::InvalidNote
+      can_retry = true
+      reason = e.message
+    else
+      return
+    end
+
+    EmailRejectionMailer.delay.rejection(reason, raw, can_retry)
   end
 end
diff --git a/lib/gitlab/email_receiver.rb b/lib/gitlab/email_receiver.rb
index 3dd8942a262..a0b4ff87e02 100644
--- a/lib/gitlab/email_receiver.rb
+++ b/lib/gitlab/email_receiver.rb
@@ -5,7 +5,7 @@ module Gitlab
     class EmailUnparsableError < ProcessingError; end
     class EmptyEmailError < ProcessingError; end
     class UserNotFoundError < ProcessingError; end
-    class UserNotAuthorizedLevelError < ProcessingError; end
+    class UserNotAuthorizedError < ProcessingError; end
     class NoteableNotFoundError < ProcessingError; end
     class AutoGeneratedEmailError < ProcessingError; end
     class SentNotificationNotFound < ProcessingError; end
@@ -21,20 +21,20 @@ module Gitlab
       raise EmailUnparsableError, e
     end
 
-    def process
+    def execute
+      raise SentNotificationNotFound unless sent_notification
+
       raise EmptyEmailError if @raw.blank?
 
       raise AutoGeneratedEmailError if message.header.to_s =~ /auto-(generated|replied)/
 
-      raise SentNotificationNotFound unless sent_notification
-
       author = sent_notification.recipient
 
       raise UserNotFoundError unless author
 
       project = sent_notification.project
 
-      raise UserNotAuthorizedLevelError unless author.can?(:create_note, project)
+      raise UserNotAuthorizedError unless author.can?(:create_note, project)
 
       raise NoteableNotFoundError unless sent_notification.noteable
 
@@ -54,7 +54,11 @@ module Gitlab
       ).execute
 
       unless note.persisted?
-        raise InvalidNote, note.errors.full_messages.join("\n")
+        message = "The comment could not be created for the following reasons:"
+        note.errors.full_messages.each do |error|
+          message << "\n\n- #{error}"
+        end
+        raise InvalidNote, message
       end
     end