diff --git a/app/services/system_note_service.rb b/app/services/system_note_service.rb
index 30a5aab13bf..06b23cd7076 100644
--- a/app/services/system_note_service.rb
+++ b/app/services/system_note_service.rb
@@ -68,21 +68,14 @@ module SystemNoteService
   #
   # Returns the created Note object
   def change_issue_assignees(issue, project, author, old_assignees)
-    body =
-      if issue.assignees.any? && old_assignees.any?
-        unassigned_users = old_assignees - issue.assignees
-        added_users = issue.assignees.to_a - old_assignees
+    unassigned_users = old_assignees - issue.assignees
+    added_users = issue.assignees.to_a - old_assignees
 
-        text_parts = []
-        text_parts << "assigned to #{added_users.map(&:to_reference).to_sentence}" if added_users.any?
-        text_parts << "unassigned #{unassigned_users.map(&:to_reference).to_sentence}" if unassigned_users.any?
+    text_parts = []
+    text_parts << "assigned to #{added_users.map(&:to_reference).to_sentence}" if added_users.any?
+    text_parts << "unassigned #{unassigned_users.map(&:to_reference).to_sentence}" if unassigned_users.any?
 
-        text_parts.join(' and ')
-      elsif old_assignees.any?
-        "removed assignee"
-      elsif issue.assignees.any?
-        "assigned to #{issue.assignees.map(&:to_reference).to_sentence}"
-      end
+    body = text_parts.join(' and ')
 
     create_note(NoteSummary.new(issue, project, author, body, action: 'assignee'))
   end
diff --git a/changelogs/unreleased/16301-update-removed-assignee-note-to-include-old-assignee-reference.yml b/changelogs/unreleased/16301-update-removed-assignee-note-to-include-old-assignee-reference.yml
new file mode 100644
index 00000000000..e94b4f8bb26
--- /dev/null
+++ b/changelogs/unreleased/16301-update-removed-assignee-note-to-include-old-assignee-reference.yml
@@ -0,0 +1,5 @@
+---
+title: "Update 'removed assignee' note to include old assignee reference"
+merge_request: 16301
+author: Maurizio De Santis
+type: changed
diff --git a/spec/services/system_note_service_spec.rb b/spec/services/system_note_service_spec.rb
index 4e640a82dfc..7c1bfa4295d 100644
--- a/spec/services/system_note_service_spec.rb
+++ b/spec/services/system_note_service_spec.rb
@@ -158,7 +158,7 @@ describe SystemNoteService do
     end
 
     it 'builds a correct phrase when assignee removed' do
-      expect(build_note([assignee1], [])).to eq 'removed assignee'
+      expect(build_note([assignee1], [])).to eq "unassigned @#{assignee1.username}"
     end
 
     it 'builds a correct phrase when assignees changed' do