diff --git a/app/services/members/authorized_destroy_service.rb b/app/services/members/authorized_destroy_service.rb
index 0f94504625a..f846d72498f 100644
--- a/app/services/members/authorized_destroy_service.rb
+++ b/app/services/members/authorized_destroy_service.rb
@@ -10,7 +10,7 @@ module Members
       return false if member.is_a?(GroupMember) && member.source.last_owner?(member.user)
 
       Member.transaction do
-        unassign_issues_and_merge_requests(member)
+        unassign_issues_and_merge_requests(member) unless member.invite?
 
         member.destroy
       end
diff --git a/spec/factories/group_members.rb b/spec/factories/group_members.rb
index 080b2e75ea1..32cbfe28a60 100644
--- a/spec/factories/group_members.rb
+++ b/spec/factories/group_members.rb
@@ -10,5 +10,11 @@ FactoryGirl.define do
     trait(:master)    { access_level GroupMember::MASTER }
     trait(:owner)     { access_level GroupMember::OWNER }
     trait(:access_request) { requested_at Time.now }
+
+    trait(:invited) do
+      user_id nil
+      invite_token 'xxx'
+      invite_email 'email@email.com'
+    end
   end
 end
diff --git a/spec/factories/project_members.rb b/spec/factories/project_members.rb
index d62799a5a47..fe4518caadf 100644
--- a/spec/factories/project_members.rb
+++ b/spec/factories/project_members.rb
@@ -9,5 +9,11 @@ FactoryGirl.define do
     trait(:developer) { access_level ProjectMember::DEVELOPER }
     trait(:master)    { access_level ProjectMember::MASTER }
     trait(:access_request) { requested_at Time.now }
+
+    trait(:invited) do
+      user_id nil
+      invite_token 'xxx'
+      invite_email 'email@email.com'
+    end
   end
 end
diff --git a/spec/services/members/authorized_destroy_service_spec.rb b/spec/services/members/authorized_destroy_service_spec.rb
index ab440d18e9f..8a6732faa19 100644
--- a/spec/services/members/authorized_destroy_service_spec.rb
+++ b/spec/services/members/authorized_destroy_service_spec.rb
@@ -10,6 +10,27 @@ describe Members::AuthorizedDestroyService, services: true do
     Issue.assigned_to(user).count + MergeRequest.assigned_to(user).count
   end
 
+  context 'Invited users' do
+    # Regression spec for issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/32504
+    it 'destroys invited project member' do
+      project.team << [member_user, :developer]
+
+      member = create :project_member, :invited, project: project
+
+      expect { described_class.new(member, member_user).execute }
+        .to change { Member.count }.from(2).to(1)
+    end
+
+    it 'destroys invited group member' do
+      group.add_developer(member_user)
+
+      member = create :group_member, :invited, group: group
+
+      expect { described_class.new(member, member_user).execute }
+        .to change { Member.count }.from(2).to(1)
+    end
+  end
+
   context 'Group member' do
     it "unassigns issues and merge requests" do
       group.add_developer(member_user)