diff --git a/discord/ext/commands/core.py b/discord/ext/commands/core.py
index 248a0e143..b0358ca9a 100644
--- a/discord/ext/commands/core.py
+++ b/discord/ext/commands/core.py
@@ -1145,6 +1145,7 @@ class GroupMixin:
         self.all_commands[command.name] = command
         for alias in command.aliases:
             if alias in self.all_commands:
+                self.remove_command(command.name)
                 raise CommandRegistrationError(alias, alias_conflict=True)
             self.all_commands[alias] = command
 
@@ -1177,7 +1178,12 @@ class GroupMixin:
 
         # we're not removing the alias so let's delete the rest of them.
         for alias in command.aliases:
-            self.all_commands.pop(alias, None)
+            cmd = self.all_commands.pop(alias, None)
+            # in the case of a CommandRegistrationError, an alias might conflict
+            # with an already existing command. If this is the case, we want to
+            # make sure the pre-existing command is not removed.
+            if cmd not in (None, command):
+                self.all_commands[alias] = cmd
         return command
 
     def walk_commands(self):