Only remove if the user is actually in the voice channel.

discord/server.py

@@ -95,7 +95,7 @@ class Member(User):
         if old_channel is None and self.voice_channel is not None:
             # we joined a channel
             self.voice_channel.voice_members.append(self)
-        elif old_channel is not None and self.voice_channel is None:
+        elif old_channel is not None and self.voice_channel is None and self in old_channel:
             # we left a channel
             old_channel.voice_members.remove(self)