From b341ae9aee9a6448c518ef5bdc91efe8a385beed Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 3 Apr 2017 17:50:13 -0700 Subject: [PATCH 01/63] Fix missing \n --- disco/util/logging.py | 1 + 1 file changed, 1 insertion(+) diff --git a/disco/util/logging.py b/disco/util/logging.py index 68af8a8..75e9229 100644 --- a/disco/util/logging.py +++ b/disco/util/logging.py @@ -9,6 +9,7 @@ LEVEL_OVERRIDES = { LOG_FORMAT = '[%(levelname)s] %(asctime)s - %(name)s:%(lineno)d - %(message)s' + def setup_logging(**kwargs): kwargs.setdefault('format', LOG_FORMAT) From f3ae56a496e416e12e0aa61b003fde53c2e83ee0 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 9 Apr 2017 22:09:34 -0700 Subject: [PATCH 02/63] Fix role mentions --- disco/types/guild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disco/types/guild.py b/disco/types/guild.py index f93a88f..600ddb0 100644 --- a/disco/types/guild.py +++ b/disco/types/guild.py @@ -102,7 +102,7 @@ class Role(SlottedModel): @property def mention(self): - return '<@{}>'.format(self.id) + return '<@&{}>'.format(self.id) @cached_property def guild(self): From 28b0bf72fb940dd1a21dd5945c16ebb6f7b63dba Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 10 Apr 2017 01:03:53 -0700 Subject: [PATCH 03/63] Plugin name should be the first trigger --- disco/bot/command.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/disco/bot/command.py b/disco/bot/command.py index 3d0398a..c5aaddf 100644 --- a/disco/bot/command.py +++ b/disco/bot/command.py @@ -140,6 +140,10 @@ class Command(object): self.update(*args, **kwargs) + @property + def name(self): + return self.triggers[0] + def __call__(self, *args, **kwargs): return self.func(*args, **kwargs) From 375b1ff942952361a961c85be56685e50cd269e7 Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 10 Apr 2017 21:48:32 -0700 Subject: [PATCH 04/63] Make sure emojis track guild_id --- disco/state.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/disco/state.py b/disco/state.py index 689ab62..2983aa0 100644 --- a/disco/state.py +++ b/disco/state.py @@ -309,6 +309,9 @@ class State(object): if event.guild_id not in self.guilds: return + for emoji in event.emojis: + emoji.guild_id = event.guild_id + self.guilds[event.guild_id].emojis = HashMap({i.id: i for i in event.emojis}) def on_presence_update(self, event): From 642542d9757914e0c0f316d25e0bc0454a041914 Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 10 Apr 2017 21:48:45 -0700 Subject: [PATCH 05/63] Add support for updating guild emoji, opening DMs --- disco/api/client.py | 6 ++++++ disco/types/guild.py | 3 +++ disco/types/user.py | 3 +++ 3 files changed, 12 insertions(+) diff --git a/disco/api/client.py b/disco/api/client.py index 0c00539..133472b 100644 --- a/disco/api/client.py +++ b/disco/api/client.py @@ -311,6 +311,12 @@ class APIClient(LoggingClass): r = self.http(Routes.USERS_ME_PATCH, json=payload) return User.create(self.client, r.json()) + def users_me_dms_create(self, recipient_id): + r = self.http(Routes.USERS_ME_DMS_CREATE, json={ + 'recipient_id': recipient_id, + }) + return Channel.create(self.client, r.json()) + def invites_get(self, invite): r = self.http(Routes.INVITES_GET, dict(invite=invite)) return Invite.create(self.client, r.json()) diff --git a/disco/types/guild.py b/disco/types/guild.py index 600ddb0..c4102c6 100644 --- a/disco/types/guild.py +++ b/disco/types/guild.py @@ -51,6 +51,9 @@ class GuildEmoji(Emoji): def __str__(self): return u'<:{}:{}>'.format(self.name, self.id) + def update(self, **kwargs): + return self.client.api.guilds_emojis_modify(self.guild_id, self.id, **kwargs) + @property def url(self): return 'https://discordapp.com/api/emojis/{}.png'.format(self.id) diff --git a/disco/types/user.py b/disco/types/user.py index 3192abc..2777c5e 100644 --- a/disco/types/user.py +++ b/disco/types/user.py @@ -45,6 +45,9 @@ class User(SlottedModel, with_equality('id'), with_hash('id')): def mention(self): return '<@{}>'.format(self.id) + def open_dm(self): + return self.client.api.users_me_dms_create(self.id) + def __str__(self): return u'{}#{}'.format(self.username, str(self.discriminator).zfill(4)) From 03dab6d8299ccad819c6df622d67c916bc79c613 Mon Sep 17 00:00:00 2001 From: Andrei Zbikowski Date: Mon, 10 Apr 2017 22:23:09 -0700 Subject: [PATCH 06/63] Voice Send Support (#17) * First pass at voice sending * more voice * Refactor playables a bit, general fixes n stuff * Cleanup * Voice encryption, dep version bump, etc fixes * Remove debugging, don't open a pipe for stderr * Refactor playables This is still a very lose concept, need to think about what the actual differences between encoders and playables are. Also some rough edges in general with the frame/sample calculations. However, this still feels miles ahead of the previous iteration. * Properly reset state when resuming from a pause * rework playables/encoding/etc a bit * Add a proxy, allow for more pipin' * Cleanup, etc * Fix resuming from a pause lerping music timestamp * Fix some incorrect bounds checks, add MemoryBufferedPlayable --- .gitignore | 1 + disco/bot/bot.py | 6 +- disco/gateway/events.py | 2 + disco/voice/__init__.py | 3 + disco/voice/client.py | 135 +++++++++++----- disco/voice/opus.py | 149 +++++++++++++++++ disco/voice/playable.py | 347 ++++++++++++++++++++++++++++++++++++++++ disco/voice/player.py | 122 ++++++++++++++ examples/music.py | 52 ++++++ requirements.txt | 7 +- 10 files changed, 775 insertions(+), 49 deletions(-) create mode 100644 disco/voice/opus.py create mode 100644 disco/voice/playable.py create mode 100644 disco/voice/player.py create mode 100644 examples/music.py diff --git a/.gitignore b/.gitignore index 54cb974..87a6c02 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ dist/ disco*.egg-info/ docs/_build storage.db +*.dca diff --git a/disco/bot/bot.py b/disco/bot/bot.py index 7693f50..8ad0fca 100644 --- a/disco/bot/bot.py +++ b/disco/bot/bot.py @@ -355,10 +355,10 @@ class Bot(LoggingClass): if event.message.author.id == self.client.state.me.id: return - if self.config.commands_allow_edit: - self.last_message_cache[event.message.channel_id] = (event.message, False) + result = self.handle_message(event.message) - self.handle_message(event.message) + if self.config.commands_allow_edit: + self.last_message_cache[event.message.channel_id] = (event.message, result) def on_message_update(self, event): if self.config.commands_allow_edit: diff --git a/disco/gateway/events.py b/disco/gateway/events.py index 5033b03..9d9d323 100644 --- a/disco/gateway/events.py +++ b/disco/gateway/events.py @@ -49,6 +49,8 @@ class GatewayEvent(six.with_metaclass(GatewayEventMeta, Model)): """ Create this GatewayEvent class from data and the client. """ + cls.raw_data = obj + # If this event is wrapping a model, pull its fields if hasattr(cls, '_wraps_model'): alias, model = cls._wraps_model diff --git a/disco/voice/__init__.py b/disco/voice/__init__.py index e69de29..b4a7f6c 100644 --- a/disco/voice/__init__.py +++ b/disco/voice/__init__.py @@ -0,0 +1,3 @@ +from disco.voice.client import * +from disco.voice.player import * +from disco.voice.playable import * diff --git a/disco/voice/client.py b/disco/voice/client.py index 69fe9d5..d0d3e09 100644 --- a/disco/voice/client.py +++ b/disco/voice/client.py @@ -3,6 +3,8 @@ import socket import struct import time +import nacl.secret + from holster.enum import Enum from holster.emitter import Emitter @@ -22,11 +24,6 @@ VoiceState = Enum( VOICE_CONNECTED=6, ) -# TODO: -# - player implementation -# - encryption -# - cleanup - class VoiceException(Exception): def __init__(self, msg, client): @@ -38,12 +35,40 @@ class UDPVoiceClient(LoggingClass): def __init__(self, vc): super(UDPVoiceClient, self).__init__() self.vc = vc + + # The underlying UDP socket self.conn = None + + # Connection information self.ip = None self.port = None + self.run_task = None self.connected = False + def send_frame(self, frame, sequence=None, timestamp=None): + # Convert the frame to a bytearray + frame = bytearray(frame) + + # First, pack the header (TODO: reuse bytearray?) + header = bytearray(24) + header[0] = 0x80 + header[1] = 0x78 + struct.pack_into('>H', header, 2, sequence or self.vc.sequence) + struct.pack_into('>I', header, 4, timestamp or self.vc.timestamp) + struct.pack_into('>i', header, 8, self.vc.ssrc) + + # Now encrypt the payload with the nonce as a header + raw = self.vc.secret_box.encrypt(bytes(frame), bytes(header)).ciphertext + + # Send the header (sans nonce padding) plus the payload + self.send(header[:12] + raw) + + # Increment our sequence counter + self.vc.sequence += 1 + if self.vc.sequence >= 65535: + self.vc.sequence = 0 + def run(self): while True: self.conn.recvfrom(4096) @@ -54,26 +79,29 @@ class UDPVoiceClient(LoggingClass): def disconnect(self): self.run_task.kill() - def connect(self, host, port, timeout=10): + def connect(self, host, port, timeout=10, addrinfo=None): self.ip = socket.gethostbyname(host) self.port = port self.conn = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - # Send discovery packet - packet = bytearray(70) - struct.pack_into('>I', packet, 0, self.vc.ssrc) - self.send(packet) + if addrinfo: + ip, port = addrinfo + else: + # Send discovery packet + packet = bytearray(70) + struct.pack_into('>I', packet, 0, self.vc.ssrc) + self.send(packet) - # Wait for a response - try: - data, addr = gevent.spawn(lambda: self.conn.recvfrom(70)).get(timeout=timeout) - except gevent.Timeout: - return (None, None) + # Wait for a response + try: + data, addr = gevent.spawn(lambda: self.conn.recvfrom(70)).get(timeout=timeout) + except gevent.Timeout: + return (None, None) - # Read IP and port - ip = str(data[4:]).split('\x00', 1)[0] - port = struct.unpack(' OpusPlayable +# FFMpegInput.youtube_dl('youtube.com/yolo').pipe(DCADOpusEncoder) => OpusPlayable +# FFMpegInput.youtube_dl('youtube.com/yolo').pipe(OpusEncoder).pipe(DuplexStream, open('cache_file.opus', 'w')) => OpusPlayable + + +class AbstractOpus(object): + def __init__(self, sampling_rate=48000, frame_length=20, channels=2): + self.sampling_rate = sampling_rate + self.frame_length = frame_length + self.channels = 2 + self.sample_size = 2 * self.channels + self.samples_per_frame = int(self.sampling_rate / 1000 * self.frame_length) + self.frame_size = self.samples_per_frame * self.sample_size + + +class BaseUtil(object): + def pipe(self, other, *args, **kwargs): + child = other(self, *args, **kwargs) + setattr(child, 'metadata', self.metadata) + setattr(child, '_parent', self) + return child + + @property + def metadata(self): + return self._metadata + + @metadata.setter + def metadata(self, value): + self._metadata = value + + +@six.add_metaclass(abc.ABCMeta) +class BasePlayable(BaseUtil): + @abc.abstractmethod + def next_frame(self): + raise NotImplementedError + + +@six.add_metaclass(abc.ABCMeta) +class BaseInput(BaseUtil): + @abc.abstractmethod + def read(self, size): + raise NotImplementedError + + @abc.abstractmethod + def fileobj(self): + raise NotImplementedError + + +class OpusFilePlayable(BasePlayable, AbstractOpus): + """ + An input which reads opus data from a file or file-like object. + """ + def __init__(self, fobj, *args, **kwargs): + super(OpusFilePlayable, self).__init__(*args, **kwargs) + self.fobj = fobj + self.done = False + + def next_frame(self): + if self.done: + return None + + header = self.fobj.read(OPUS_HEADER_SIZE) + if len(header) < OPUS_HEADER_SIZE: + self.done = True + return None + + data_size = struct.unpack('0]/bestaudio/best'}) + + if self._url: + obj = ydl.extract_info(self._url, download=False, process=False) + if 'entries' in obj: + self._ie_info = obj['entries'] + else: + self._ie_info = [obj] + + self._info = ydl.process_ie_result(self._ie_info, download=False) + return self._info + + @property + def _metadata(self): + return self.info + + @classmethod + def many(cls, url, *args, **kwargs): + import youtube_dl + + ydl = youtube_dl.YoutubeDL({'format': 'webm[abr>0]/bestaudio/best'}) + info = ydl.extract_info(url, download=False, process=False) + + if 'entries' not in info: + yield cls(ie_info=info, *args, **kwargs) + raise StopIteration + + for item in info['entries']: + yield cls(ie_info=item, *args, **kwargs) + + @property + def source(self): + return self.info['url'] + + +class BufferedOpusEncoderPlayable(BasePlayable, AbstractOpus, OpusEncoder): + def __init__(self, source, *args, **kwargs): + self.source = source + self.frames = Queue(kwargs.pop('queue_size', 4096)) + super(BufferedOpusEncoderPlayable, self).__init__(*args, **kwargs) + gevent.spawn(self._encoder_loop) + + def _encoder_loop(self): + while self.source: + raw = self.source.read(self.frame_size) + if len(raw) < self.frame_size: + break + + self.frames.put(self.encode(raw, self.samples_per_frame)) + gevent.idle() + self.source = None + + def next_frame(self): + if not self.source: + return None + return self.frames.get() + + +class DCADOpusEncoderPlayable(BasePlayable, AbstractOpus, OpusEncoder): + def __init__(self, source, *args, **kwargs): + self.source = source + self.command = kwargs.pop('command', 'dcad') + super(DCADOpusEncoderPlayable, self).__init__(*args, **kwargs) + + self._done = False + self._proc = None + + @property + def proc(self): + if not self._proc: + source = obj = self.source.fileobj() + if not hasattr(obj, 'fileno'): + source = subprocess.PIPE + + self._proc = subprocess.Popen([ + self.command, + '--channels', str(self.channels), + '--rate', str(self.sampling_rate), + '--size', str(self.samples_per_frame), + '--bitrate', '128', + '--fec', + '--packet-loss-percent', '30', + '--input', 'pipe:0', + '--output', 'pipe:1', + ], stdin=source, stdout=subprocess.PIPE) + + def writer(): + while True: + data = obj.read(2048) + if len(data) > 0: + self._proc.stdin.write(data) + if len(data) < 2048: + break + + if source == subprocess.PIPE: + gevent.spawn(writer) + return self._proc + + def next_frame(self): + if self._done: + return None + + header = self.proc.stdout.read(OPUS_HEADER_SIZE) + if len(header) < OPUS_HEADER_SIZE: + self._done = True + return + + size = struct.unpack('') + def on_play(self, event, url): + item = FFmpegInput.youtube_dl(url).pipe(DCADOpusEncoderPlayable) + self.get_player(event.guild.id).queue.put(item) + + @Plugin.command('pause') + def on_pause(self, event): + self.get_player(event.guild.id).pause() + + @Plugin.command('resume') + def on_resume(self, event): + self.get_player(event.guild.id).resume() diff --git a/requirements.txt b/requirements.txt index 3b0894b..ed9a25c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ -gevent==1.1.2 +gevent==1.2.1 holster==1.0.11 inflection==0.3.1 -requests==2.11.1 +requests==2.13.0 six==1.10.0 -websocket-client==0.37.0 +websocket-client==0.40.0 +pynacl==1.1.2 From d98432db6dc375df05806015d03f1cf0c993e6d8 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 11 Apr 2017 07:12:07 -0700 Subject: [PATCH 07/63] Release v0.0.8 --- disco/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disco/__init__.py b/disco/__init__.py index 262e0b7..c3f2301 100644 --- a/disco/__init__.py +++ b/disco/__init__.py @@ -1 +1 @@ -VERSION = '0.0.7' +VERSION = '0.0.8' From 6af9e2445627a9aa6d15e2c5de4fd808bf449a8b Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 11 Apr 2017 07:38:27 -0700 Subject: [PATCH 08/63] Add support for leaving guilds --- disco/api/client.py | 3 +++ disco/api/http.py | 2 +- disco/types/guild.py | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/disco/api/client.py b/disco/api/client.py index 133472b..254bb62 100644 --- a/disco/api/client.py +++ b/disco/api/client.py @@ -311,6 +311,9 @@ class APIClient(LoggingClass): r = self.http(Routes.USERS_ME_PATCH, json=payload) return User.create(self.client, r.json()) + def users_me_guilds_delete(self, guild): + self.http(Routes.USERS_ME_GUILDS_DELETE, dict(guild=guild)) + def users_me_dms_create(self, recipient_id): r = self.http(Routes.USERS_ME_DMS_CREATE, json={ 'recipient_id': recipient_id, diff --git a/disco/api/http.py b/disco/api/http.py index 088b69e..5dff893 100644 --- a/disco/api/http.py +++ b/disco/api/http.py @@ -108,7 +108,7 @@ class Routes(object): USERS_ME_GET = (HTTPMethod.GET, USERS + '/@me') USERS_ME_PATCH = (HTTPMethod.PATCH, USERS + '/@me') USERS_ME_GUILDS_LIST = (HTTPMethod.GET, USERS + '/@me/guilds') - USERS_ME_GUILDS_LEAVE = (HTTPMethod.DELETE, USERS + '/@me/guilds/{guild}') + USERS_ME_GUILDS_DELETE = (HTTPMethod.DELETE, USERS + '/@me/guilds/{guild}') USERS_ME_DMS_LIST = (HTTPMethod.GET, USERS + '/@me/channels') USERS_ME_DMS_CREATE = (HTTPMethod.POST, USERS + '/@me/channels') USERS_ME_CONNECTIONS_LIST = (HTTPMethod.GET, USERS + '/@me/connections') diff --git a/disco/types/guild.py b/disco/types/guild.py index c4102c6..15c1c17 100644 --- a/disco/types/guild.py +++ b/disco/types/guild.py @@ -420,3 +420,6 @@ class Guild(SlottedModel, Permissible): def create_channel(self, *args, **kwargs): return self.client.api.guilds_channels_create(self.id, *args, **kwargs) + + def leave(self): + return self.client.api.users_me_guilds_delete(self.id) From b96b7e5fa8fd599a7d8e859018f4d1a3e917b9fc Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 11 Apr 2017 07:38:34 -0700 Subject: [PATCH 09/63] Add Guild.owner to grab a guild owners member --- disco/types/guild.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/disco/types/guild.py b/disco/types/guild.py index 15c1c17..fb59ac5 100644 --- a/disco/types/guild.py +++ b/disco/types/guild.py @@ -313,6 +313,10 @@ class Guild(SlottedModel, Permissible): self.attach(six.itervalues(self.emojis), {'guild_id': self.id}) self.attach(six.itervalues(self.voice_states), {'guild_id': self.id}) + @cached_property + def owner(self): + return self.members.get(self.owner_id) + def get_permissions(self, member): """ Get the permissions a user has in this guild. From fa6cddc2f5e1bdfc886926a41145f2d470872a77 Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 12 Apr 2017 14:37:44 -0700 Subject: [PATCH 10/63] Add support for constructing webhook from a URL --- disco/types/webhook.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/disco/types/webhook.py b/disco/types/webhook.py index 4a630d3..e9c6080 100644 --- a/disco/types/webhook.py +++ b/disco/types/webhook.py @@ -1,8 +1,13 @@ +import re + from disco.types.base import SlottedModel, Field, snowflake from disco.types.user import User from disco.util.functional import cached_property +WEBHOOK_URL_RE = re.compile(r'\/api\/webhooks\/(\d+)\/(.[^/]+)') + + class Webhook(SlottedModel): id = Field(snowflake) guild_id = Field(snowflake) @@ -12,6 +17,14 @@ class Webhook(SlottedModel): avatar = Field(str) token = Field(str) + @classmethod + def from_url(cls, url): + results = WEBHOOK_URL_RE.findall(url) + if len(results) != 1: + return Exception('Invalid Webhook URL') + + return cls(id=results[0][0], token=results[0][1]) + @cached_property def guild(self): return self.client.state.guilds.get(self.guild_id) From 968745341b937383d7e7f011f56bc8d110185839 Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 12 Apr 2017 14:52:15 -0700 Subject: [PATCH 11/63] Create a non-authed client when using Webhook.from_url --- disco/api/http.py | 4 +++- disco/types/webhook.py | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/disco/api/http.py b/disco/api/http.py index 5dff893..b8d7e03 100644 --- a/disco/api/http.py +++ b/disco/api/http.py @@ -189,13 +189,15 @@ class HTTPClient(LoggingClass): self.limiter = RateLimiter() self.headers = { - 'Authorization': 'Bot ' + token, 'User-Agent': 'DiscordBot (https://github.com/b1naryth1ef/disco {}) Python/{} requests/{}'.format( disco_version, py_version, requests_version), } + if token: + self.headers['Authorization'] = 'Bot ' + token + def __call__(self, route, args=None, **kwargs): return self.call(route, args, **kwargs) diff --git a/disco/types/webhook.py b/disco/types/webhook.py index e9c6080..aba55a5 100644 --- a/disco/types/webhook.py +++ b/disco/types/webhook.py @@ -19,7 +19,9 @@ class Webhook(SlottedModel): @classmethod def from_url(cls, url): - results = WEBHOOK_URL_RE.findall(url) + from disco.api.client import APIClient + + results = WEBHOOK_URL_RE.findall(url, client=APIClient(None)) if len(results) != 1: return Exception('Invalid Webhook URL') @@ -45,10 +47,11 @@ class Webhook(SlottedModel): else: return self.client.api.webhooks_modify(self.id, name, avatar) - def execute(self, content=None, username=None, avatar_url=None, tts=False, fobj=None, embeds=[], wait=False): + def execute(self, content=None, username=None, avatar_url=None, tts=False, fobj=None, embeds=[], wait=False, client=None): # TODO: support file stuff properly + client = client or self.client.api - return self.client.api.webhooks_token_execute(self.id, self.token, { + return client.webhooks_token_execute(self.id, self.token, { 'content': content, 'username': username, 'avatar_url': avatar_url, From e9317e9ee443fe3739da192f191cb9ddc7502362 Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 12 Apr 2017 14:54:34 -0700 Subject: [PATCH 12/63] What am I even smoking --- disco/types/webhook.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/disco/types/webhook.py b/disco/types/webhook.py index aba55a5..b8930c7 100644 --- a/disco/types/webhook.py +++ b/disco/types/webhook.py @@ -18,14 +18,17 @@ class Webhook(SlottedModel): token = Field(str) @classmethod - def from_url(cls, url): + def execute_url(cls, url, **kwargs): from disco.api.client import APIClient - results = WEBHOOK_URL_RE.findall(url, client=APIClient(None)) + results = WEBHOOK_URL_RE.findall(url) if len(results) != 1: return Exception('Invalid Webhook URL') - return cls(id=results[0][0], token=results[0][1]) + return cls(id=results[0][0], token=results[0][1]).execute( + client=APIClient(None), + **kwargs + ) @cached_property def guild(self): From 3c3a318ae8781264f779614707248527b0998858 Mon Sep 17 00:00:00 2001 From: Andrei Date: Thu, 13 Apr 2017 11:38:22 -0700 Subject: [PATCH 13/63] Improve channels_messages_create function signature --- disco/api/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/disco/api/client.py b/disco/api/client.py index 254bb62..9f826f5 100644 --- a/disco/api/client.py +++ b/disco/api/client.py @@ -88,13 +88,15 @@ class APIClient(LoggingClass): r = self.http(Routes.CHANNELS_MESSAGES_GET, dict(channel=channel, message=message)) return Message.create(self.client, r.json()) - def channels_messages_create(self, channel, content, nonce=None, tts=False, attachment=None, embed=None): + def channels_messages_create(self, channel, content=None, nonce=None, tts=False, attachment=None, embed=None): payload = { - 'content': content, 'nonce': nonce, 'tts': tts, } + if content: + payload['content'] = content + if embed: payload['embed'] = embed.to_dict() From f2f9a968048706ee37a234efaa2fbae2182b4c1f Mon Sep 17 00:00:00 2001 From: Andrei Date: Thu, 13 Apr 2017 11:38:33 -0700 Subject: [PATCH 14/63] Add more visibility into resumes on GatewayClient --- disco/gateway/client.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/disco/gateway/client.py b/disco/gateway/client.py index 7386bb5..11c4c12 100644 --- a/disco/gateway/client.py +++ b/disco/gateway/client.py @@ -53,6 +53,8 @@ class GatewayClient(LoggingClass): self.session_id = None self.reconnects = 0 self.shutting_down = False + self.replaying = False + self.replayed_events = 0 # Cached gateway URL self._cached_gateway_url = None @@ -81,6 +83,8 @@ class GatewayClient(LoggingClass): obj = GatewayEvent.from_dispatch(self.client, packet) self.log.debug('Dispatching %s', obj.__class__.__name__) self.client.events.emit(obj.__class__.__name__, obj) + if self.replaying: + self.replayed_events += 1 def handle_heartbeat(self, _): self._send(OPCode.HEARTBEAT, self.seq) @@ -105,8 +109,9 @@ class GatewayClient(LoggingClass): self.reconnects = 0 def on_resumed(self, _): - self.log.info('Recieved RESUMED') + self.log.info('RESUME completed, replayed %s events', self.resumed_events) self.reconnects = 0 + self.replaying = False def connect_and_run(self, gateway_url=None): if not gateway_url: @@ -154,6 +159,7 @@ class GatewayClient(LoggingClass): def on_open(self): if self.seq and self.session_id: self.log.info('WS Opened: attempting resume w/ SID: %s SEQ: %s', self.session_id, self.seq) + self.replaying = True self.send(OPCode.RESUME, { 'token': self.client.config.token, 'session_id': self.session_id, @@ -188,6 +194,8 @@ class GatewayClient(LoggingClass): self.log.info('WS Closed: shutting down') return + self.replaying = False + # Track reconnect attempts self.reconnects += 1 self.log.info('WS Closed: [%s] %s (%s)', code, reason, self.reconnects) From b0854c096999555906a55fe7feb4deaf2d4c8f62 Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 14 Apr 2017 14:34:21 -0700 Subject: [PATCH 15/63] Remote duration argument-parser field Libs should just implement this manually. --- disco/bot/parser.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/disco/bot/parser.py b/disco/bot/parser.py index 722abe6..e9cabfc 100644 --- a/disco/bot/parser.py +++ b/disco/bot/parser.py @@ -15,12 +15,6 @@ TYPE_MAP = { 'snowflake': lambda ctx, data: int(data), } -try: - import dateparser - TYPE_MAP['duration'] = lambda ctx, data: dateparser.parse(data, settings={'TIMEZONE': 'UTC'}) -except ImportError: - pass - def to_bool(ctx, data): if data in BOOL_OPTS: From b340440ec12d63adc836a13aba21e717e7d1eebb Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 14 Apr 2017 14:35:06 -0700 Subject: [PATCH 16/63] Fix the way cached presences work There where various oddities and bugs here, namely: - Guild kept its initial `presences` list even though this is basically useless, and was never kept up-to-date - State did not properly use all the fields of presence_update, and was creating some annoying recursive oddities w/ user and presence binding --- disco/gateway/events.py | 1 + disco/state.py | 31 +++++++++++++++++++++++-------- disco/types/guild.py | 3 +-- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/disco/gateway/events.py b/disco/gateway/events.py index 9d9d323..d14a13d 100644 --- a/disco/gateway/events.py +++ b/disco/gateway/events.py @@ -153,6 +153,7 @@ class GuildCreate(GatewayEvent): and if None, this is a normal guild join event. """ unavailable = Field(bool) + presences = ListField(Presence) @property def created(self): diff --git a/disco/state.py b/disco/state.py index 2983aa0..560c34c 100644 --- a/disco/state.py +++ b/disco/state.py @@ -315,15 +315,30 @@ class State(object): self.guilds[event.guild_id].emojis = HashMap({i.id: i for i in event.emojis}) def on_presence_update(self, event): - if event.user.id in self.users: - self.users[event.user.id].update(event.presence.user) - self.users[event.user.id].presence = event.presence - event.presence.user = self.users[event.user.id] - - if event.guild_id not in self.guilds: + # Grab a copy of the user, and clear it out. All the operations below + # do not want the nested user object, as the normaly structure is + # user -> presence, and without clearing this becomes recursive + user = event.presence.user + user.presence = event.presence + event.presence.user = None + + # if we have the user tracked locally, we can just use the presence + # update to update both their presence and the cached user object. + if user.id in self.users: + self.users[user.id].update(user) + else: + # Otherwise this user does not exist in our local cache, so we can + # use this opportunity to add them. They will quickly fall out of + # scope and be deleted if they aren't used below + self.users[user.id] = user + + # Some updates come with a guild_id and roles the user is in, we should + # use this to update the guild member, but only if we have the guild + # cached. + if event.roles is UNSET or event.guild_id not in self.guilds: return - if event.user.id not in self.guilds[event.guild_id].members: + if user.id not in self.guilds[event.guild_id].members: return - self.guilds[event.guild_id].members[event.user.id].user.update(event.user) + self.guilds[event.guild_id].members[user.id].roles = event.roles diff --git a/disco/types/guild.py b/disco/types/guild.py index fb59ac5..10a0d6c 100644 --- a/disco/types/guild.py +++ b/disco/types/guild.py @@ -9,7 +9,7 @@ from disco.util.functional import cached_property from disco.types.base import ( SlottedModel, Field, ListField, AutoDictField, snowflake, text, binary, enum, datetime ) -from disco.types.user import User, Presence +from disco.types.user import User from disco.types.voice import VoiceState from disco.types.channel import Channel from disco.types.message import Emoji @@ -300,7 +300,6 @@ class Guild(SlottedModel, Permissible): emojis = AutoDictField(GuildEmoji, 'id') voice_states = AutoDictField(VoiceState, 'session_id') member_count = Field(int) - presences = ListField(Presence) synced = Field(bool, default=False) From 13c97211905a5c04c0aa7f9bd6f79cd2c980ab25 Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 14 Apr 2017 14:36:17 -0700 Subject: [PATCH 17/63] Users with administrator can execute all permissions This isn't totally true because of heiarchy, but we can think about how to handle that later as it requires interface changes. --- disco/types/permissions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/disco/types/permissions.py b/disco/types/permissions.py index 6e4d9f3..6157bd9 100644 --- a/disco/types/permissions.py +++ b/disco/types/permissions.py @@ -40,6 +40,10 @@ class PermissionValue(object): self.value = value def can(self, *perms): + # Administrator permission overwrites all others + if self.administrator: + return True + for perm in perms: if isinstance(perm, EnumAttr): perm = perm.value From e78c99b60561248589290f6020cc55369f8e15e0 Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 14 Apr 2017 14:58:24 -0700 Subject: [PATCH 18/63] Add tests/CI, fix playable import error on py3 --- .travis.yml | 12 ++++++++++++ disco/voice/playable.py | 2 +- setup.py | 9 +++++++++ tests/test_imports.py | 42 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 .travis.yml create mode 100644 tests/test_imports.py diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..61a1ecd --- /dev/null +++ b/.travis.yml @@ -0,0 +1,12 @@ +langauge: python + +python: + - '2.7' + - '3.2' + - '3.3' + - '3.4' + - '3.5' + - '3.6' + - 'nightly' + +script: 'python setup.py test' diff --git a/disco/voice/playable.py b/disco/voice/playable.py index aaeb7bd..ee9437b 100644 --- a/disco/voice/playable.py +++ b/disco/voice/playable.py @@ -14,7 +14,7 @@ from disco.voice.opus import OpusEncoder try: from cStringIO import cStringIO as StringIO except: - from StringIO import StringIO + from six import StringIO OPUS_HEADER_SIZE = struct.calcsize(' Date: Fri, 14 Apr 2017 15:05:10 -0700 Subject: [PATCH 19/63] Add some tests around embeds --- tests/test_embeds.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/test_embeds.py diff --git a/tests/test_embeds.py b/tests/test_embeds.py new file mode 100644 index 0000000..ef60361 --- /dev/null +++ b/tests/test_embeds.py @@ -0,0 +1,32 @@ +from unittest import TestCase + +from datetime import datetime +from disco.types.message import MessageEmbed + + +class TestEmbeds(TestCase): + def test_empty_embed(self): + embed = MessageEmbed() + self.assertDictEqual( + embed.to_dict(), + { + 'image': {}, + 'author': {}, + 'video': {}, + 'thumbnail': {}, + 'footer': {}, + 'fields': [], + 'type': 'rich', + }) + + def test_embed(self): + embed = MessageEmbed( + title='Test Title', + description='Test Description', + url='https://test.url/' + ) + obj = embed.to_dict() + self.assertEqual(obj['title'], 'Test Title') + self.assertEqual(obj['description'], 'Test Description') + self.assertEqual(obj['url'], 'https://test.url/') + From 5ed5f525edad87c3823993393dabbc78f4d4bb2f Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 14 Apr 2017 15:08:18 -0700 Subject: [PATCH 20/63] voice - only return metadata in property if we have metadata set --- disco/voice/playable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disco/voice/playable.py b/disco/voice/playable.py index ee9437b..2dbd6f7 100644 --- a/disco/voice/playable.py +++ b/disco/voice/playable.py @@ -46,7 +46,7 @@ class BaseUtil(object): @property def metadata(self): - return self._metadata + return getattr(self, '_metadata', None) @metadata.setter def metadata(self, value): From cc170e2cff2aaa71921e2e1c72c3cc137d66ea39 Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 14 Apr 2017 15:27:02 -0700 Subject: [PATCH 21/63] s/langauge/language --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 61a1ecd..451ca83 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -langauge: python +language: python python: - '2.7' From 6819ca6d6087a3a97ece4efec81accea8470d7ae Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 14 Apr 2017 15:39:44 -0700 Subject: [PATCH 22/63] Python 3.2 doesn't have unicode literal backwards-compat :rofl: --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 451ca83..fa4dc21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: python python: - '2.7' - - '3.2' - '3.3' - '3.4' - '3.5' From bcb3ac682ebd83b0b773f7a2884164828cc404b2 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 15 Apr 2017 13:16:52 -0700 Subject: [PATCH 23/63] Cache PIP w/ travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index fa4dc21..4e6a402 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ language: python +cache: pip + python: - '2.7' - '3.3' From 54222989ba5c04678c2f32a6472213251f4f08e5 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 15 Apr 2017 13:17:04 -0700 Subject: [PATCH 24/63] Switch from API v6 -> v7 to get more useful errors Might need to tweak APIException --- disco/api/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disco/api/http.py b/disco/api/http.py index b8d7e03..462ad8e 100644 --- a/disco/api/http.py +++ b/disco/api/http.py @@ -176,7 +176,7 @@ class HTTPClient(LoggingClass): A simple HTTP client which wraps the requests library, adding support for Discords rate-limit headers, authorization, and request/response validation. """ - BASE_URL = 'https://discordapp.com/api/v6' + BASE_URL = 'https://discordapp.com/api/v7' MAX_RETRIES = 5 def __init__(self, token): From 7ee41771b6fc7172aa53c989ada47978179030ca Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 15 Apr 2017 13:17:23 -0700 Subject: [PATCH 25/63] Update docs on Client.update_presence, fix Channel.send_message --- disco/client.py | 20 +++++++++++++++++--- disco/types/channel.py | 2 +- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/disco/client.py b/disco/client.py index a5791ee..a453d97 100644 --- a/disco/client.py +++ b/disco/client.py @@ -105,7 +105,21 @@ class Client(LoggingClass): localf=lambda: self.manhole_locals) self.manhole.start() - def update_presence(self, game=None, status=None, afk=False, since=0.0): + def update_presence(self, status, game=None, afk=False, since=0.0): + """ + Updates the current clients presence. + + Params + ------ + status : `user.Status` + The clients current status. + game : `user.Game` + If passed, the game object to set for the users presence. + afk : bool + Whether the client is currently afk. + since : float + How long the client has been afk for (in seconds). + """ if game and not isinstance(game, Game): raise TypeError('Game must be a Game model') @@ -126,12 +140,12 @@ class Client(LoggingClass): def run(self): """ - Run the client (e.g. the :class:`GatewayClient`) in a new greenlet. + Run the client (e.g. the `GatewayClient`) in a new greenlet. """ return gevent.spawn(self.gw.run) def run_forever(self): """ - Run the client (e.g. the :class:`GatewayClient`) in the current greenlet. + Run the client (e.g. the `GatewayClient`) in the current greenlet. """ return self.gw.run() diff --git a/disco/types/channel.py b/disco/types/channel.py index 311ca5c..1d373a3 100644 --- a/disco/types/channel.py +++ b/disco/types/channel.py @@ -244,7 +244,7 @@ class Channel(SlottedModel, Permissible): def create_webhook(self, name=None, avatar=None): return self.client.api.channels_webhooks_create(self.id, name, avatar) - def send_message(self, content, nonce=None, tts=False, attachment=None, embed=None): + def send_message(self, content=None, nonce=None, tts=False, attachment=None, embed=None): """ Send a message in this channel. From c41dca5512109c91aed13eed68bbe420277b5d12 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 18 Apr 2017 05:25:00 -0700 Subject: [PATCH 26/63] Make pynacl optional dependency --- disco/voice/client.py | 5 ++++- requirements-optional.txt | 1 + requirements.txt | 1 - 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 requirements-optional.txt diff --git a/disco/voice/client.py b/disco/voice/client.py index d0d3e09..0c9ce6c 100644 --- a/disco/voice/client.py +++ b/disco/voice/client.py @@ -3,7 +3,10 @@ import socket import struct import time -import nacl.secret +try: + import nacl.secret +except ImportError: + print 'WARNING: nacl is not installed, voice support is disabled' from holster.enum import Enum from holster.emitter import Emitter diff --git a/requirements-optional.txt b/requirements-optional.txt new file mode 100644 index 0000000..44182d9 --- /dev/null +++ b/requirements-optional.txt @@ -0,0 +1 @@ +pynacl==1.1.2 diff --git a/requirements.txt b/requirements.txt index ed9a25c..a22822d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,3 @@ inflection==0.3.1 requests==2.13.0 six==1.10.0 websocket-client==0.40.0 -pynacl==1.1.2 From a50f864c143345f3693b70f7ca94b193024072e2 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 18 Apr 2017 07:03:16 -0700 Subject: [PATCH 27/63] Ensure Unset is falsey --- disco/types/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/disco/types/base.py b/disco/types/base.py index c3a061b..4c0d916 100644 --- a/disco/types/base.py +++ b/disco/types/base.py @@ -25,6 +25,9 @@ class Unset(object): def __nonzero__(self): return False + def __bool__(self): + return False + UNSET = Unset() From 3dc6e6f99d11956c2b4474588612e76fc7a6bb38 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 18 Apr 2017 07:03:27 -0700 Subject: [PATCH 28/63] Fix possible edge case when bot has nickname and is mentioned Its valid for a user mention to be sent and parsed when a nickname mention is expected. As such we should filter both from the message contents to avoid message processing issues. --- disco/bot/bot.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/disco/bot/bot.py b/disco/bot/bot.py index 8ad0fca..2631cb0 100644 --- a/disco/bot/bot.py +++ b/disco/bot/bot.py @@ -267,7 +267,10 @@ class Bot(LoggingClass): if msg.guild: member = msg.guild.get_member(self.client.state.me) if member: - content = content.replace(member.mention, '', 1) + # If nickname is set, filter both the normal and nick mentions + if member.nick: + content = content.replace(member.mention, '', 1) + content = content.replace(member.user.mention, '', 1) else: content = content.replace(self.client.state.me.mention, '', 1) elif mention_everyone: From 517c0037eee0c16acdc65d88b8c5e766a03c780a Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 18 Apr 2017 07:05:06 -0700 Subject: [PATCH 29/63] Fix various Python 3 voice issues - Use print function for nacl library warning - Use BufferedIO instead of StringIO on Python3, we're dealing with bytes here - Fix MRO issue w/ BufferedOpusEncoder - Fix race condition on loading url information within YoutubeDLPlayable --- disco/voice/client.py | 4 ++- disco/voice/playable.py | 54 ++++++++++++++++++++++----------------- examples/music.py | 4 +-- requirements-optional.txt | 1 + 4 files changed, 36 insertions(+), 27 deletions(-) diff --git a/disco/voice/client.py b/disco/voice/client.py index 0c9ce6c..6f2eae3 100644 --- a/disco/voice/client.py +++ b/disco/voice/client.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import gevent import socket import struct @@ -6,7 +8,7 @@ import time try: import nacl.secret except ImportError: - print 'WARNING: nacl is not installed, voice support is disabled' + print('WARNING: nacl is not installed, voice support is disabled') from holster.enum import Enum from holster.emitter import Emitter diff --git a/disco/voice/playable.py b/disco/voice/playable.py index 2dbd6f7..98cbbc2 100644 --- a/disco/voice/playable.py +++ b/disco/voice/playable.py @@ -5,28 +5,24 @@ import gevent import struct import subprocess - +from gevent.lock import Semaphore from gevent.queue import Queue from disco.voice.opus import OpusEncoder try: - from cStringIO import cStringIO as StringIO + from cStringIO import cStringIO as BufferedIO except: - from six import StringIO + if six.PY2: + from StringIO import StringIO as BufferedIO + else: + from io import BytesIO as BufferedIO OPUS_HEADER_SIZE = struct.calcsize(' OpusPlayable -# FFMpegInput.youtube_dl('youtube.com/yolo').pipe(DCADOpusEncoder) => OpusPlayable -# FFMpegInput.youtube_dl('youtube.com/yolo').pipe(OpusEncoder).pipe(DuplexStream, open('cache_file.opus', 'w')) => OpusPlayable - - class AbstractOpus(object): def __init__(self, sampling_rate=48000, frame_length=20, channels=2): self.sampling_rate = sampling_rate @@ -116,7 +112,7 @@ class FFmpegInput(BaseInput, AbstractOpus): # First read blocks until the subprocess finishes if not self._buffer: data, _ = self.proc.communicate() - self._buffer = StringIO(data) + self._buffer = BufferedIO(data) # Subsequent reads can just do dis thang return self._buffer.read(sz) @@ -155,22 +151,24 @@ class YoutubeDLInput(FFmpegInput): self._url = url self._ie_info = ie_info self._info = None + self._info_lock = Semaphore() @property def info(self): - if not self._info: - import youtube_dl - ydl = youtube_dl.YoutubeDL({'format': 'webm[abr>0]/bestaudio/best'}) + with self._info_lock: + if not self._info: + import youtube_dl + ydl = youtube_dl.YoutubeDL({'format': 'webm[abr>0]/bestaudio/best'}) - if self._url: - obj = ydl.extract_info(self._url, download=False, process=False) - if 'entries' in obj: - self._ie_info = obj['entries'] - else: - self._ie_info = [obj] + if self._url: + obj = ydl.extract_info(self._url, download=False, process=False) + if 'entries' in obj: + self._ie_info = obj['entries'][0] + else: + self._ie_info = obj - self._info = ydl.process_ie_result(self._ie_info, download=False) - return self._info + self._info = ydl.process_ie_result(self._ie_info, download=False) + return self._info @property def _metadata(self): @@ -195,11 +193,19 @@ class YoutubeDLInput(FFmpegInput): return self.info['url'] -class BufferedOpusEncoderPlayable(BasePlayable, AbstractOpus, OpusEncoder): +class BufferedOpusEncoderPlayable(BasePlayable, OpusEncoder, AbstractOpus): def __init__(self, source, *args, **kwargs): self.source = source self.frames = Queue(kwargs.pop('queue_size', 4096)) - super(BufferedOpusEncoderPlayable, self).__init__(*args, **kwargs) + + # Call the AbstractOpus constructor, as we need properties it sets + AbstractOpus.__init__(self, *args, **kwargs) + + # Then call the OpusEncoder constructor, which requires some properties + # that AbstractOpus sets up + OpusEncoder.__init__(self, self.sampling_rate, self.channels) + + # Spawn the encoder loop gevent.spawn(self._encoder_loop) def _encoder_loop(self): diff --git a/examples/music.py b/examples/music.py index 63232b9..2c376c3 100644 --- a/examples/music.py +++ b/examples/music.py @@ -1,7 +1,7 @@ from disco.bot import Plugin from disco.bot.command import CommandError from disco.voice.player import Player -from disco.voice.playable import FFmpegInput, DCADOpusEncoderPlayable +from disco.voice.playable import YoutubeDLInput, BufferedOpusEncoderPlayable from disco.voice.client import VoiceException @@ -40,7 +40,7 @@ class MusicPlugin(Plugin): @Plugin.command('play', '') def on_play(self, event, url): - item = FFmpegInput.youtube_dl(url).pipe(DCADOpusEncoderPlayable) + item = YoutubeDLInput(url).pipe(BufferedOpusEncoderPlayable) self.get_player(event.guild.id).queue.put(item) @Plugin.command('pause') diff --git a/requirements-optional.txt b/requirements-optional.txt index 44182d9..5402825 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -1 +1,2 @@ pynacl==1.1.2 +youtube-dl==2017.4.17 From 4bab008f790256c41c762d73609078bc0a733dbd Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 18 Apr 2017 07:37:11 -0700 Subject: [PATCH 30/63] Fix BufferedOpusEncoderPlayable cutting off playback too soon Previously the signal for when BufferedOpusEncoderPlayable was completed, was when the encoding process finished. However considering we buffer the encoding, this would cause the playable to complete 4096 frames before it was actually done. We now properly just enqueue a None (aka complete signal) when encoding is done, so that playback will complete fully. --- disco/voice/playable.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/disco/voice/playable.py b/disco/voice/playable.py index 98cbbc2..758b357 100644 --- a/disco/voice/playable.py +++ b/disco/voice/playable.py @@ -217,10 +217,9 @@ class BufferedOpusEncoderPlayable(BasePlayable, OpusEncoder, AbstractOpus): self.frames.put(self.encode(raw, self.samples_per_frame)) gevent.idle() self.source = None + self.frames.put(None) def next_frame(self): - if not self.source: - return None return self.frames.get() From 1db1fc3bf3d7df361b1c987a8f70bae9a0432f33 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 18 Apr 2017 08:47:36 -0700 Subject: [PATCH 31/63] s/resumed_events/replayed_events typo --- disco/gateway/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disco/gateway/client.py b/disco/gateway/client.py index 11c4c12..8ed210f 100644 --- a/disco/gateway/client.py +++ b/disco/gateway/client.py @@ -109,7 +109,7 @@ class GatewayClient(LoggingClass): self.reconnects = 0 def on_resumed(self, _): - self.log.info('RESUME completed, replayed %s events', self.resumed_events) + self.log.info('RESUME completed, replayed %s events', self.replayed_events) self.reconnects = 0 self.replaying = False From 1a9b973a0cf1060354765eadf4229c2b8fb1bfb6 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 18 Apr 2017 11:26:02 -0700 Subject: [PATCH 32/63] content is optional for channels_messages_modify --- disco/api/client.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/disco/api/client.py b/disco/api/client.py index 9f826f5..e1adee7 100644 --- a/disco/api/client.py +++ b/disco/api/client.py @@ -109,10 +109,11 @@ class APIClient(LoggingClass): return Message.create(self.client, r.json()) - def channels_messages_modify(self, channel, message, content, embed=None): - payload = { - 'content': content, - } + def channels_messages_modify(self, channel, message, content=None, embed=None): + payload = {} + + if content: + payload['content'] = content if embed: payload['embed'] = embed.to_dict() From 5eeec20a75af773766ab15721cac1985fbddee60 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 18 Apr 2017 11:26:30 -0700 Subject: [PATCH 33/63] Fix overflowing timestamp in voice player This would cause heavy packet loss once our timestamp reached the int max. --- disco/voice/player.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/disco/voice/player.py b/disco/voice/player.py index 9abe075..bc1add2 100644 --- a/disco/voice/player.py +++ b/disco/voice/player.py @@ -7,6 +7,8 @@ from holster.emitter import Emitter from disco.voice.client import VoiceState +MAX_TIMESTAMP = 4294967295 + class Player(object): Events = Enum( @@ -93,6 +95,8 @@ class Player(object): self.client.send_frame(frame) self.client.timestamp += item.samples_per_frame + if self.client.timestamp > MAX_TIMESTAMP: + self.client.timestamp = 0 frame = item.next_frame() if frame is None: From bad953e58db2603cda36a9f562616fb1ae1933ce Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 18 Apr 2017 12:59:09 -0700 Subject: [PATCH 34/63] Fix smashing PresenceUpdate.user value in state module --- disco/client.py | 24 ++++++++++++------------ disco/state.py | 45 ++++++++++++++++++++++++--------------------- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/disco/client.py b/disco/client.py index a453d97..478f226 100644 --- a/disco/client.py +++ b/disco/client.py @@ -15,13 +15,13 @@ from disco.util.backdoor import DiscoBackdoorServer class ClientConfig(Config): """ - Configuration for the :class:`Client`. + Configuration for the `Client`. Attributes ---------- token : str Discord authentication token, can be validated using the - :func:`disco.util.token.is_valid_token` function. + `disco.util.token.is_valid_token` function. shard_id : int The shard ID for the current client instance. shard_count : int @@ -53,32 +53,32 @@ class Client(LoggingClass): """ Class representing the base entry point that should be used in almost all implementation cases. This class wraps the functionality of both the REST API - (:class:`disco.api.client.APIClient`) and the realtime gateway API - (:class:`disco.gateway.client.GatewayClient`). + (`disco.api.client.APIClient`) and the realtime gateway API + (`disco.gateway.client.GatewayClient`). Parameters ---------- - config : :class:`ClientConfig` + config : `ClientConfig` Configuration for this client instance. Attributes ---------- - config : :class:`ClientConfig` + config : `ClientConfig` The runtime configuration for this client. - events : :class:`Emitter` + events : `Emitter` An emitter which emits Gateway events. - packets : :class:`Emitter` + packets : `Emitter` An emitter which emits Gateway packets. - state : :class:`State` + state : `State` The state tracking object. - api : :class:`APIClient` + api : `APIClient` The API client. - gw : :class:`GatewayClient` + gw : `GatewayClient` The gateway client. manhole_locals : dict Dictionary of local variables for each manhole connection. This can be modified to add/modify local variables. - manhole : Optional[:class:`BackdoorServer`] + manhole : Optional[`BackdoorServer`] Gevent backdoor server (if the manhole is enabled). """ def __init__(self, config): diff --git a/disco/state.py b/disco/state.py index 560c34c..932bc33 100644 --- a/disco/state.py +++ b/disco/state.py @@ -1,4 +1,5 @@ import six +import copy import weakref import inflection @@ -42,10 +43,10 @@ class StateConfig(Config): find they do not need and may be experiencing memory pressure can disable this feature entirely using this attribute. track_messages_size : int - The size of the deque for each channel. Using this you can calculate the - total number of possible :class:`StackMessage` objects kept in memory, - using: `total_mesages_size * total_channels`. This can be tweaked based - on usage to help prevent memory pressure. + The size of the messages deque for each channel. This value can be used + to calculate the total number of possible `StackMessage` objects kept in + memory, simply: `total_messages_size * total_channels`. This value can + be tweaked based on usage and to help prevent memory pressure. sync_guild_members : bool If true, guilds will be automatically synced when they are initially loaded or joined. Generally this setting is OK for smaller bots, however bots in over @@ -60,31 +61,31 @@ class StateConfig(Config): class State(object): """ The State class is used to track global state based on events emitted from - the :class:`GatewayClient`. State tracking is a core component of the Disco - client, providing the mechanism for most of the higher-level utility functions. + the `GatewayClient`. State tracking is a core component of the Disco client, + providing the mechanism for most of the higher-level utility functions. Attributes ---------- EVENTS : list(str) A list of all events the State object binds to - client : :class:`disco.client.Client` + client : `disco.client.Client` The Client instance this state is attached to - config : :class:`StateConfig` + config : `StateConfig` The configuration for this state instance - me : :class:`disco.types.user.User` + me : `User` The currently logged in user - dms : dict(snowflake, :class:`disco.types.channel.Channel`) + dms : dict(snowflake, `Channel`) Mapping of all known DM Channels - guilds : dict(snowflake, :class:`disco.types.guild.Guild`) + guilds : dict(snowflake, `Guild`) Mapping of all known/loaded Guilds - channels : dict(snowflake, :class:`disco.types.channel.Channel`) + channels : dict(snowflake, `Channel`) Weak mapping of all known/loaded Channels - users : dict(snowflake, :class:`disco.types.user.User`) + users : dict(snowflake, `User`) Weak mapping of all known/loaded Users - voice_states : dict(str, :class:`disco.types.voice.VoiceState`) + voice_states : dict(str, `VoiceState`) Weak mapping of all known/active Voice States - messages : Optional[dict(snowflake, :class:`deque`)] - Mapping of channel ids to deques containing :class:`StackMessage` objects + messages : Optional[dict(snowflake, deque)] + Mapping of channel ids to deques containing `StackMessage` objects """ EVENTS = [ 'Ready', 'GuildCreate', 'GuildUpdate', 'GuildDelete', 'GuildMemberAdd', 'GuildMemberRemove', @@ -315,12 +316,14 @@ class State(object): self.guilds[event.guild_id].emojis = HashMap({i.id: i for i in event.emojis}) def on_presence_update(self, event): - # Grab a copy of the user, and clear it out. All the operations below - # do not want the nested user object, as the normaly structure is - # user -> presence, and without clearing this becomes recursive + # Grab a copy of the entire presence object, and clear the user out. All + # the operations below do not require or want a nested user object, and + # having it set just complicates the general structure. We copy so as + # to avoid interferring with other events that may require the nested + # user object. user = event.presence.user - user.presence = event.presence - event.presence.user = None + user.presence = copy.deepcopy(event.presence) + user.presence.user = None # if we have the user tracked locally, we can just use the presence # update to update both their presence and the cached user object. From 363d3b057cfae721779176802b364c55de45b574 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 18 Apr 2017 13:17:59 -0700 Subject: [PATCH 35/63] Revert presence stuff for now, --- disco/state.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/disco/state.py b/disco/state.py index 932bc33..37a38ff 100644 --- a/disco/state.py +++ b/disco/state.py @@ -1,5 +1,4 @@ import six -import copy import weakref import inflection @@ -316,14 +315,9 @@ class State(object): self.guilds[event.guild_id].emojis = HashMap({i.id: i for i in event.emojis}) def on_presence_update(self, event): - # Grab a copy of the entire presence object, and clear the user out. All - # the operations below do not require or want a nested user object, and - # having it set just complicates the general structure. We copy so as - # to avoid interferring with other events that may require the nested - # user object. + # TODO: this is recursive, we hackfix in model, but its still lame ATM user = event.presence.user - user.presence = copy.deepcopy(event.presence) - user.presence.user = None + user.presence = event.presence # if we have the user tracked locally, we can just use the presence # update to update both their presence and the cached user object. From 4fcdb025d03c5bd08194299d1a0e958541bcb227 Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 19 Apr 2017 07:10:01 -0700 Subject: [PATCH 36/63] Add a callback within FileProxyPlayable Useful for applications looking to confirm a fileproxyplayable has fully completed _before_ doing something with it (e.g. caching) --- disco/voice/playable.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/disco/voice/playable.py b/disco/voice/playable.py index 758b357..55d80fa 100644 --- a/disco/voice/playable.py +++ b/disco/voice/playable.py @@ -285,6 +285,7 @@ class DCADOpusEncoderPlayable(BasePlayable, AbstractOpus, OpusEncoder): class FileProxyPlayable(BasePlayable, AbstractOpus): def __init__(self, other, output, *args, **kwargs): self.flush = kwargs.pop('flush', False) + self.on_complete = kwargs.pop('on_complete', None) super(FileProxyPlayable, self).__init__(*args, **kwargs) self.other = other self.output = output @@ -300,6 +301,7 @@ class FileProxyPlayable(BasePlayable, AbstractOpus): self.output.flush() else: self.output.flush() + self.on_complete() self.output.close() return frame From 281e68623270d558f005837f6ecdd48a00652ba2 Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 19 Apr 2017 07:10:37 -0700 Subject: [PATCH 37/63] Add some additional requirement options --- requirements-optional.txt | 2 -- setup.py | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-) delete mode 100644 requirements-optional.txt diff --git a/requirements-optional.txt b/requirements-optional.txt deleted file mode 100644 index 5402825..0000000 --- a/requirements-optional.txt +++ /dev/null @@ -1,2 +0,0 @@ -pynacl==1.1.2 -youtube-dl==2017.4.17 diff --git a/setup.py b/setup.py index ef74c07..2b2a0f1 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,11 @@ with open('requirements.txt') as f: with open('README.md') as f: readme = f.read() +extras_require = { + 'voice': ['pynacl==1.1.2'], + 'performance': ['erlpack==0.3.2'], +} + setup( name='disco-py', author='b1nzy', @@ -27,6 +32,7 @@ setup( long_description=readme, include_package_data=True, install_requires=requirements, + extras_require=extras_require, test_suite='setup.run_tests', classifiers=[ 'Development Status :: 4 - Beta', From f155128bb06ff9a017c0347fa30f16c68360af9a Mon Sep 17 00:00:00 2001 From: Jamie Bishop Date: Fri, 21 Apr 2017 18:28:36 +0100 Subject: [PATCH 38/63] Fixed typo in example (#23) --- examples/basic_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/basic_plugin.py b/examples/basic_plugin.py index 57faae0..a48b21b 100644 --- a/examples/basic_plugin.py +++ b/examples/basic_plugin.py @@ -50,7 +50,7 @@ class BasicPlugin(Plugin): if not users: event.msg.reply("Couldn't find user for your query: `{}`".format(query)) elif len(users) > 1: - event.msg.reply('I found too many userse ({}) for your query: `{}`'.format(len(users), query)) + event.msg.reply('I found too many users ({}) for your query: `{}`'.format(len(users), query)) else: user = users[0] parts = [] From 43c183801100b90739ccc44601cb20a32ba4ffd8 Mon Sep 17 00:00:00 2001 From: Andrei Zbikowski Date: Fri, 21 Apr 2017 10:36:04 -0700 Subject: [PATCH 39/63] Refactor storage module, greatly simplify (#22) --- .gitignore | 1 + disco/api/client.py | 9 ++- disco/bot/bot.py | 7 +- disco/bot/providers/__init__.py | 15 ---- disco/bot/providers/base.py | 134 -------------------------------- disco/bot/providers/disk.py | 54 ------------- disco/bot/providers/memory.py | 5 -- disco/bot/providers/redis.py | 48 ------------ disco/bot/providers/rocksdb.py | 52 ------------- disco/bot/storage.py | 97 ++++++++++++++++++----- disco/types/channel.py | 4 +- disco/util/sanitize.py | 31 ++++++++ examples/storage.py | 33 ++++++++ 13 files changed, 157 insertions(+), 333 deletions(-) delete mode 100644 disco/bot/providers/__init__.py delete mode 100644 disco/bot/providers/base.py delete mode 100644 disco/bot/providers/disk.py delete mode 100644 disco/bot/providers/memory.py delete mode 100644 disco/bot/providers/redis.py delete mode 100644 disco/bot/providers/rocksdb.py create mode 100644 disco/util/sanitize.py create mode 100644 examples/storage.py diff --git a/.gitignore b/.gitignore index 87a6c02..1346a5f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ dist/ disco*.egg-info/ docs/_build storage.db +storage.json *.dca diff --git a/disco/api/client.py b/disco/api/client.py index e1adee7..448d2f7 100644 --- a/disco/api/client.py +++ b/disco/api/client.py @@ -3,6 +3,7 @@ import json from disco.api.http import Routes, HTTPClient from disco.util.logging import LoggingClass +from disco.util.sanitize import S from disco.types.user import User from disco.types.message import Message @@ -88,13 +89,15 @@ class APIClient(LoggingClass): r = self.http(Routes.CHANNELS_MESSAGES_GET, dict(channel=channel, message=message)) return Message.create(self.client, r.json()) - def channels_messages_create(self, channel, content=None, nonce=None, tts=False, attachment=None, embed=None): + def channels_messages_create(self, channel, content=None, nonce=None, tts=False, attachment=None, embed=None, sanitize=False): payload = { 'nonce': nonce, 'tts': tts, } if content: + if sanitize: + content = S(content) payload['content'] = content if embed: @@ -109,10 +112,12 @@ class APIClient(LoggingClass): return Message.create(self.client, r.json()) - def channels_messages_modify(self, channel, message, content=None, embed=None): + def channels_messages_modify(self, channel, message, content=None, embed=None, sanitize=False): payload = {} if content: + if sanitize: + content = S(content) payload['content'] = content if embed: diff --git a/disco/bot/bot.py b/disco/bot/bot.py index 2631cb0..e7ccad4 100644 --- a/disco/bot/bot.py +++ b/disco/bot/bot.py @@ -81,12 +81,13 @@ class BotConfig(Config): commands_group_abbrev = True plugin_config_provider = None - plugin_config_format = 'yaml' + plugin_config_format = 'json' plugin_config_dir = 'config' storage_enabled = True - storage_provider = 'memory' - storage_config = {} + storage_fsync = True + storage_serializer = 'json' + storage_path = 'storage.json' class Bot(LoggingClass): diff --git a/disco/bot/providers/__init__.py b/disco/bot/providers/__init__.py deleted file mode 100644 index 3ec985f..0000000 --- a/disco/bot/providers/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -import inspect -import importlib - -from .base import BaseProvider - - -def load_provider(name): - try: - mod = importlib.import_module('disco.bot.providers.' + name) - except ImportError: - mod = importlib.import_module(name) - - for entry in filter(inspect.isclass, map(lambda i: getattr(mod, i), dir(mod))): - if issubclass(entry, BaseProvider) and entry != BaseProvider: - return entry diff --git a/disco/bot/providers/base.py b/disco/bot/providers/base.py deleted file mode 100644 index 0f14f3d..0000000 --- a/disco/bot/providers/base.py +++ /dev/null @@ -1,134 +0,0 @@ -import six -import pickle - -from six.moves import map, UserDict - - -ROOT_SENTINEL = u'\u200B' -SEP_SENTINEL = u'\u200D' -OBJ_SENTINEL = u'\u200C' -CAST_SENTINEL = u'\u24EA' - - -def join_key(*args): - nargs = [] - for arg in args: - if not isinstance(arg, six.string_types): - arg = CAST_SENTINEL + pickle.dumps(arg) - nargs.append(arg) - return SEP_SENTINEL.join(nargs) - - -def true_key(key): - key = key.rsplit(SEP_SENTINEL, 1)[-1] - if key.startswith(CAST_SENTINEL): - return pickle.loads(key) - return key - - -class BaseProvider(object): - def __init__(self, config): - self.config = config - self.data = {} - - def exists(self, key): - return key in self.data - - def keys(self, other): - count = other.count(SEP_SENTINEL) + 1 - for key in self.data.keys(): - if key.startswith(other) and key.count(SEP_SENTINEL) == count: - yield key - - def get_many(self, keys): - for key in keys: - yield key, self.get(key) - - def get(self, key): - return self.data[key] - - def set(self, key, value): - self.data[key] = value - - def delete(self, key): - del self.data[key] - - def load(self): - pass - - def save(self): - pass - - def root(self): - return StorageDict(self) - - -class StorageDict(UserDict): - def __init__(self, parent_or_provider, key=None): - if isinstance(parent_or_provider, BaseProvider): - self.provider = parent_or_provider - self.parent = None - else: - self.parent = parent_or_provider - self.provider = self.parent.provider - self._key = key or ROOT_SENTINEL - - def keys(self): - return map(true_key, self.provider.keys(self.key)) - - def values(self): - for key in self.keys(): - yield self.provider.get(key) - - def items(self): - for key in self.keys(): - yield (true_key(key), self.provider.get(key)) - - def ensure(self, key, typ=dict): - if key not in self: - self[key] = typ() - return self[key] - - def update(self, obj): - for k, v in six.iteritems(obj): - self[k] = v - - @property - def data(self): - obj = {} - - for raw, value in self.provider.get_many(self.provider.keys(self.key)): - key = true_key(raw) - - if value == OBJ_SENTINEL: - value = self.__class__(self, key=key).data - obj[key] = value - return obj - - @property - def key(self): - if self.parent is not None: - return join_key(self.parent.key, self._key) - return self._key - - def __setitem__(self, key, value): - if isinstance(value, dict): - obj = self.__class__(self, key) - obj.update(value) - value = OBJ_SENTINEL - - self.provider.set(join_key(self.key, key), value) - - def __getitem__(self, key): - res = self.provider.get(join_key(self.key, key)) - - if res == OBJ_SENTINEL: - return self.__class__(self, key) - - return res - - def __delitem__(self, key): - return self.provider.delete(join_key(self.key, key)) - - def __contains__(self, key): - return self.provider.exists(join_key(self.key, key)) diff --git a/disco/bot/providers/disk.py b/disco/bot/providers/disk.py deleted file mode 100644 index af259e1..0000000 --- a/disco/bot/providers/disk.py +++ /dev/null @@ -1,54 +0,0 @@ -import os -import gevent - -from disco.util.serializer import Serializer -from .base import BaseProvider - - -class DiskProvider(BaseProvider): - def __init__(self, config): - super(DiskProvider, self).__init__(config) - self.format = config.get('format', 'pickle') - self.path = config.get('path', 'storage') + '.' + self.format - self.fsync = config.get('fsync', False) - self.fsync_changes = config.get('fsync_changes', 1) - - self.autosave_task = None - self.change_count = 0 - - def autosave_loop(self, interval): - while True: - gevent.sleep(interval) - self.save() - - def _on_change(self): - if self.fsync: - self.change_count += 1 - - if self.change_count >= self.fsync_changes: - self.save() - self.change_count = 0 - - def load(self): - if not os.path.exists(self.path): - return - - if self.config.get('autosave', True): - self.autosave_task = gevent.spawn( - self.autosave_loop, - self.config.get('autosave_interval', 120)) - - with open(self.path, 'r') as f: - self.data = Serializer.loads(self.format, f.read()) - - def save(self): - with open(self.path, 'w') as f: - f.write(Serializer.dumps(self.format, self.data)) - - def set(self, key, value): - super(DiskProvider, self).set(key, value) - self._on_change() - - def delete(self, key): - super(DiskProvider, self).delete(key) - self._on_change() diff --git a/disco/bot/providers/memory.py b/disco/bot/providers/memory.py deleted file mode 100644 index 17ad47b..0000000 --- a/disco/bot/providers/memory.py +++ /dev/null @@ -1,5 +0,0 @@ -from .base import BaseProvider - - -class MemoryProvider(BaseProvider): - pass diff --git a/disco/bot/providers/redis.py b/disco/bot/providers/redis.py deleted file mode 100644 index f5e1375..0000000 --- a/disco/bot/providers/redis.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import absolute_import - -import redis - -from itertools import izip - -from disco.util.serializer import Serializer -from .base import BaseProvider, SEP_SENTINEL - - -class RedisProvider(BaseProvider): - def __init__(self, config): - super(RedisProvider, self).__init__(config) - self.format = config.get('format', 'pickle') - self.conn = None - - def load(self): - self.conn = redis.Redis( - host=self.config.get('host', 'localhost'), - port=self.config.get('port', 6379), - db=self.config.get('db', 0)) - - def exists(self, key): - return self.conn.exists(key) - - def keys(self, other): - count = other.count(SEP_SENTINEL) + 1 - for key in self.conn.scan_iter(u'{}*'.format(other)): - key = key.decode('utf-8') - if key.count(SEP_SENTINEL) == count: - yield key - - def get_many(self, keys): - keys = list(keys) - if not len(keys): - raise StopIteration - - for key, value in izip(keys, self.conn.mget(keys)): - yield (key, Serializer.loads(self.format, value)) - - def get(self, key): - return Serializer.loads(self.format, self.conn.get(key)) - - def set(self, key, value): - self.conn.set(key, Serializer.dumps(self.format, value)) - - def delete(self, key): - self.conn.delete(key) diff --git a/disco/bot/providers/rocksdb.py b/disco/bot/providers/rocksdb.py deleted file mode 100644 index 986268d..0000000 --- a/disco/bot/providers/rocksdb.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import absolute_import - -import six -import rocksdb - -from itertools import izip -from six.moves import map - -from disco.util.serializer import Serializer -from .base import BaseProvider, SEP_SENTINEL - - -class RocksDBProvider(BaseProvider): - def __init__(self, config): - super(RocksDBProvider, self).__init__(config) - self.format = config.get('format', 'pickle') - self.path = config.get('path', 'storage.db') - self.db = None - - @staticmethod - def k(k): - return bytes(k) if six.PY3 else str(k.encode('utf-8')) - - def load(self): - self.db = rocksdb.DB(self.path, rocksdb.Options(create_if_missing=True)) - - def exists(self, key): - return self.db.get(self.k(key)) is not None - - # TODO prefix extractor - def keys(self, other): - count = other.count(SEP_SENTINEL) + 1 - it = self.db.iterkeys() - it.seek_to_first() - - for key in it: - key = key.decode('utf-8') - if key.startswith(other) and key.count(SEP_SENTINEL) == count: - yield key - - def get_many(self, keys): - for key, value in izip(keys, self.db.multi_get(list(map(self.k, keys)))): - yield (key, Serializer.loads(self.format, value.decode('utf-8'))) - - def get(self, key): - return Serializer.loads(self.format, self.db.get(self.k(key)).decode('utf-8')) - - def set(self, key, value): - self.db.put(self.k(key), Serializer.dumps(self.format, value)) - - def delete(self, key): - self.db.delete(self.k(key)) diff --git a/disco/bot/storage.py b/disco/bot/storage.py index 812d79c..5b03a07 100644 --- a/disco/bot/storage.py +++ b/disco/bot/storage.py @@ -1,26 +1,87 @@ -from .providers import load_provider +import os +from six.moves import UserDict -class Storage(object): - def __init__(self, ctx, config): +from disco.util.hashmap import HashMap +from disco.util.serializer import Serializer + + +class StorageHashMap(HashMap): + def __init__(self, data): + self.data = data + + +class ContextAwareProxy(UserDict): + def __init__(self, ctx): self.ctx = ctx - self.config = config - self.provider = load_provider(config.provider)(config.config) - self.provider.load() - self.root = self.provider.root() @property - def plugin(self): - return self.root.ensure('plugins').ensure(self.ctx['plugin'].name) + def data(self): + return self.ctx() - @property - def guild(self): - return self.plugin.ensure('guilds').ensure(self.ctx['guild'].id) - @property - def channel(self): - return self.plugin.ensure('channels').ensure(self.ctx['channel'].id) +class StorageDict(UserDict): + def __init__(self, parent, data): + self._parent = parent + self.data = data - @property - def user(self): - return self.plugin.ensure('users').ensure(self.ctx['user'].id) + def update(self, other): + self.data.update(other) + self._parent._update() + + def __setitem__(self, key, value): + self.data[key] = value + self._parent._update() + + def __delitem__(self, key): + del self.data[key] + self._parent._update() + + +class Storage(object): + def __init__(self, ctx, config): + self._ctx = ctx + self._path = config.path + self._serializer = config.serializer + self._fsync = config.fsync + self._data = {} + + if os.path.exists(self._path): + with open(self._path, 'r') as f: + self._data = Serializer.loads(self._serializer, f.read()) + + def __getitem__(self, key): + if key not in self._data: + self._data[key] = {} + return StorageHashMap(StorageDict(self, self._data[key])) + + def _update(self): + if self._fsync: + self.save() + + def save(self): + if not self._path: + return + + with open(self._path, 'w') as f: + f.write(Serializer.dumps(self._serializer, self._data)) + + def guild(self, key): + return ContextAwareProxy( + lambda: self['_g{}:{}'.format(self._ctx['guild'].id, key)] + ) + + def channel(self, key): + return ContextAwareProxy( + lambda: self['_c{}:{}'.format(self._ctx['channel'].id, key)] + ) + + def plugin(self, key): + return ContextAwareProxy( + lambda: self['_p{}:{}'.format(self._ctx['plugin'].name, key)] + ) + + def user(self, key): + return ContextAwareProxy( + lambda: self['_u{}:{}'.format(self._ctx['user'].id, key)] + ) diff --git a/disco/types/channel.py b/disco/types/channel.py index 1d373a3..48a10c1 100644 --- a/disco/types/channel.py +++ b/disco/types/channel.py @@ -244,7 +244,7 @@ class Channel(SlottedModel, Permissible): def create_webhook(self, name=None, avatar=None): return self.client.api.channels_webhooks_create(self.id, name, avatar) - def send_message(self, content=None, nonce=None, tts=False, attachment=None, embed=None): + def send_message(self, *args, **kwargs): """ Send a message in this channel. @@ -262,7 +262,7 @@ class Channel(SlottedModel, Permissible): :class:`disco.types.message.Message` The created message. """ - return self.client.api.channels_messages_create(self.id, content, nonce, tts, attachment, embed) + return self.client.api.channels_messages_create(self.id, *args, **kwargs) def connect(self, *args, **kwargs): """ diff --git a/disco/util/sanitize.py b/disco/util/sanitize.py new file mode 100644 index 0000000..e12287e --- /dev/null +++ b/disco/util/sanitize.py @@ -0,0 +1,31 @@ +import re + + +# Zero width (non-rendering) space that can be used to escape mentions +ZERO_WIDTH_SPACE = u'\u200B' + +# A grave-looking character that can be used to escape codeblocks +MODIFIER_GRAVE_ACCENT = u'\u02CB' + +# Regex which matches all possible mention combinations, this may be over-zealous +# but its better safe than sorry. +MENTION_RE = re.compile('<[@|#][!|&]?([0-9]+)>|@everyone') + + +def _re_sub_mention(mention): + if '#' in mention: + return ZERO_WIDTH_SPACE.join(mention.split('#', 1)) + elif '@' in mention: + return ZERO_WIDTH_SPACE.join(mention.split('@', 1)) + else: + return mention + + +def S(text, escape_mentions=True, escape_codeblocks=False): + if escape_mentions: + text = MENTION_RE.sub(_re_sub_mention, text) + + if escape_codeblocks: + text = text.replace('`', MODIFIER_GRAVE_ACCENT) + + return text diff --git a/examples/storage.py b/examples/storage.py new file mode 100644 index 0000000..c8e5ce3 --- /dev/null +++ b/examples/storage.py @@ -0,0 +1,33 @@ +from disco.bot import Plugin + + +class BasicPlugin(Plugin): + def load(self, ctx): + super(BasicPlugin, self).load(ctx) + self.tags = self.storage.guild('tags') + + @Plugin.command('add', ' ', group='tags') + def on_tags_add(self, event, name, value): + if name in self.tags: + return event.msg.reply('That tag already exists!') + + self.tags[name] = value + return event.msg.reply(u':ok_hand: created the tag {}'.format(name), sanitize=True) + + @Plugin.command('get', '', group='tags') + def on_tags_get(self, event, name): + if name not in self.tags: + return event.msg.reply('That tag does not exist!') + + return event.msg.reply(self.tags[name], sanitize=True) + + @Plugin.command('delete', '', group='tags', aliases=['del', 'rmv', 'remove']) + def on_tags_delete(self, event, name): + if name not in self.tags: + return event.msg.reply('That tag does not exist!') + + del self.tags[name] + + return event.msg.reply(u':ok_hand: I deleted the {} tag for you'.format( + name + ), sanitize=True) From db54d2a3921b3a7eae0c254e632121c19d62005a Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 21 Apr 2017 10:54:42 -0700 Subject: [PATCH 40/63] Fix issues with calculating abbreviations (closes #15) --- disco/bot/bot.py | 39 ++++++++++++++++++++++++++------------- tests/test_bot.py | 26 ++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 13 deletions(-) create mode 100644 tests/test_bot.py diff --git a/disco/bot/bot.py b/disco/bot/bot.py index e7ccad4..52b4aab 100644 --- a/disco/bot/bot.py +++ b/disco/bot/bot.py @@ -65,6 +65,7 @@ class BotConfig(Config): The directory plugin configuration is located within. """ levels = {} + plugins = [] plugin_config = {} commands_enabled = True @@ -196,28 +197,40 @@ class Bot(LoggingClass): Called when a plugin is loaded/unloaded to recompute internal state. """ if self.config.commands_group_abbrev: - self.compute_group_abbrev() + groups = set(command.group for command in self.commands if command.group) + self.group_abbrev = self.compute_group_abbrev(groups) self.compute_command_matches_re() - def compute_group_abbrev(self): + def compute_group_abbrev(self, groups): """ Computes all possible abbreviations for a command grouping. """ - self.group_abbrev = {} - groups = set(command.group for command in self.commands if command.group) - + # For the first pass, we just want to compute each groups possible + # abbreviations that don't conflict with eachother. + possible = {} for group in groups: - grp = group - while grp: - # If the group already exists, means someone else thought they - # could use it so we need yank it from them (and not use it) - if grp in list(six.itervalues(self.group_abbrev)): - self.group_abbrev = {k: v for k, v in six.iteritems(self.group_abbrev) if v != grp} + for index in range(len(group)): + current = group[:index] + if current in possible: + possible[current] = None else: - self.group_abbrev[group] = grp + possible[current] = group + + # Now, we want to compute the actual shortest abbreivation out of the + # possible ones + result = {} + for abbrev, group in six.iteritems(possible): + if not group: + continue - grp = grp[:-1] + if group in result: + if len(abbrev) < len(result[group]): + result[group] = abbrev + else: + result[group] = abbrev + + return result def compute_command_matches_re(self): """ diff --git a/tests/test_bot.py b/tests/test_bot.py new file mode 100644 index 0000000..5650dc2 --- /dev/null +++ b/tests/test_bot.py @@ -0,0 +1,26 @@ +from unittest import TestCase + +from disco.client import ClientConfig, Client +from disco.bot.bot import Bot + + +class TestBot(TestCase): + def setUp(self): + self.client = Client(ClientConfig( + {'config': 'TEST_TOKEN'} + )) + self.bot = Bot(self.client) + + def test_command_abbreviation(self): + groups = ['config', 'copy', 'copez', 'copypasta'] + result = self.bot.compute_group_abbrev(groups) + self.assertDictEqual(result, { + 'config': 'con', + 'copypasta': 'copy', + 'copez': 'cope', + }) + + def test_command_abbreivation_conflicting(self): + groups = ['cat', 'cap', 'caz', 'cas'] + result = self.bot.compute_group_abbrev(groups) + self.assertDictEqual(result, {}) From b82c3ab69a581864db768fde74502a1a6ea54c79 Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 21 Apr 2017 19:47:54 -0700 Subject: [PATCH 41/63] Add explicit_content_filter and default_message_notifications to guilds Previously missing. --- disco/types/guild.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/disco/types/guild.py b/disco/types/guild.py index 10a0d6c..e5d056d 100644 --- a/disco/types/guild.py +++ b/disco/types/guild.py @@ -23,6 +23,17 @@ VerificationLevel = Enum( HIGH=3, ) +ExplicitContentFilterLevel = Enum( + NONE=0, + WITHOUT_ROLES=1, + ALL=2 +) + +DefaultMessageNotificationsLevel = Enum( + ALL_MESSAGES=0, + ONLY_MENTIONS=1, +) + class GuildEmoji(Emoji): """ @@ -292,6 +303,8 @@ class Guild(SlottedModel, Permissible): afk_timeout = Field(int) embed_enabled = Field(bool) verification_level = Field(enum(VerificationLevel)) + explicit_content_filter = Field(enum(ExplicitContentFilterLevel)) + default_message_notifications = Field(enum(DefaultMessageNotificationsLevel)) mfa_level = Field(int) features = ListField(str) members = AutoDictField(GuildMember, 'id') From a52ab456b1e75a77f36267ceaa61a5ca5eda795d Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 21 Apr 2017 19:48:27 -0700 Subject: [PATCH 42/63] Deprecate Message.create_reaction in favor of Message.add_reaction This naming scheme was garbage and always tripped me up. In favor of having a more intuitive (and less _technically_ correct) interface, we'll swap the name up. --- disco/types/message.py | 7 +++++++ disco/util/logging.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/disco/types/message.py b/disco/types/message.py index 9b33d11..0d1f219 100644 --- a/disco/types/message.py +++ b/disco/types/message.py @@ -1,5 +1,6 @@ import re import six +import warnings import functools import unicodedata @@ -315,6 +316,12 @@ class Message(SlottedModel): ) def create_reaction(self, emoji): + warnings.warn( + 'Message.create_reaction will be deprecated soon, use Message.add_reaction', + DeprecationWarning) + return self.add_reaction(emoji) + + def add_reaction(self, emoji): if isinstance(emoji, Emoji): emoji = emoji.to_string() self.client.api.channels_messages_reactions_create( diff --git a/disco/util/logging.py b/disco/util/logging.py index 75e9229..61a7b42 100644 --- a/disco/util/logging.py +++ b/disco/util/logging.py @@ -1,5 +1,6 @@ from __future__ import absolute_import +import warnings import logging @@ -13,7 +14,14 @@ LOG_FORMAT = '[%(levelname)s] %(asctime)s - %(name)s:%(lineno)d - %(message)s' def setup_logging(**kwargs): kwargs.setdefault('format', LOG_FORMAT) + # Setup warnings module correctly + warnings.simplefilter('always', DeprecationWarning) + logging.captureWarnings(True) + + # Pass through our basic configuration logging.basicConfig(**kwargs) + + # Override some noisey loggers for logger, level in LEVEL_OVERRIDES.items(): logging.getLogger(logger).setLevel(level) From a8a2c64e9591c2d97f08cd306b3708798115bdf0 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 22 Apr 2017 00:34:02 -0700 Subject: [PATCH 43/63] Add support for multiple attachments This also deprecates the attachment keyword argument. --- disco/api/client.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/disco/api/client.py b/disco/api/client.py index 448d2f7..45d765f 100644 --- a/disco/api/client.py +++ b/disco/api/client.py @@ -1,5 +1,6 @@ import six import json +import warnings from disco.api.http import Routes, HTTPClient from disco.util.logging import LoggingClass @@ -89,12 +90,20 @@ class APIClient(LoggingClass): r = self.http(Routes.CHANNELS_MESSAGES_GET, dict(channel=channel, message=message)) return Message.create(self.client, r.json()) - def channels_messages_create(self, channel, content=None, nonce=None, tts=False, attachment=None, embed=None, sanitize=False): + def channels_messages_create(self, channel, content=None, nonce=None, tts=False, + attachment=None, attachments=[], embed=None, sanitize=False): + payload = { 'nonce': nonce, 'tts': tts, } + if attachment: + attachments = [attachment] + warnings.warn( + 'attachment kwarg has been deprecated, switch to using attachments with a list', + DeprecationWarning) + if content: if sanitize: content = S(content) @@ -103,10 +112,22 @@ class APIClient(LoggingClass): if embed: payload['embed'] = embed.to_dict() - if attachment: - r = self.http(Routes.CHANNELS_MESSAGES_CREATE, dict(channel=channel), data={'payload_json': json.dumps(payload)}, files={ - 'file': (attachment[0], attachment[1]) - }) + if attachments: + if len(attachments) > 1: + files = { + 'file{}'.format(idx): tuple(i) for idx, i in enumerate(attachments) + } + else: + files = { + 'file': tuple(attachments[0]), + } + + r = self.http( + Routes.CHANNELS_MESSAGES_CREATE, + dict(channel=channel), + data={'payload_json': json.dumps(payload)}, + files=files + ) else: r = self.http(Routes.CHANNELS_MESSAGES_CREATE, dict(channel=channel), json=payload) From bd431d949e670bf0aff247661df5b6623c62b2e4 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 22 Apr 2017 15:15:21 -0700 Subject: [PATCH 44/63] Bump holster version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a22822d..cd7837b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ gevent==1.2.1 -holster==1.0.11 +holster==1.0.12 inflection==0.3.1 requests==2.13.0 six==1.10.0 From 701b0d2437408076f67246cf07a14c83f8d7700d Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 24 Apr 2017 12:56:15 -0700 Subject: [PATCH 45/63] Bump holster, fix bug with sanitizing mentions --- disco/util/sanitize.py | 3 ++- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/disco/util/sanitize.py b/disco/util/sanitize.py index e12287e..e5ecf74 100644 --- a/disco/util/sanitize.py +++ b/disco/util/sanitize.py @@ -9,10 +9,11 @@ MODIFIER_GRAVE_ACCENT = u'\u02CB' # Regex which matches all possible mention combinations, this may be over-zealous # but its better safe than sorry. -MENTION_RE = re.compile('<[@|#][!|&]?([0-9]+)>|@everyone') +MENTION_RE = re.compile('<([@|#][!|&]?[0-9]+>|@everyone|@here)') def _re_sub_mention(mention): + mention = mention.group(1) if '#' in mention: return ZERO_WIDTH_SPACE.join(mention.split('#', 1)) elif '@' in mention: diff --git a/requirements.txt b/requirements.txt index cd7837b..a489053 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ gevent==1.2.1 -holster==1.0.12 +holster==1.0.13 inflection==0.3.1 requests==2.13.0 six==1.10.0 From 7256ff35fc8d00777879c20f649ece5dfa9fbb67 Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 24 Apr 2017 12:56:40 -0700 Subject: [PATCH 46/63] Fix API-returned GuildEmoji's not having guild_id set --- disco/api/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/disco/api/client.py b/disco/api/client.py index 45d765f..bf0ec30 100644 --- a/disco/api/client.py +++ b/disco/api/client.py @@ -324,11 +324,11 @@ class APIClient(LoggingClass): def guilds_emojis_create(self, guild, **kwargs): r = self.http(Routes.GUILDS_EMOJIS_CREATE, dict(guild=guild), json=kwargs) - return GuildEmoji.create(self.client, r.json()) + return GuildEmoji.create(self.client, r.json(), guild_id=guild) def guilds_emojis_modify(self, guild, emoji, **kwargs): r = self.http(Routes.GUILDS_EMOJIS_MODIFY, dict(guild=guild, emoji=emoji), json=kwargs) - return GuildEmoji.create(self.client, r.json()) + return GuildEmoji.create(self.client, r.json(), guild_id=guild) def guilds_emojis_delete(self, guild, emoji): self.http(Routes.GUILDS_EMOJIS_DELETE, dict(guild=guild, emoji=emoji)) From 1f50274554995500ba178c6b39ec35963a501fda Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 24 Apr 2017 12:57:02 -0700 Subject: [PATCH 47/63] Fix users loaded in GuildCreate missing presence information The presence data comes sep. from the users info here, so we need to load it. --- disco/state.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/disco/state.py b/disco/state.py index 37a38ff..f914b29 100644 --- a/disco/state.py +++ b/disco/state.py @@ -186,6 +186,9 @@ class State(object): for member in six.itervalues(event.guild.members): self.users[member.user.id] = member.user + for presence in event.presences: + self.users[presence.user.id].presence = presence + for voice_state in six.itervalues(event.guild.voice_states): self.voice_states[voice_state.session_id] = voice_state From 33f77869d5cbf51db8d93420109cac3967e01bef Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 24 Apr 2017 12:57:37 -0700 Subject: [PATCH 48/63] Add GuildEmoji.delete --- disco/types/guild.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/disco/types/guild.py b/disco/types/guild.py index e5d056d..cbd9ff0 100644 --- a/disco/types/guild.py +++ b/disco/types/guild.py @@ -65,6 +65,9 @@ class GuildEmoji(Emoji): def update(self, **kwargs): return self.client.api.guilds_emojis_modify(self.guild_id, self.id, **kwargs) + def delete(self): + return self.client.api.guilds_emojis_delete(self.guild_id, self.id) + @property def url(self): return 'https://discordapp.com/api/emojis/{}.png'.format(self.id) From 2970466311248978a15f3bc3a0ab6e7f55f72614 Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 24 Apr 2017 15:30:43 -0700 Subject: [PATCH 49/63] Add Chainable/async utils --- disco/types/base.py | 17 +++++------ disco/util/chains.py | 71 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 10 deletions(-) create mode 100644 disco/util/chains.py diff --git a/disco/types/base.py b/disco/types/base.py index 4c0d916..d396762 100644 --- a/disco/types/base.py +++ b/disco/types/base.py @@ -6,8 +6,9 @@ import functools from holster.enum import BaseEnumMeta, EnumAttr from datetime import datetime as real_datetime -from disco.util.functional import CachedSlotProperty +from disco.util.chains import Chainable from disco.util.hashmap import HashMap +from disco.util.functional import CachedSlotProperty DATETIME_FORMATS = [ '%Y-%m-%dT%H:%M:%S.%f', @@ -273,15 +274,7 @@ class ModelMeta(type): return super(ModelMeta, mcs).__new__(mcs, name, parents, dct) -class AsyncChainable(object): - __slots__ = [] - - def after(self, delay): - gevent.sleep(delay) - return self - - -class Model(six.with_metaclass(ModelMeta, AsyncChainable)): +class Model(six.with_metaclass(ModelMeta, Chainable)): __slots__ = ['client'] def __init__(self, *args, **kwargs): @@ -297,6 +290,10 @@ class Model(six.with_metaclass(ModelMeta, AsyncChainable)): self.load(obj) self.validate() + def after(self, delay): + gevent.sleep(delay) + return self + def validate(self): pass diff --git a/disco/util/chains.py b/disco/util/chains.py new file mode 100644 index 0000000..5ea261f --- /dev/null +++ b/disco/util/chains.py @@ -0,0 +1,71 @@ +import gevent + +""" +Object.chain -> creates a chain where each action happens after the last + pass_result = False -> whether the result of the last action is passed, or the original + +Object.async_chain -> creates an async chain where each action happens at the same time +""" + + +class Chainable(object): + __slots__ = [] + + def chain(self, pass_result=True): + return Chain(self, pass_result=pass_result, async_=False) + + def async_chain(self): + return Chain(self, pass_result=False, async_=True) + + +class Chain(object): + def __init__(self, obj, pass_result=True, async_=False): + self._obj = obj + self._pass_result = pass_result + self._async = async_ + self._parts = [] + + @property + def obj(self): + if isinstance(self._obj, Chain): + return self._obj._next() + return self._obj + + def __getattr__(self, item): + func = getattr(self.obj, item) + if not func or not callable(func): + return func + + def _wrapped(*args, **kwargs): + inst = gevent.spawn(func, *args, **kwargs) + self._parts.append(inst) + + # If async, just return instantly + if self._async: + return self + + # Otherwise return a chain + return Chain(self) + return _wrapped + + def _next(self): + res = self._parts[0].get() + if self._pass_result: + return res + return self + + def then(self, func, *args, **kwargs): + inst = gevent.spawn(func, *args, **kwargs) + self._parts.append(inst) + if self._async: + return self + return Chain(self) + + def first(self): + return self._obj + + def get(self, timeout=None): + return gevent.wait(self._parts, timeout=timeout) + + def wait(self, timeout=None): + gevent.joinall(self._parts, timeout=None) From d9f619f0075919301264b6656aa6c88619d93fba Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 24 Apr 2017 15:31:01 -0700 Subject: [PATCH 50/63] Rename Invite.create -> create_for_channel This was smashing a base model method --- disco/types/invite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/disco/types/invite.py b/disco/types/invite.py index 0cc4852..7f22a57 100644 --- a/disco/types/invite.py +++ b/disco/types/invite.py @@ -1,4 +1,4 @@ -from disco.types.base import SlottedModel, Field, datetime +from disco.types.base import SlottedModel, Field, datetime from disco.types.user import User from disco.types.guild import Guild from disco.types.channel import Channel @@ -40,7 +40,7 @@ class Invite(SlottedModel): created_at = Field(datetime) @classmethod - def create(cls, channel, max_age=86400, max_uses=0, temporary=False, unique=False): + def create_for_channel(cls, channel, max_age=86400, max_uses=0, temporary=False, unique=False): return channel.client.api.channels_invites_create( channel.id, max_age=max_age, From 5ca596b889b7ba076cf504a752ff8eb87bc3f2b0 Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 24 Apr 2017 15:31:47 -0700 Subject: [PATCH 51/63] Add ability to pass conditional to Plugin.wait_for_event --- disco/bot/plugin.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/disco/bot/plugin.py b/disco/bot/plugin.py index aeed5b3..088f4da 100644 --- a/disco/bot/plugin.py +++ b/disco/bot/plugin.py @@ -209,15 +209,22 @@ class Plugin(LoggingClass, PluginDeco): def handle_exception(self, greenlet, event): pass - def wait_for_event(self, event_name, **kwargs): + def wait_for_event(self, event_name, conditional=None, **kwargs): result = AsyncResult() listener = None def _event_callback(event): for k, v in kwargs.items(): - if getattr(event, k) != v: + obj = event + for inst in k.split('__'): + obj = getattr(obj, inst) + + if obj != v: break else: + if conditional and not conditional(event): + return + listener.remove() return result.set(event) From 8687a47c146fd824b3985bafd98344d125601002 Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 24 Apr 2017 15:32:17 -0700 Subject: [PATCH 52/63] Add Guild.get_invites, Guild.get_emojis, MessageReactionAdd.delete --- disco/api/client.py | 4 ++++ disco/gateway/events.py | 8 ++++++++ disco/types/guild.py | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/disco/api/client.py b/disco/api/client.py index bf0ec30..04b9b32 100644 --- a/disco/api/client.py +++ b/disco/api/client.py @@ -314,6 +314,10 @@ class APIClient(LoggingClass): def guilds_roles_delete(self, guild, role): self.http(Routes.GUILDS_ROLES_DELETE, dict(guild=guild, role=role)) + def guilds_invites_list(self, guild): + r = self.http(Routes.GUILDS_INVITES_LIST, dict(guild=guild)) + return Invite.create_map(self.client, r.json()) + def guilds_webhooks_list(self, guild): r = self.http(Routes.GUILDS_WEBHOOKS_LIST, dict(guild=guild)) return Webhook.create_map(self.client, r.json()) diff --git a/disco/gateway/events.py b/disco/gateway/events.py index d14a13d..3f90ea8 100644 --- a/disco/gateway/events.py +++ b/disco/gateway/events.py @@ -610,6 +610,14 @@ class MessageReactionAdd(GatewayEvent): user_id = Field(snowflake) emoji = Field(MessageReactionEmoji) + def delete(self): + self.client.api.channels_messages_reactions_delete( + self.channel_id, + self.message_id, + self.emoji, + self.uesr_id + ) + @property def channel(self): return self.client.state.channels.get(self.channel_id) diff --git a/disco/types/guild.py b/disco/types/guild.py index cbd9ff0..594d56a 100644 --- a/disco/types/guild.py +++ b/disco/types/guild.py @@ -442,3 +442,9 @@ class Guild(SlottedModel, Permissible): def leave(self): return self.client.api.users_me_guilds_delete(self.id) + + def get_invites(self): + return self.client.api.guilds_invites_list(self.id) + + def get_emojis(self): + return self.client.api.guilds_emojis_list(self.id) From c7a4b25c7bca83349faa087f336bf551e4f8bf46 Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 24 Apr 2017 15:40:52 -0700 Subject: [PATCH 53/63] s/uesr_id/user_id --- disco/gateway/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disco/gateway/events.py b/disco/gateway/events.py index 3f90ea8..dd38ae7 100644 --- a/disco/gateway/events.py +++ b/disco/gateway/events.py @@ -615,7 +615,7 @@ class MessageReactionAdd(GatewayEvent): self.channel_id, self.message_id, self.emoji, - self.uesr_id + self.user_id ) @property From 34615cc398ca41f6be0c51173bc3c322f56932f3 Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 24 Apr 2017 15:42:52 -0700 Subject: [PATCH 54/63] Use proper URL format for deleting message reaction --- disco/gateway/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disco/gateway/events.py b/disco/gateway/events.py index dd38ae7..f2da195 100644 --- a/disco/gateway/events.py +++ b/disco/gateway/events.py @@ -614,7 +614,7 @@ class MessageReactionAdd(GatewayEvent): self.client.api.channels_messages_reactions_delete( self.channel_id, self.message_id, - self.emoji, + self.emoji.to_string() if self.emoji.id else self.emoji.name, self.user_id ) From fb9128a0926558d3ed2f94cb75df64a6d047f818 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 25 Apr 2017 06:55:05 -0700 Subject: [PATCH 55/63] Bump holster version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a489053..a0de4f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ gevent==1.2.1 -holster==1.0.13 +holster==1.0.14 inflection==0.3.1 requests==2.13.0 six==1.10.0 From 3410849543ee60ff7a57ddfa0ff975f0ce308747 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 25 Apr 2017 06:55:18 -0700 Subject: [PATCH 56/63] Fix cases where member presence can be stomped on --- disco/state.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/disco/state.py b/disco/state.py index f914b29..848665b 100644 --- a/disco/state.py +++ b/disco/state.py @@ -184,7 +184,8 @@ class State(object): self.channels.update(event.guild.channels) for member in six.itervalues(event.guild.members): - self.users[member.user.id] = member.user + if member.user.id not in self.users: + self.users[member.user.id] = member.user for presence in event.presences: self.users[presence.user.id].presence = presence @@ -285,7 +286,8 @@ class State(object): for member in event.members: member.guild_id = guild.id guild.members[member.id] = member - self.users[member.id] = member.user + if member.id not in self.users: + self.users[member.id] = member.user def on_guild_role_create(self, event): if event.guild_id not in self.guilds: From a544a6d020448e582432b1bf2a958d69ee0b658e Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 25 Apr 2017 06:55:32 -0700 Subject: [PATCH 57/63] Add snowflake-utils from_datetime and from_timestamp --- disco/util/snowflake.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/disco/util/snowflake.py b/disco/util/snowflake.py index b2f512f..2fa915d 100644 --- a/disco/util/snowflake.py +++ b/disco/util/snowflake.py @@ -20,6 +20,14 @@ def to_unix_ms(snowflake): return (int(snowflake) >> 22) + DISCORD_EPOCH +def from_datetime(date): + return from_timestamp(int(date.isoformat('%s'))) + + +def from_timestamp(ts): + return long(ts * 1000.0 - DISCORD_EPOCH) << 22 + + def to_snowflake(i): if isinstance(i, six.integer_types): return i From 898ba53f27d46ce2f029cf4878aff2a9cbc789b6 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 25 Apr 2017 07:00:09 -0700 Subject: [PATCH 58/63] s/isoformat/strftime --- disco/util/snowflake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disco/util/snowflake.py b/disco/util/snowflake.py index 2fa915d..a1e041d 100644 --- a/disco/util/snowflake.py +++ b/disco/util/snowflake.py @@ -21,7 +21,7 @@ def to_unix_ms(snowflake): def from_datetime(date): - return from_timestamp(int(date.isoformat('%s'))) + return from_timestamp(int(date.strftime('%s'))) def from_timestamp(ts): From 9da191ae91c57c6b93ceb67797d5246d50980fa2 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 25 Apr 2017 07:27:03 -0700 Subject: [PATCH 59/63] Proper time conversion --- disco/util/snowflake.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/disco/util/snowflake.py b/disco/util/snowflake.py index a1e041d..a9aeb15 100644 --- a/disco/util/snowflake.py +++ b/disco/util/snowflake.py @@ -2,6 +2,7 @@ import six from datetime import datetime +UNIX_EPOCH = datetime(1970, 1, 1) DISCORD_EPOCH = 1420070400000 @@ -21,7 +22,7 @@ def to_unix_ms(snowflake): def from_datetime(date): - return from_timestamp(int(date.strftime('%s'))) + return from_timestamp((date - UNIX_EPOCH).total_seconds()) def from_timestamp(ts): From 216197790247588413de27f00c2d6690e7d4d656 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 25 Apr 2017 08:10:55 -0700 Subject: [PATCH 60/63] Fix issue with sanitization --- disco/util/sanitize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/disco/util/sanitize.py b/disco/util/sanitize.py index e5ecf74..23a11e4 100644 --- a/disco/util/sanitize.py +++ b/disco/util/sanitize.py @@ -9,15 +9,15 @@ MODIFIER_GRAVE_ACCENT = u'\u02CB' # Regex which matches all possible mention combinations, this may be over-zealous # but its better safe than sorry. -MENTION_RE = re.compile('<([@|#][!|&]?[0-9]+>|@everyone|@here)') +MENTION_RE = re.compile('?') def _re_sub_mention(mention): mention = mention.group(1) if '#' in mention: - return ZERO_WIDTH_SPACE.join(mention.split('#', 1)) + return (u'#' + ZERO_WIDTH_SPACE).join(mention.split('#', 1)) elif '@' in mention: - return ZERO_WIDTH_SPACE.join(mention.split('@', 1)) + return (u'@' + ZERO_WIDTH_SPACE).join(mention.split('@', 1)) else: return mention From 01820d2bc9bcc85e56daf2289b2124c8d8bd7862 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 25 Apr 2017 14:23:47 -0700 Subject: [PATCH 61/63] Fix some issues with the way command regexes work - Remove the limitation that prevented us from surpassing 100 commands - Fix CommandEvent.name being the full command string, and not just the command name - Fix the error thrown on mismatched args being utter trash --- disco/bot/bot.py | 2 +- disco/bot/command.py | 22 +++++++++++++--------- tests/test_bot.py | 23 ++++++++++++++++++++++- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/disco/bot/bot.py b/disco/bot/bot.py index 52b4aab..18260b9 100644 --- a/disco/bot/bot.py +++ b/disco/bot/bot.py @@ -237,7 +237,7 @@ class Bot(LoggingClass): Computes a single regex which matches all possible command combinations. """ commands = list(self.commands) - re_str = '|'.join(command.regex for command in commands) + re_str = '|'.join(command.regex(grouped=False) for command in commands) if re_str: self.command_matches_re = re.compile(re_str, re.I) else: diff --git a/disco/bot/command.py b/disco/bot/command.py index c5aaddf..9a03ebe 100644 --- a/disco/bot/command.py +++ b/disco/bot/command.py @@ -6,6 +6,7 @@ from disco.bot.parser import ArgumentSet, ArgumentError from disco.util.functional import cached_property ARGS_REGEX = '(?: ((?:\n|.)*)$|$)' +ARGS_UNGROUPED_REGEX = '(?: (?:\n|.)*$|$)' USER_MENTION_RE = re.compile('<@!?([0-9]+)>') ROLE_MENTION_RE = re.compile('<@&([0-9]+)>') @@ -44,11 +45,11 @@ class CommandEvent(object): self.command = command self.msg = msg self.match = match - self.name = self.match.group(0) + self.name = self.match.group(1).strip() self.args = [] - if self.match.group(1): - self.args = [i for i in self.match.group(1).strip().split(' ') if i] + if self.match.group(2): + self.args = [i for i in self.match.group(2).strip().split(' ') if i] @property def codeblock(self): @@ -222,10 +223,9 @@ class Command(object): """ A compiled version of this command's regex. """ - return re.compile(self.regex, re.I) + return re.compile(self.regex(), re.I) - @property - def regex(self): + def regex(self, grouped=True): """ The regex string that defines/triggers this command. """ @@ -235,10 +235,13 @@ class Command(object): group = '' if self.group: if self.group in self.plugin.bot.group_abbrev: - group = '{}(?:\w+)? '.format(self.plugin.bot.group_abbrev.get(self.group)) + group = '(?:\w+)? '.format(self.plugin.bot.group_abbrev.get(self.group)) else: group = self.group + ' ' - return '^{}(?:{})'.format(group, '|'.join(self.triggers)) + ARGS_REGEX + return ('^{}({})' if grouped else '^{}(?:{})').format( + group, + '|'.join(self.triggers) + ) + (ARGS_REGEX if grouped else ARGS_UNGROUPED_REGEX) def execute(self, event): """ @@ -251,9 +254,10 @@ class Command(object): Whether this command was successful """ if len(event.args) < self.args.required_length: - raise CommandError('{} requires {} arguments (passed {})'.format( + raise CommandError(u'Command {} requires {} arguments (`{}`) passed {}'.format( event.name, self.args.required_length, + self.raw_args, len(event.args) )) diff --git a/tests/test_bot.py b/tests/test_bot.py index 5650dc2..566de0a 100644 --- a/tests/test_bot.py +++ b/tests/test_bot.py @@ -2,6 +2,13 @@ from unittest import TestCase from disco.client import ClientConfig, Client from disco.bot.bot import Bot +from disco.bot.command import Command + + +class MockBot(Bot): + @property + def commands(self): + return getattr(self, '_commands', []) class TestBot(TestCase): @@ -9,7 +16,7 @@ class TestBot(TestCase): self.client = Client(ClientConfig( {'config': 'TEST_TOKEN'} )) - self.bot = Bot(self.client) + self.bot = MockBot(self.client) def test_command_abbreviation(self): groups = ['config', 'copy', 'copez', 'copypasta'] @@ -24,3 +31,17 @@ class TestBot(TestCase): groups = ['cat', 'cap', 'caz', 'cas'] result = self.bot.compute_group_abbrev(groups) self.assertDictEqual(result, {}) + + def test_many_commands(self): + self.bot._commands = [ + Command(None, None, 'test{}'.format(i), '') + for i in range(1000) + ] + + self.bot.compute_command_matches_re() + match = self.bot.command_matches_re.match('test5 123') + self.assertNotEqual(match, None) + + match = self.bot._commands[0].compiled_regex.match('test0 123 456') + self.assertEqual(match.group(1).strip(), 'test0') + self.assertEqual(match.group(2).strip(), '123 456') From 3e7c0a1ae324935420f2208c9bfb404247a00904 Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 26 Apr 2017 13:15:23 -0700 Subject: [PATCH 62/63] Add support for detecting NSFW channels --- disco/types/channel.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/disco/types/channel.py b/disco/types/channel.py index 48a10c1..39d2f47 100644 --- a/disco/types/channel.py +++ b/disco/types/channel.py @@ -1,3 +1,4 @@ +import re import six from six.moves import map @@ -11,6 +12,9 @@ from disco.types.permissions import Permissions, Permissible, PermissionValue from disco.voice.client import VoiceClient +NSFW_RE = re.compile('^nsfw(-|$)') + + ChannelType = Enum( GUILD_TEXT=0, DM=1, @@ -179,6 +183,13 @@ class Channel(SlottedModel, Permissible): """ return self.type in (ChannelType.DM, ChannelType.GROUP_DM) + @property + def is_nsfw(self): + """ + Whether this channel is an NSFW channel. + """ + return self.type == ChannelType.GUILD_TEXT and NSFW_RE.match(self.name) + @property def is_voice(self): """ From 6e8719b128ba54f78f63b06429198b468a56a632 Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 26 Apr 2017 13:20:41 -0700 Subject: [PATCH 63/63] Add tests for Channels --- tests/test_channel.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/test_channel.py diff --git a/tests/test_channel.py b/tests/test_channel.py new file mode 100644 index 0000000..56d5227 --- /dev/null +++ b/tests/test_channel.py @@ -0,0 +1,21 @@ +from unittest import TestCase + +from disco.types.channel import Channel, ChannelType + + +class TestChannel(TestCase): + def test_nsfw_channel(self): + channel = Channel( + name='nsfw-testing', + type=ChannelType.GUILD_TEXT) + self.assertTrue(channel.is_nsfw) + + channel = Channel( + name='nsfw-testing', + type=ChannelType.GUILD_VOICE) + self.assertFalse(channel.is_nsfw) + + channel = Channel( + name='nsfw_testing', + type=ChannelType.GUILD_TEXT) + self.assertFalse(channel.is_nsfw)