From 58f51a94f937f65f6c1af6c87d174388afab034b Mon Sep 17 00:00:00 2001 From: Rossen Georgiev Date: Sat, 25 Apr 2020 13:07:15 +0100 Subject: [PATCH] move proto related util functions to steam.utils.proto --- CHANGES.md | 1 + docs/api/steam.utils.rst | 8 ++ steam/client/__init__.py | 3 +- steam/client/builtins/apps.py | 3 +- steam/client/builtins/gameservers.py | 3 +- steam/client/builtins/unified_messages.py | 2 +- steam/client/builtins/user.py | 2 +- steam/client/gc.py | 2 +- steam/core/cm.py | 3 +- steam/core/msg/headers.py | 2 +- steam/guard.py | 2 +- steam/utils/__init__.py | 110 --------------------- steam/utils/proto.py | 114 ++++++++++++++++++++++ tests/{test_util.py => test_utils.py} | 72 +++++++------- 14 files changed, 173 insertions(+), 154 deletions(-) create mode 100644 steam/utils/proto.py rename tests/{test_util.py => test_utils.py} (66%) diff --git a/CHANGES.md b/CHANGES.md index d6442f9..15c5dd8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,7 @@ This release brings breaking changes - Updated all enums - Removed imports from 'steam' namespace - Renamed `steam.util` to `steam.utils` +- Moved proto utils to `steam.utils.proto` ### steam.steamid diff --git a/docs/api/steam.utils.rst b/docs/api/steam.utils.rst index 81d31b8..48ad00f 100644 --- a/docs/api/steam.utils.rst +++ b/docs/api/steam.utils.rst @@ -14,6 +14,14 @@ utils.binary :undoc-members: :show-inheritance: +utils.proto +------------ + +.. automodule:: steam.utils.proto + :members: + :undoc-members: + :show-inheritance: + utils.throttle -------------- diff --git a/steam/client/__init__.py b/steam/client/__init__.py index d098c44..1a21d21 100644 --- a/steam/client/__init__.py +++ b/steam/client/__init__.py @@ -31,7 +31,8 @@ from steam.core.crypto import sha1_hash from steam.steamid import SteamID from steam.exceptions import SteamError from steam.client.builtins import BuiltinBase -from steam.utils import ip_from_int, ip_to_int, proto_fill_from_dict +from steam.utils import ip_from_int, ip_to_int +from steam.utils.proto import proto_fill_from_dict if six.PY2: _cli_input = raw_input diff --git a/steam/client/builtins/apps.py b/steam/client/builtins/apps.py index f852253..d8d3b5c 100644 --- a/steam/client/builtins/apps.py +++ b/steam/client/builtins/apps.py @@ -2,7 +2,8 @@ import vdf from steam.enums import EResult, EServerType from steam.enums.emsg import EMsg from steam.core.msg import MsgProto -from steam.utils import ip_from_int, proto_fill_from_dict +from steam.utils import ip_from_int +from steam.utils.proto import proto_fill_from_dict class Apps(object): diff --git a/steam/client/builtins/gameservers.py b/steam/client/builtins/gameservers.py index 4c42ac0..d6304b0 100644 --- a/steam/client/builtins/gameservers.py +++ b/steam/client/builtins/gameservers.py @@ -38,7 +38,8 @@ from steam.steamid import SteamID from steam.core.msg import MsgProto from steam.enums import EResult from steam.enums.emsg import EMsg -from steam.utils import ip_to_int, ip_from_int, proto_to_dict +from steam.utils import ip_to_int, ip_from_int +from steam.utils.proto import proto_to_dict class GameServers(object): diff --git a/steam/client/builtins/unified_messages.py b/steam/client/builtins/unified_messages.py index d5a33f6..5c3a159 100644 --- a/steam/client/builtins/unified_messages.py +++ b/steam/client/builtins/unified_messages.py @@ -27,7 +27,7 @@ The backend might error out, but we still get response. Here is how to check for """ from steam.core.msg import MsgProto, get_um from steam.enums.emsg import EMsg -from steam.utils import proto_fill_from_dict +from steam.utils.proto import proto_fill_from_dict class UnifiedMessages(object): diff --git a/steam/client/builtins/user.py b/steam/client/builtins/user.py index 8a43b0d..60fd13a 100644 --- a/steam/client/builtins/user.py +++ b/steam/client/builtins/user.py @@ -3,7 +3,7 @@ from steam.client.user import SteamUser from steam.enums import EPersonaState, EChatEntryType, EType, EClientUIMode from steam.enums.emsg import EMsg from steam.core.msg import MsgProto -from steam.utils import proto_fill_from_dict +from steam.utils.proto import proto_fill_from_dict class User(object): EVENT_CHAT_MESSAGE = 'chat_message' diff --git a/steam/client/gc.py b/steam/client/gc.py index 3beced8..6d3120f 100644 --- a/steam/client/gc.py +++ b/steam/client/gc.py @@ -40,7 +40,7 @@ protobufs needed to (de)serialize message for communication with GC. import logging import gevent from eventemitter import EventEmitter -from steam.utils import set_proto_bit, clear_proto_bit, is_proto +from steam.utils.proto import set_proto_bit, clear_proto_bit, is_proto from steam.enums.emsg import EMsg from steam.enums import EResult from steam.core.msg import GCMsgHdr, GCMsgHdrProto, MsgProto diff --git a/steam/core/cm.py b/steam/core/cm.py index 7916862..a92bac9 100644 --- a/steam/core/cm.py +++ b/steam/core/cm.py @@ -19,7 +19,8 @@ from steam.core import crypto from steam.core.connection import TCPConnection from steam.core.msg import Msg, MsgProto from eventemitter import EventEmitter -from steam.utils import ip_from_int, is_proto, clear_proto_bit +from steam.utils import ip_from_int +from steam.utils.proto import is_proto, clear_proto_bit class CMClient(EventEmitter): diff --git a/steam/core/msg/headers.py b/steam/core/msg/headers.py index e1afffd..9944604 100644 --- a/steam/core/msg/headers.py +++ b/steam/core/msg/headers.py @@ -3,7 +3,7 @@ import struct from steam.enums.emsg import EMsg from steam.protobufs import steammessages_base_pb2 from steam.protobufs import gc_pb2 -from steam.utils import set_proto_bit, clear_proto_bit +from steam.utils.proto import set_proto_bit, clear_proto_bit class MsgHdr: diff --git a/steam/guard.py b/steam/guard.py index e9ae3f7..0627567 100644 --- a/steam/guard.py +++ b/steam/guard.py @@ -58,7 +58,7 @@ from steam.steamid import SteamID from steam.core.crypto import hmac_sha1, sha1_hash from steam.enums.common import EResult from steam.webauth import MobileWebAuth -from steam.utils import proto_to_dict +from steam.utils.proto import proto_to_dict class SteamAuthenticator(object): diff --git a/steam/utils/__init__.py b/steam/utils/__init__.py index 750a892..3159bcb 100644 --- a/steam/utils/__init__.py +++ b/steam/utils/__init__.py @@ -4,15 +4,7 @@ import weakref import struct import socket import sys -import six from six.moves import xrange as _range -from types import GeneratorType as _GeneratorType -from google.protobuf.message import Message as _ProtoMessageType - -if six.PY2: - _list_types = list, xrange, _GeneratorType -else: - _list_types = list, range, _GeneratorType, map, filter def ip_from_int(ip): @@ -35,108 +27,6 @@ def ip_to_int(ip): return struct.unpack(">L", socket.inet_aton(ip))[0] -protobuf_mask = 0x80000000 - -def is_proto(emsg): - """ - :param emsg: emsg number - :type emsg: int - :return: True or False - :rtype: bool - """ - return (int(emsg) & protobuf_mask) > 0 - -def set_proto_bit(emsg): - """ - :param emsg: emsg number - :type emsg: int - :return: emsg with proto bit set - :rtype: int - """ - return int(emsg) | protobuf_mask - -def clear_proto_bit(emsg): - """ - :param emsg: emsg number - :type emsg: int - :return: emsg with proto bit removed - :rtype: int - """ - return int(emsg) & ~protobuf_mask - -def proto_to_dict(message): - """Converts protobuf message instance to dict - - :param message: protobuf message instance - :return: parameters and their values - :rtype: dict - :raises: :class:`.TypeError` if ``message`` is not a proto message - """ - if not isinstance(message, _ProtoMessageType): - raise TypeError("Expected `message` to be a instance of protobuf message") - - data = {} - - for desc, field in message.ListFields(): - if desc.type == desc.TYPE_MESSAGE: - if desc.label == desc.LABEL_REPEATED: - data[desc.name] = list(map(proto_to_dict, field)) - else: - data[desc.name] = proto_to_dict(field) - else: - data[desc.name] = list(field) if desc.label == desc.LABEL_REPEATED else field - - return data - -def proto_fill_from_dict(message, data, clear=True): - """Fills protobuf message parameters inplace from a :class:`dict` - - :param message: protobuf message instance - :param data: parameters and values - :type data: dict - :param clear: whether clear exisiting values - :type clear: bool - :return: value of message paramater - :raises: incorrect types or values will raise - """ - if not isinstance(message, _ProtoMessageType): - raise TypeError("Expected `message` to be a instance of protobuf message") - if not isinstance(data, dict): - raise TypeError("Expected `data` to be of type `dict`") - - if clear: message.Clear() - field_descs = message.DESCRIPTOR.fields_by_name - - for key, val in data.items(): - desc = field_descs[key] - - if desc.type == desc.TYPE_MESSAGE: - if desc.label == desc.LABEL_REPEATED: - if not isinstance(val, _list_types): - raise TypeError("Expected %s to be of type list, got %s" % (repr(key), type(val))) - - list_ref = getattr(message, key) - - # Takes care of overwriting list fields when merging partial data (clear=False) - if not clear: del list_ref[:] # clears the list - - for item in val: - item_message = getattr(message, key).add() - proto_fill_from_dict(item_message, item) - else: - if not isinstance(val, dict): - raise TypeError("Expected %s to be of type dict, got %s" % (repr(key), type(dict))) - - proto_fill_from_dict(getattr(message, key), val) - else: - if isinstance(val, _list_types): - list_ref = getattr(message, key) - if not clear: del list_ref[:] # clears the list - list_ref.extend(val) - else: - setattr(message, key, val) - - return message def chunks(arr, size): """Splits a list into chunks diff --git a/steam/utils/proto.py b/steam/utils/proto.py new file mode 100644 index 0000000..4138703 --- /dev/null +++ b/steam/utils/proto.py @@ -0,0 +1,114 @@ + +import six +from types import GeneratorType as _GeneratorType +from google.protobuf.message import Message as _ProtoMessageType + + +if six.PY2: + _list_types = list, xrange, _GeneratorType +else: + _list_types = list, range, _GeneratorType, map, filter + +protobuf_mask = 0x80000000 + + +def is_proto(emsg): + """ + :param emsg: emsg number + :type emsg: int + :return: True or False + :rtype: bool + """ + return (int(emsg) & protobuf_mask) > 0 + +def set_proto_bit(emsg): + """ + :param emsg: emsg number + :type emsg: int + :return: emsg with proto bit set + :rtype: int + """ + return int(emsg) | protobuf_mask + +def clear_proto_bit(emsg): + """ + :param emsg: emsg number + :type emsg: int + :return: emsg with proto bit removed + :rtype: int + """ + return int(emsg) & ~protobuf_mask + +def proto_to_dict(message): + """Converts protobuf message instance to dict + + :param message: protobuf message instance + :return: parameters and their values + :rtype: dict + :raises: :class:`.TypeError` if ``message`` is not a proto message + """ + if not isinstance(message, _ProtoMessageType): + raise TypeError("Expected `message` to be a instance of protobuf message") + + data = {} + + for desc, field in message.ListFields(): + if desc.type == desc.TYPE_MESSAGE: + if desc.label == desc.LABEL_REPEATED: + data[desc.name] = list(map(proto_to_dict, field)) + else: + data[desc.name] = proto_to_dict(field) + else: + data[desc.name] = list(field) if desc.label == desc.LABEL_REPEATED else field + + return data + +def proto_fill_from_dict(message, data, clear=True): + """Fills protobuf message parameters inplace from a :class:`dict` + + :param message: protobuf message instance + :param data: parameters and values + :type data: dict + :param clear: whether clear exisiting values + :type clear: bool + :return: value of message paramater + :raises: incorrect types or values will raise + """ + if not isinstance(message, _ProtoMessageType): + raise TypeError("Expected `message` to be a instance of protobuf message") + if not isinstance(data, dict): + raise TypeError("Expected `data` to be of type `dict`") + + if clear: message.Clear() + field_descs = message.DESCRIPTOR.fields_by_name + + for key, val in data.items(): + desc = field_descs[key] + + if desc.type == desc.TYPE_MESSAGE: + if desc.label == desc.LABEL_REPEATED: + if not isinstance(val, _list_types): + raise TypeError("Expected %s to be of type list, got %s" % (repr(key), type(val))) + + list_ref = getattr(message, key) + + # Takes care of overwriting list fields when merging partial data (clear=False) + if not clear: del list_ref[:] # clears the list + + for item in val: + item_message = getattr(message, key).add() + proto_fill_from_dict(item_message, item) + else: + if not isinstance(val, dict): + raise TypeError("Expected %s to be of type dict, got %s" % (repr(key), type(dict))) + + proto_fill_from_dict(getattr(message, key), val) + else: + if isinstance(val, _list_types): + list_ref = getattr(message, key) + if not clear: del list_ref[:] # clears the list + list_ref.extend(val) + else: + setattr(message, key, val) + + return message diff --git a/tests/test_util.py b/tests/test_utils.py similarity index 66% rename from tests/test_util.py rename to tests/test_utils.py index 7a69719..32a0f9e 100644 --- a/tests/test_util.py +++ b/tests/test_utils.py @@ -1,6 +1,7 @@ import unittest -import steam.util as ut -import steam.util.web as uweb +import steam.utils as ut +import steam.utils.proto as utp +import steam.utils.web as uweb import requests from steam.protobufs.test_messages_pb2 import ComplexProtoMessage @@ -17,27 +18,28 @@ class Util_Functions(unittest.TestCase): self.assertEqual(ut.ip_to_int('12.34.56.78'), 203569230) self.assertEqual(ut.ip_to_int('255.255.255.255'), 4294967295) + def test_make_requests_session(self): + self.assertIsInstance(uweb.make_requests_session(), requests.Session) + +class Util_Proto_Functions(unittest.TestCase): def test_is_proto(self): - self.assertTrue(ut.is_proto(proto_mask)) - self.assertTrue(ut.is_proto(proto_mask | 123456)) - self.assertFalse(ut.is_proto(0)) - self.assertFalse(ut.is_proto(proto_mask - 1)) - self.assertFalse(ut.is_proto(proto_mask << 1)) + self.assertTrue(utp.is_proto(proto_mask)) + self.assertTrue(utp.is_proto(proto_mask | 123456)) + self.assertFalse(utp.is_proto(0)) + self.assertFalse(utp.is_proto(proto_mask - 1)) + self.assertFalse(utp.is_proto(proto_mask << 1)) def test_set_proto_big(self): - self.assertFalse(ut.is_proto(0)) - self.assertTrue(ut.is_proto(ut.set_proto_bit(0))) - self.assertFalse(ut.is_proto(1)) - self.assertTrue(ut.is_proto(ut.set_proto_bit(1))) + self.assertFalse(utp.is_proto(0)) + self.assertTrue(utp.is_proto(utp.set_proto_bit(0))) + self.assertFalse(utp.is_proto(1)) + self.assertTrue(utp.is_proto(utp.set_proto_bit(1))) def test_clear_proto_big(self): - self.assertEqual(ut.clear_proto_bit(0), 0) - self.assertEqual(ut.clear_proto_bit(123), 123) - self.assertEqual(ut.clear_proto_bit(proto_mask | 123), 123) - self.assertEqual(ut.clear_proto_bit((proto_mask - 1) | proto_mask), proto_mask - 1) - - def test_make_requests_session(self): - self.assertIsInstance(uweb.make_requests_session(), requests.Session) + self.assertEqual(utp.clear_proto_bit(0), 0) + self.assertEqual(utp.clear_proto_bit(123), 123) + self.assertEqual(utp.clear_proto_bit(proto_mask | 123), 123) + self.assertEqual(utp.clear_proto_bit((proto_mask - 1) | proto_mask), proto_mask - 1) class Util_Proto(unittest.TestCase): def setUp(self): @@ -58,16 +60,16 @@ class Util_Proto(unittest.TestCase): 'number64': 72057594037927936 } - ut.proto_fill_from_dict(self.msg, DATA) + utp.proto_fill_from_dict(self.msg, DATA) - RESULT = ut.proto_to_dict(self.msg) + RESULT = utp.proto_to_dict(self.msg) self.assertEqual(DATA, RESULT) def test_proto_from_dict_merge(self): self.msg.list_number32.extend([1,2,3]) - ut.proto_fill_from_dict(self.msg, {'list_number32': [4,5,6]}, clear=False) + utp.proto_fill_from_dict(self.msg, {'list_number32': [4,5,6]}, clear=False) self.assertEqual(self.msg.list_number32, [4,5,6]) @@ -75,40 +77,40 @@ class Util_Proto(unittest.TestCase): self.msg.messages.add(text='one') self.msg.messages.add(text='two') - ut.proto_fill_from_dict(self.msg, {'messages': [{'text': 'three'}]}, clear=False) + utp.proto_fill_from_dict(self.msg, {'messages': [{'text': 'three'}]}, clear=False) self.assertEqual(len(self.msg.messages), 1) self.assertEqual(self.msg.messages[0].text, 'three') def test_proto_from_dict__dict_insteadof_list(self): with self.assertRaises(TypeError): - ut.proto_fill_from_dict(self.msg, {'list_number32': [{}, {}]}) + utp.proto_fill_from_dict(self.msg, {'list_number32': [{}, {}]}) def test_proto_from_dict__list_insteadof_dict(self): with self.assertRaises(TypeError): - ut.proto_fill_from_dict(self.msg, {'messages': [1,2,3]}) + utp.proto_fill_from_dict(self.msg, {'messages': [1,2,3]}) def test_proto_fill_from_dict__list(self): - ut.proto_fill_from_dict(self.msg, {'list_number32': [1,2,3]}) + utp.proto_fill_from_dict(self.msg, {'list_number32': [1,2,3]}) self.assertEqual(self.msg.list_number32, [1,2,3]) def test_proto_fill_from_dict__dict_list(self): - ut.proto_fill_from_dict(self.msg, {'messages': [{'text': 'one'}, {'text': 'two'}]}) + utp.proto_fill_from_dict(self.msg, {'messages': [{'text': 'one'}, {'text': 'two'}]}) self.assertEqual(len(self.msg.messages), 2) self.assertEqual(self.msg.messages[0].text, 'one') self.assertEqual(self.msg.messages[1].text, 'two') def test_proto_fill_from_dict__list(self): - ut.proto_fill_from_dict(self.msg, {'list_number32': range(10)}) + utp.proto_fill_from_dict(self.msg, {'list_number32': range(10)}) self.assertEqual(self.msg.list_number32, list(range(10))) def test_proto_fill_from_dict__generator(self): - ut.proto_fill_from_dict(self.msg, {'list_number32': (x for x in [1,2,3])}) + utp.proto_fill_from_dict(self.msg, {'list_number32': (x for x in [1,2,3])}) self.assertEqual(self.msg.list_number32, [1,2,3]) def test_proto_fill_from_dict__dict_generator(self): - ut.proto_fill_from_dict(self.msg, {'messages': (x for x in [{'text': 'one'}, {'text': 'two'}])}) + utp.proto_fill_from_dict(self.msg, {'messages': (x for x in [{'text': 'one'}, {'text': 'two'}])}) self.assertEqual(len(self.msg.messages), 2) self.assertEqual(self.msg.messages[0].text, 'one') self.assertEqual(self.msg.messages[1].text, 'two') @@ -119,7 +121,7 @@ class Util_Proto(unittest.TestCase): yield 2 yield 3 - ut.proto_fill_from_dict(self.msg, {'list_number32': number_gen()}) + utp.proto_fill_from_dict(self.msg, {'list_number32': number_gen()}) self.assertEqual(self.msg.list_number32, [1,2,3]) def test_proto_fill_from_dict__dict_func_generator(self): @@ -127,28 +129,28 @@ class Util_Proto(unittest.TestCase): yield {'text': 'one'} yield {'text': 'two'} - ut.proto_fill_from_dict(self.msg, {'messages': dict_gen()}) + utp.proto_fill_from_dict(self.msg, {'messages': dict_gen()}) self.assertEqual(len(self.msg.messages), 2) self.assertEqual(self.msg.messages[0].text, 'one') self.assertEqual(self.msg.messages[1].text, 'two') def test_proto_fill_from_dict__map(self): - ut.proto_fill_from_dict(self.msg, {'list_number32': map(int, [1,2,3])}) + utp.proto_fill_from_dict(self.msg, {'list_number32': map(int, [1,2,3])}) self.assertEqual(self.msg.list_number32, [1,2,3]) def test_proto_fill_from_dict__dict_map(self): - ut.proto_fill_from_dict(self.msg, {'messages': map(dict, [{'text': 'one'}, {'text': 'two'}])}) + utp.proto_fill_from_dict(self.msg, {'messages': map(dict, [{'text': 'one'}, {'text': 'two'}])}) self.assertEqual(len(self.msg.messages), 2) self.assertEqual(self.msg.messages[0].text, 'one') self.assertEqual(self.msg.messages[1].text, 'two') def test_proto_fill_from_dict__filter(self): - ut.proto_fill_from_dict(self.msg, {'list_number32': filter(lambda x: True, [1,2,3])}) + utp.proto_fill_from_dict(self.msg, {'list_number32': filter(lambda x: True, [1,2,3])}) self.assertEqual(self.msg.list_number32, [1,2,3]) def test_proto_fill_from_dict__dict_filter(self): - ut.proto_fill_from_dict(self.msg, {'messages': filter(lambda x: True, [{'text': 'one'}, {'text': 'two'}])}) + utp.proto_fill_from_dict(self.msg, {'messages': filter(lambda x: True, [{'text': 'one'}, {'text': 'two'}])}) self.assertEqual(len(self.msg.messages), 2) self.assertEqual(self.msg.messages[0].text, 'one') self.assertEqual(self.msg.messages[1].text, 'two')