move proto related util functions to steam.utils.proto

CHANGES.md

@@ -10,6 +10,7 @@ This release brings breaking changes
 - Updated all enums
 - Removed imports from 'steam' namespace
 - Renamed `steam.util` to `steam.utils`
+- Moved proto utils to `steam.utils.proto`
 
 ### steam.steamid
 

docs/api/steam.utils.rst

@@ -14,6 +14,14 @@ utils.binary
     :undoc-members:
     :show-inheritance:
 
+utils.proto
+------------
+
+.. automodule:: steam.utils.proto
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 utils.throttle
 --------------
 

steam/client/__init__.py

@@ -31,7 +31,8 @@ from steam.core.crypto import sha1_hash
 from steam.steamid import SteamID
 from steam.exceptions import SteamError
 from steam.client.builtins import BuiltinBase
-from steam.utils import ip_from_int, ip_to_int, proto_fill_from_dict
+from steam.utils import ip_from_int, ip_to_int
+from steam.utils.proto import proto_fill_from_dict
 
 if six.PY2:
     _cli_input = raw_input

steam/client/builtins/apps.py

@@ -2,7 +2,8 @@ import vdf
 from steam.enums import EResult, EServerType
 from steam.enums.emsg import EMsg
 from steam.core.msg import MsgProto
-from steam.utils import ip_from_int, proto_fill_from_dict
+from steam.utils import ip_from_int
+from steam.utils.proto import proto_fill_from_dict
 
 
 class Apps(object):

steam/client/builtins/gameservers.py

@@ -38,7 +38,8 @@ from steam.steamid import SteamID
 from steam.core.msg import MsgProto
 from steam.enums import EResult
 from steam.enums.emsg import EMsg
-from steam.utils import ip_to_int, ip_from_int, proto_to_dict
+from steam.utils import ip_to_int, ip_from_int
+from steam.utils.proto import proto_to_dict
 
 
 class GameServers(object):

steam/client/builtins/unified_messages.py

@@ -27,7 +27,7 @@ The backend might error out, but we still get response. Here is how to check for
 """
 from steam.core.msg import MsgProto, get_um
 from steam.enums.emsg import EMsg
-from steam.utils import proto_fill_from_dict
+from steam.utils.proto import proto_fill_from_dict
 
 
 class UnifiedMessages(object):

steam/client/builtins/user.py

@@ -3,7 +3,7 @@ from steam.client.user import SteamUser
 from steam.enums import EPersonaState, EChatEntryType, EType, EClientUIMode
 from steam.enums.emsg import EMsg
 from steam.core.msg import MsgProto
-from steam.utils import proto_fill_from_dict
+from steam.utils.proto import proto_fill_from_dict
 
 class User(object):
     EVENT_CHAT_MESSAGE = 'chat_message'

steam/client/gc.py

@@ -40,7 +40,7 @@ protobufs needed to (de)serialize message for communication with GC.
 import logging
 import gevent
 from eventemitter import EventEmitter
-from steam.utils import set_proto_bit, clear_proto_bit, is_proto
+from steam.utils.proto import set_proto_bit, clear_proto_bit, is_proto
 from steam.enums.emsg import EMsg
 from steam.enums import EResult
 from steam.core.msg import GCMsgHdr, GCMsgHdrProto, MsgProto

steam/core/cm.py

@@ -19,7 +19,8 @@ from steam.core import crypto
 from steam.core.connection import TCPConnection
 from steam.core.msg import Msg, MsgProto
 from eventemitter import EventEmitter
-from steam.utils import ip_from_int, is_proto, clear_proto_bit
+from steam.utils import ip_from_int
+from steam.utils.proto import is_proto, clear_proto_bit
 
 
 class CMClient(EventEmitter):

steam/core/msg/headers.py

@@ -3,7 +3,7 @@ import struct
 from steam.enums.emsg import EMsg
 from steam.protobufs import steammessages_base_pb2
 from steam.protobufs import gc_pb2
-from steam.utils import set_proto_bit, clear_proto_bit
+from steam.utils.proto import set_proto_bit, clear_proto_bit
 
 
 class MsgHdr:

steam/guard.py

@@ -58,7 +58,7 @@ from steam.steamid import SteamID
 from steam.core.crypto import hmac_sha1, sha1_hash
 from steam.enums.common import EResult
 from steam.webauth import MobileWebAuth
-from steam.utils import proto_to_dict
+from steam.utils.proto import proto_to_dict
 
 
 class SteamAuthenticator(object):

steam/utils/__init__.py

@@ -4,15 +4,7 @@ import weakref
 import struct
 import socket
 import sys
-import six
 from six.moves import xrange as _range
-from types import GeneratorType as _GeneratorType
-from google.protobuf.message import Message as _ProtoMessageType
-
-if six.PY2:
-    _list_types = list, xrange, _GeneratorType
-else:
-    _list_types = list, range, _GeneratorType, map, filter
 
 
 def ip_from_int(ip):
@@ -35,108 +27,6 @@ def ip_to_int(ip):
     return struct.unpack(">L", socket.inet_aton(ip))[0]
 
 
-protobuf_mask = 0x80000000
-
-def is_proto(emsg):
-    """
-    :param emsg: emsg number
-    :type emsg: int
-    :return: True or False
-    :rtype: bool
-    """
-    return (int(emsg) & protobuf_mask) > 0
-
-def set_proto_bit(emsg):
-    """
-    :param emsg: emsg number
-    :type emsg: int
-    :return: emsg with proto bit set
-    :rtype: int
-    """
-    return int(emsg) | protobuf_mask
-
-def clear_proto_bit(emsg):
-    """
-    :param emsg: emsg number
-    :type emsg: int
-    :return: emsg with proto bit removed
-    :rtype: int
-    """
-    return int(emsg) & ~protobuf_mask
-
-def proto_to_dict(message):
-    """Converts protobuf message instance to dict
-
-    :param message: protobuf message instance
-    :return: parameters and their values
-    :rtype: dict
-    :raises: :class:`.TypeError` if ``message`` is not a proto message
-    """
-    if not isinstance(message, _ProtoMessageType):
-        raise TypeError("Expected `message` to be a instance of protobuf message")
-
-    data = {}
-
-    for desc, field in message.ListFields():
-        if desc.type == desc.TYPE_MESSAGE:
-            if desc.label == desc.LABEL_REPEATED:
-                data[desc.name] = list(map(proto_to_dict, field))
-            else:
-                data[desc.name] = proto_to_dict(field)
-        else:
-            data[desc.name] = list(field) if desc.label == desc.LABEL_REPEATED else field
-
-    return data
-
-def proto_fill_from_dict(message, data, clear=True):
-    """Fills protobuf message parameters inplace from a :class:`dict`
-
-    :param message: protobuf message instance
-    :param data: parameters and values
-    :type data: dict
-    :param clear: whether clear exisiting values
-    :type clear: bool
-    :return: value of message paramater
-    :raises: incorrect types or values will raise
-    """
-    if not isinstance(message, _ProtoMessageType):
-        raise TypeError("Expected `message` to be a instance of protobuf message")
-    if not isinstance(data, dict):
-        raise TypeError("Expected `data` to be of type `dict`")
-
-    if clear: message.Clear()
-    field_descs = message.DESCRIPTOR.fields_by_name
-
-    for key, val in data.items():
-        desc = field_descs[key]
-
-        if desc.type == desc.TYPE_MESSAGE:
-            if desc.label == desc.LABEL_REPEATED:
-                if not isinstance(val, _list_types):
-                    raise TypeError("Expected %s to be of type list, got %s" % (repr(key), type(val)))
-
-                list_ref = getattr(message, key)
-
-                # Takes care of overwriting list fields when merging partial data (clear=False)
-                if not clear: del list_ref[:]  # clears the list
-
-                for item in val:
-                    item_message = getattr(message, key).add()
-                    proto_fill_from_dict(item_message, item)
-            else:
-                if not isinstance(val, dict):
-                    raise TypeError("Expected %s to be of type dict, got %s" % (repr(key), type(dict)))
-
-                proto_fill_from_dict(getattr(message, key), val)
-        else:
-            if isinstance(val, _list_types):
-                list_ref = getattr(message, key)
-                if not clear: del list_ref[:]  # clears the list
-                list_ref.extend(val)
-            else:
-                setattr(message, key, val)
-
-    return message
 
 def chunks(arr, size):
     """Splits a list into chunks

steam/utils/proto.py

@@ -0,0 +1,114 @@
+
+import six
+from types import GeneratorType as _GeneratorType
+from google.protobuf.message import Message as _ProtoMessageType
+
+
+if six.PY2:
+    _list_types = list, xrange, _GeneratorType
+else:
+    _list_types = list, range, _GeneratorType, map, filter
+
+protobuf_mask = 0x80000000
+
+
+def is_proto(emsg):
+    """
+    :param emsg: emsg number
+    :type emsg: int
+    :return: True or False
+    :rtype: bool
+    """
+    return (int(emsg) & protobuf_mask) > 0
+
+def set_proto_bit(emsg):
+    """
+    :param emsg: emsg number
+    :type emsg: int
+    :return: emsg with proto bit set
+    :rtype: int
+    """
+    return int(emsg) | protobuf_mask
+
+def clear_proto_bit(emsg):
+    """
+    :param emsg: emsg number
+    :type emsg: int
+    :return: emsg with proto bit removed
+    :rtype: int
+    """
+    return int(emsg) & ~protobuf_mask
+
+def proto_to_dict(message):
+    """Converts protobuf message instance to dict
+
+    :param message: protobuf message instance
+    :return: parameters and their values
+    :rtype: dict
+    :raises: :class:`.TypeError` if ``message`` is not a proto message
+    """
+    if not isinstance(message, _ProtoMessageType):
+        raise TypeError("Expected `message` to be a instance of protobuf message")
+
+    data = {}
+
+    for desc, field in message.ListFields():
+        if desc.type == desc.TYPE_MESSAGE:
+            if desc.label == desc.LABEL_REPEATED:
+                data[desc.name] = list(map(proto_to_dict, field))
+            else:
+                data[desc.name] = proto_to_dict(field)
+        else:
+            data[desc.name] = list(field) if desc.label == desc.LABEL_REPEATED else field
+
+    return data
+
+def proto_fill_from_dict(message, data, clear=True):
+    """Fills protobuf message parameters inplace from a :class:`dict`
+
+    :param message: protobuf message instance
+    :param data: parameters and values
+    :type data: dict
+    :param clear: whether clear exisiting values
+    :type clear: bool
+    :return: value of message paramater
+    :raises: incorrect types or values will raise
+    """
+    if not isinstance(message, _ProtoMessageType):
+        raise TypeError("Expected `message` to be a instance of protobuf message")
+    if not isinstance(data, dict):
+        raise TypeError("Expected `data` to be of type `dict`")
+
+    if clear: message.Clear()
+    field_descs = message.DESCRIPTOR.fields_by_name
+
+    for key, val in data.items():
+        desc = field_descs[key]
+
+        if desc.type == desc.TYPE_MESSAGE:
+            if desc.label == desc.LABEL_REPEATED:
+                if not isinstance(val, _list_types):
+                    raise TypeError("Expected %s to be of type list, got %s" % (repr(key), type(val)))
+
+                list_ref = getattr(message, key)
+
+                # Takes care of overwriting list fields when merging partial data (clear=False)
+                if not clear: del list_ref[:]  # clears the list
+
+                for item in val:
+                    item_message = getattr(message, key).add()
+                    proto_fill_from_dict(item_message, item)
+            else:
+                if not isinstance(val, dict):
+                    raise TypeError("Expected %s to be of type dict, got %s" % (repr(key), type(dict)))
+
+                proto_fill_from_dict(getattr(message, key), val)
+        else:
+            if isinstance(val, _list_types):
+                list_ref = getattr(message, key)
+                if not clear: del list_ref[:]  # clears the list
+                list_ref.extend(val)
+            else:
+                setattr(message, key, val)
+
+    return message

tests/test_util.py → tests/test_utils.py

@@ -1,6 +1,7 @@
 import unittest
-import steam.util as ut
+import steam.utils as ut
-import steam.util.web as uweb
+import steam.utils.proto as utp
+import steam.utils.web as uweb
 import requests
 from steam.protobufs.test_messages_pb2 import ComplexProtoMessage
 
@@ -17,27 +18,28 @@ class Util_Functions(unittest.TestCase):
         self.assertEqual(ut.ip_to_int('12.34.56.78'), 203569230)
         self.assertEqual(ut.ip_to_int('255.255.255.255'), 4294967295)
 
+    def test_make_requests_session(self):
+        self.assertIsInstance(uweb.make_requests_session(), requests.Session)
+
+class Util_Proto_Functions(unittest.TestCase):
     def test_is_proto(self):
-        self.assertTrue(ut.is_proto(proto_mask))
+        self.assertTrue(utp.is_proto(proto_mask))
-        self.assertTrue(ut.is_proto(proto_mask | 123456))
+        self.assertTrue(utp.is_proto(proto_mask | 123456))
-        self.assertFalse(ut.is_proto(0))
+        self.assertFalse(utp.is_proto(0))
-        self.assertFalse(ut.is_proto(proto_mask - 1))
+        self.assertFalse(utp.is_proto(proto_mask - 1))
-        self.assertFalse(ut.is_proto(proto_mask << 1))
+        self.assertFalse(utp.is_proto(proto_mask << 1))
 
     def test_set_proto_big(self):
-        self.assertFalse(ut.is_proto(0))
+        self.assertFalse(utp.is_proto(0))
-        self.assertTrue(ut.is_proto(ut.set_proto_bit(0)))
+        self.assertTrue(utp.is_proto(utp.set_proto_bit(0)))
-        self.assertFalse(ut.is_proto(1))
+        self.assertFalse(utp.is_proto(1))
-        self.assertTrue(ut.is_proto(ut.set_proto_bit(1)))
+        self.assertTrue(utp.is_proto(utp.set_proto_bit(1)))
 
     def test_clear_proto_big(self):
-        self.assertEqual(ut.clear_proto_bit(0), 0)
+        self.assertEqual(utp.clear_proto_bit(0), 0)
-        self.assertEqual(ut.clear_proto_bit(123), 123)
+        self.assertEqual(utp.clear_proto_bit(123), 123)
-        self.assertEqual(ut.clear_proto_bit(proto_mask | 123), 123)
+        self.assertEqual(utp.clear_proto_bit(proto_mask | 123), 123)
-        self.assertEqual(ut.clear_proto_bit((proto_mask - 1) | proto_mask), proto_mask - 1)
+        self.assertEqual(utp.clear_proto_bit((proto_mask - 1) | proto_mask), proto_mask - 1)
-
-    def test_make_requests_session(self):
-        self.assertIsInstance(uweb.make_requests_session(), requests.Session)
 
 class Util_Proto(unittest.TestCase):
     def setUp(self):
@@ -58,16 +60,16 @@ class Util_Proto(unittest.TestCase):
                 'number64': 72057594037927936
                 }
 
-        ut.proto_fill_from_dict(self.msg, DATA)
+        utp.proto_fill_from_dict(self.msg, DATA)
 
-        RESULT = ut.proto_to_dict(self.msg)
+        RESULT = utp.proto_to_dict(self.msg)
 
         self.assertEqual(DATA, RESULT)
 
     def test_proto_from_dict_merge(self):
         self.msg.list_number32.extend([1,2,3])
 
-        ut.proto_fill_from_dict(self.msg, {'list_number32': [4,5,6]}, clear=False)
+        utp.proto_fill_from_dict(self.msg, {'list_number32': [4,5,6]}, clear=False)
 
         self.assertEqual(self.msg.list_number32, [4,5,6])
 
@@ -75,40 +77,40 @@ class Util_Proto(unittest.TestCase):
         self.msg.messages.add(text='one')
         self.msg.messages.add(text='two')
 
-        ut.proto_fill_from_dict(self.msg, {'messages': [{'text': 'three'}]}, clear=False)
+        utp.proto_fill_from_dict(self.msg, {'messages': [{'text': 'three'}]}, clear=False)
 
         self.assertEqual(len(self.msg.messages), 1)
         self.assertEqual(self.msg.messages[0].text, 'three')
 
     def test_proto_from_dict__dict_insteadof_list(self):
         with self.assertRaises(TypeError):
-            ut.proto_fill_from_dict(self.msg, {'list_number32': [{}, {}]})
+            utp.proto_fill_from_dict(self.msg, {'list_number32': [{}, {}]})
 
     def test_proto_from_dict__list_insteadof_dict(self):
         with self.assertRaises(TypeError):
-            ut.proto_fill_from_dict(self.msg, {'messages': [1,2,3]})
+            utp.proto_fill_from_dict(self.msg, {'messages': [1,2,3]})
 
     def test_proto_fill_from_dict__list(self):
-        ut.proto_fill_from_dict(self.msg, {'list_number32': [1,2,3]})
+        utp.proto_fill_from_dict(self.msg, {'list_number32': [1,2,3]})
         self.assertEqual(self.msg.list_number32, [1,2,3])
 
     def test_proto_fill_from_dict__dict_list(self):
-        ut.proto_fill_from_dict(self.msg, {'messages': [{'text': 'one'}, {'text': 'two'}]})
+        utp.proto_fill_from_dict(self.msg, {'messages': [{'text': 'one'}, {'text': 'two'}]})
         self.assertEqual(len(self.msg.messages), 2)
         self.assertEqual(self.msg.messages[0].text, 'one')
         self.assertEqual(self.msg.messages[1].text, 'two')
 
     def test_proto_fill_from_dict__list(self):
-        ut.proto_fill_from_dict(self.msg, {'list_number32': range(10)})
+        utp.proto_fill_from_dict(self.msg, {'list_number32': range(10)})
         self.assertEqual(self.msg.list_number32, list(range(10)))
 
 
     def test_proto_fill_from_dict__generator(self):
-        ut.proto_fill_from_dict(self.msg, {'list_number32': (x for x in [1,2,3])})
+        utp.proto_fill_from_dict(self.msg, {'list_number32': (x for x in [1,2,3])})
         self.assertEqual(self.msg.list_number32, [1,2,3])
 
     def test_proto_fill_from_dict__dict_generator(self):
-        ut.proto_fill_from_dict(self.msg, {'messages': (x for x in [{'text': 'one'}, {'text': 'two'}])})
+        utp.proto_fill_from_dict(self.msg, {'messages': (x for x in [{'text': 'one'}, {'text': 'two'}])})
         self.assertEqual(len(self.msg.messages), 2)
         self.assertEqual(self.msg.messages[0].text, 'one')
         self.assertEqual(self.msg.messages[1].text, 'two')
@@ -119,7 +121,7 @@ class Util_Proto(unittest.TestCase):
             yield 2
             yield 3
 
-        ut.proto_fill_from_dict(self.msg, {'list_number32': number_gen()})
+        utp.proto_fill_from_dict(self.msg, {'list_number32': number_gen()})
         self.assertEqual(self.msg.list_number32, [1,2,3])
 
     def test_proto_fill_from_dict__dict_func_generator(self):
@@ -127,28 +129,28 @@ class Util_Proto(unittest.TestCase):
             yield {'text': 'one'}
             yield {'text': 'two'}
 
-        ut.proto_fill_from_dict(self.msg, {'messages': dict_gen()})
+        utp.proto_fill_from_dict(self.msg, {'messages': dict_gen()})
         self.assertEqual(len(self.msg.messages), 2)
         self.assertEqual(self.msg.messages[0].text, 'one')
         self.assertEqual(self.msg.messages[1].text, 'two')
 
 
     def test_proto_fill_from_dict__map(self):
-        ut.proto_fill_from_dict(self.msg, {'list_number32': map(int, [1,2,3])})
+        utp.proto_fill_from_dict(self.msg, {'list_number32': map(int, [1,2,3])})
         self.assertEqual(self.msg.list_number32, [1,2,3])
 
     def test_proto_fill_from_dict__dict_map(self):
-        ut.proto_fill_from_dict(self.msg, {'messages': map(dict, [{'text': 'one'}, {'text': 'two'}])})
+        utp.proto_fill_from_dict(self.msg, {'messages': map(dict, [{'text': 'one'}, {'text': 'two'}])})
         self.assertEqual(len(self.msg.messages), 2)
         self.assertEqual(self.msg.messages[0].text, 'one')
         self.assertEqual(self.msg.messages[1].text, 'two')
 
     def test_proto_fill_from_dict__filter(self):
-        ut.proto_fill_from_dict(self.msg, {'list_number32': filter(lambda x: True, [1,2,3])})
+        utp.proto_fill_from_dict(self.msg, {'list_number32': filter(lambda x: True, [1,2,3])})
         self.assertEqual(self.msg.list_number32, [1,2,3])
 
     def test_proto_fill_from_dict__dict_filter(self):
-        ut.proto_fill_from_dict(self.msg, {'messages': filter(lambda x: True, [{'text': 'one'}, {'text': 'two'}])})
+        utp.proto_fill_from_dict(self.msg, {'messages': filter(lambda x: True, [{'text': 'one'}, {'text': 'two'}])})
         self.assertEqual(len(self.msg.messages), 2)
         self.assertEqual(self.msg.messages[0].text, 'one')
         self.assertEqual(self.msg.messages[1].text, 'two')