Browse Source

move proto related util functions to steam.utils.proto

pull/254/head
Rossen Georgiev 5 years ago
parent
commit
58f51a94f9
  1. 1
      CHANGES.md
  2. 8
      docs/api/steam.utils.rst
  3. 3
      steam/client/__init__.py
  4. 3
      steam/client/builtins/apps.py
  5. 3
      steam/client/builtins/gameservers.py
  6. 2
      steam/client/builtins/unified_messages.py
  7. 2
      steam/client/builtins/user.py
  8. 2
      steam/client/gc.py
  9. 3
      steam/core/cm.py
  10. 2
      steam/core/msg/headers.py
  11. 2
      steam/guard.py
  12. 110
      steam/utils/__init__.py
  13. 114
      steam/utils/proto.py
  14. 72
      tests/test_utils.py

1
CHANGES.md

@ -10,6 +10,7 @@ This release brings breaking changes
- Updated all enums
- Removed imports from 'steam' namespace
- Renamed `steam.util` to `steam.utils`
- Moved proto utils to `steam.utils.proto`
### steam.steamid

8
docs/api/steam.utils.rst

@ -14,6 +14,14 @@ utils.binary
:undoc-members:
:show-inheritance:
utils.proto
------------
.. automodule:: steam.utils.proto
:members:
:undoc-members:
:show-inheritance:
utils.throttle
--------------

3
steam/client/__init__.py

@ -31,7 +31,8 @@ from steam.core.crypto import sha1_hash
from steam.steamid import SteamID
from steam.exceptions import SteamError
from steam.client.builtins import BuiltinBase
from steam.utils import ip_from_int, ip_to_int, proto_fill_from_dict
from steam.utils import ip_from_int, ip_to_int
from steam.utils.proto import proto_fill_from_dict
if six.PY2:
_cli_input = raw_input

3
steam/client/builtins/apps.py

@ -2,7 +2,8 @@ import vdf
from steam.enums import EResult, EServerType
from steam.enums.emsg import EMsg
from steam.core.msg import MsgProto
from steam.utils import ip_from_int, proto_fill_from_dict
from steam.utils import ip_from_int
from steam.utils.proto import proto_fill_from_dict
class Apps(object):

3
steam/client/builtins/gameservers.py

@ -38,7 +38,8 @@ from steam.steamid import SteamID
from steam.core.msg import MsgProto
from steam.enums import EResult
from steam.enums.emsg import EMsg
from steam.utils import ip_to_int, ip_from_int, proto_to_dict
from steam.utils import ip_to_int, ip_from_int
from steam.utils.proto import proto_to_dict
class GameServers(object):

2
steam/client/builtins/unified_messages.py

@ -27,7 +27,7 @@ The backend might error out, but we still get response. Here is how to check for
"""
from steam.core.msg import MsgProto, get_um
from steam.enums.emsg import EMsg
from steam.utils import proto_fill_from_dict
from steam.utils.proto import proto_fill_from_dict
class UnifiedMessages(object):

2
steam/client/builtins/user.py

@ -3,7 +3,7 @@ from steam.client.user import SteamUser
from steam.enums import EPersonaState, EChatEntryType, EType, EClientUIMode
from steam.enums.emsg import EMsg
from steam.core.msg import MsgProto
from steam.utils import proto_fill_from_dict
from steam.utils.proto import proto_fill_from_dict
class User(object):
EVENT_CHAT_MESSAGE = 'chat_message'

2
steam/client/gc.py

@ -40,7 +40,7 @@ protobufs needed to (de)serialize message for communication with GC.
import logging
import gevent
from eventemitter import EventEmitter
from steam.utils import set_proto_bit, clear_proto_bit, is_proto
from steam.utils.proto import set_proto_bit, clear_proto_bit, is_proto
from steam.enums.emsg import EMsg
from steam.enums import EResult
from steam.core.msg import GCMsgHdr, GCMsgHdrProto, MsgProto

3
steam/core/cm.py

@ -19,7 +19,8 @@ from steam.core import crypto
from steam.core.connection import TCPConnection
from steam.core.msg import Msg, MsgProto
from eventemitter import EventEmitter
from steam.utils import ip_from_int, is_proto, clear_proto_bit
from steam.utils import ip_from_int
from steam.utils.proto import is_proto, clear_proto_bit
class CMClient(EventEmitter):

2
steam/core/msg/headers.py

@ -3,7 +3,7 @@ import struct
from steam.enums.emsg import EMsg
from steam.protobufs import steammessages_base_pb2
from steam.protobufs import gc_pb2
from steam.utils import set_proto_bit, clear_proto_bit
from steam.utils.proto import set_proto_bit, clear_proto_bit
class MsgHdr:

2
steam/guard.py

@ -58,7 +58,7 @@ from steam.steamid import SteamID
from steam.core.crypto import hmac_sha1, sha1_hash
from steam.enums.common import EResult
from steam.webauth import MobileWebAuth
from steam.utils import proto_to_dict
from steam.utils.proto import proto_to_dict
class SteamAuthenticator(object):

110
steam/utils/__init__.py

@ -4,15 +4,7 @@ import weakref
import struct
import socket
import sys
import six
from six.moves import xrange as _range
from types import GeneratorType as _GeneratorType
from google.protobuf.message import Message as _ProtoMessageType
if six.PY2:
_list_types = list, xrange, _GeneratorType
else:
_list_types = list, range, _GeneratorType, map, filter
def ip_from_int(ip):
@ -35,108 +27,6 @@ def ip_to_int(ip):
return struct.unpack(">L", socket.inet_aton(ip))[0]
protobuf_mask = 0x80000000
def is_proto(emsg):
"""
:param emsg: emsg number
:type emsg: int
:return: True or False
:rtype: bool
"""
return (int(emsg) & protobuf_mask) > 0
def set_proto_bit(emsg):
"""
:param emsg: emsg number
:type emsg: int
:return: emsg with proto bit set
:rtype: int
"""
return int(emsg) | protobuf_mask
def clear_proto_bit(emsg):
"""
:param emsg: emsg number
:type emsg: int
:return: emsg with proto bit removed
:rtype: int
"""
return int(emsg) & ~protobuf_mask
def proto_to_dict(message):
"""Converts protobuf message instance to dict
:param message: protobuf message instance
:return: parameters and their values
:rtype: dict
:raises: :class:`.TypeError` if ``message`` is not a proto message
"""
if not isinstance(message, _ProtoMessageType):
raise TypeError("Expected `message` to be a instance of protobuf message")
data = {}
for desc, field in message.ListFields():
if desc.type == desc.TYPE_MESSAGE:
if desc.label == desc.LABEL_REPEATED:
data[desc.name] = list(map(proto_to_dict, field))
else:
data[desc.name] = proto_to_dict(field)
else:
data[desc.name] = list(field) if desc.label == desc.LABEL_REPEATED else field
return data
def proto_fill_from_dict(message, data, clear=True):
"""Fills protobuf message parameters inplace from a :class:`dict`
:param message: protobuf message instance
:param data: parameters and values
:type data: dict
:param clear: whether clear exisiting values
:type clear: bool
:return: value of message paramater
:raises: incorrect types or values will raise
"""
if not isinstance(message, _ProtoMessageType):
raise TypeError("Expected `message` to be a instance of protobuf message")
if not isinstance(data, dict):
raise TypeError("Expected `data` to be of type `dict`")
if clear: message.Clear()
field_descs = message.DESCRIPTOR.fields_by_name
for key, val in data.items():
desc = field_descs[key]
if desc.type == desc.TYPE_MESSAGE:
if desc.label == desc.LABEL_REPEATED:
if not isinstance(val, _list_types):
raise TypeError("Expected %s to be of type list, got %s" % (repr(key), type(val)))
list_ref = getattr(message, key)
# Takes care of overwriting list fields when merging partial data (clear=False)
if not clear: del list_ref[:] # clears the list
for item in val:
item_message = getattr(message, key).add()
proto_fill_from_dict(item_message, item)
else:
if not isinstance(val, dict):
raise TypeError("Expected %s to be of type dict, got %s" % (repr(key), type(dict)))
proto_fill_from_dict(getattr(message, key), val)
else:
if isinstance(val, _list_types):
list_ref = getattr(message, key)
if not clear: del list_ref[:] # clears the list
list_ref.extend(val)
else:
setattr(message, key, val)
return message
def chunks(arr, size):
"""Splits a list into chunks

114
steam/utils/proto.py

@ -0,0 +1,114 @@
import six
from types import GeneratorType as _GeneratorType
from google.protobuf.message import Message as _ProtoMessageType
if six.PY2:
_list_types = list, xrange, _GeneratorType
else:
_list_types = list, range, _GeneratorType, map, filter
protobuf_mask = 0x80000000
def is_proto(emsg):
"""
:param emsg: emsg number
:type emsg: int
:return: True or False
:rtype: bool
"""
return (int(emsg) & protobuf_mask) > 0
def set_proto_bit(emsg):
"""
:param emsg: emsg number
:type emsg: int
:return: emsg with proto bit set
:rtype: int
"""
return int(emsg) | protobuf_mask
def clear_proto_bit(emsg):
"""
:param emsg: emsg number
:type emsg: int
:return: emsg with proto bit removed
:rtype: int
"""
return int(emsg) & ~protobuf_mask
def proto_to_dict(message):
"""Converts protobuf message instance to dict
:param message: protobuf message instance
:return: parameters and their values
:rtype: dict
:raises: :class:`.TypeError` if ``message`` is not a proto message
"""
if not isinstance(message, _ProtoMessageType):
raise TypeError("Expected `message` to be a instance of protobuf message")
data = {}
for desc, field in message.ListFields():
if desc.type == desc.TYPE_MESSAGE:
if desc.label == desc.LABEL_REPEATED:
data[desc.name] = list(map(proto_to_dict, field))
else:
data[desc.name] = proto_to_dict(field)
else:
data[desc.name] = list(field) if desc.label == desc.LABEL_REPEATED else field
return data
def proto_fill_from_dict(message, data, clear=True):
"""Fills protobuf message parameters inplace from a :class:`dict`
:param message: protobuf message instance
:param data: parameters and values
:type data: dict
:param clear: whether clear exisiting values
:type clear: bool
:return: value of message paramater
:raises: incorrect types or values will raise
"""
if not isinstance(message, _ProtoMessageType):
raise TypeError("Expected `message` to be a instance of protobuf message")
if not isinstance(data, dict):
raise TypeError("Expected `data` to be of type `dict`")
if clear: message.Clear()
field_descs = message.DESCRIPTOR.fields_by_name
for key, val in data.items():
desc = field_descs[key]
if desc.type == desc.TYPE_MESSAGE:
if desc.label == desc.LABEL_REPEATED:
if not isinstance(val, _list_types):
raise TypeError("Expected %s to be of type list, got %s" % (repr(key), type(val)))
list_ref = getattr(message, key)
# Takes care of overwriting list fields when merging partial data (clear=False)
if not clear: del list_ref[:] # clears the list
for item in val:
item_message = getattr(message, key).add()
proto_fill_from_dict(item_message, item)
else:
if not isinstance(val, dict):
raise TypeError("Expected %s to be of type dict, got %s" % (repr(key), type(dict)))
proto_fill_from_dict(getattr(message, key), val)
else:
if isinstance(val, _list_types):
list_ref = getattr(message, key)
if not clear: del list_ref[:] # clears the list
list_ref.extend(val)
else:
setattr(message, key, val)
return message

72
tests/test_util.py → tests/test_utils.py

@ -1,6 +1,7 @@
import unittest
import steam.util as ut
import steam.util.web as uweb
import steam.utils as ut
import steam.utils.proto as utp
import steam.utils.web as uweb
import requests
from steam.protobufs.test_messages_pb2 import ComplexProtoMessage
@ -17,27 +18,28 @@ class Util_Functions(unittest.TestCase):
self.assertEqual(ut.ip_to_int('12.34.56.78'), 203569230)
self.assertEqual(ut.ip_to_int('255.255.255.255'), 4294967295)
def test_make_requests_session(self):
self.assertIsInstance(uweb.make_requests_session(), requests.Session)
class Util_Proto_Functions(unittest.TestCase):
def test_is_proto(self):
self.assertTrue(ut.is_proto(proto_mask))
self.assertTrue(ut.is_proto(proto_mask | 123456))
self.assertFalse(ut.is_proto(0))
self.assertFalse(ut.is_proto(proto_mask - 1))
self.assertFalse(ut.is_proto(proto_mask << 1))
self.assertTrue(utp.is_proto(proto_mask))
self.assertTrue(utp.is_proto(proto_mask | 123456))
self.assertFalse(utp.is_proto(0))
self.assertFalse(utp.is_proto(proto_mask - 1))
self.assertFalse(utp.is_proto(proto_mask << 1))
def test_set_proto_big(self):
self.assertFalse(ut.is_proto(0))
self.assertTrue(ut.is_proto(ut.set_proto_bit(0)))
self.assertFalse(ut.is_proto(1))
self.assertTrue(ut.is_proto(ut.set_proto_bit(1)))
self.assertFalse(utp.is_proto(0))
self.assertTrue(utp.is_proto(utp.set_proto_bit(0)))
self.assertFalse(utp.is_proto(1))
self.assertTrue(utp.is_proto(utp.set_proto_bit(1)))
def test_clear_proto_big(self):
self.assertEqual(ut.clear_proto_bit(0), 0)
self.assertEqual(ut.clear_proto_bit(123), 123)
self.assertEqual(ut.clear_proto_bit(proto_mask | 123), 123)
self.assertEqual(ut.clear_proto_bit((proto_mask - 1) | proto_mask), proto_mask - 1)
def test_make_requests_session(self):
self.assertIsInstance(uweb.make_requests_session(), requests.Session)
self.assertEqual(utp.clear_proto_bit(0), 0)
self.assertEqual(utp.clear_proto_bit(123), 123)
self.assertEqual(utp.clear_proto_bit(proto_mask | 123), 123)
self.assertEqual(utp.clear_proto_bit((proto_mask - 1) | proto_mask), proto_mask - 1)
class Util_Proto(unittest.TestCase):
def setUp(self):
@ -58,16 +60,16 @@ class Util_Proto(unittest.TestCase):
'number64': 72057594037927936
}
ut.proto_fill_from_dict(self.msg, DATA)
utp.proto_fill_from_dict(self.msg, DATA)
RESULT = ut.proto_to_dict(self.msg)
RESULT = utp.proto_to_dict(self.msg)
self.assertEqual(DATA, RESULT)
def test_proto_from_dict_merge(self):
self.msg.list_number32.extend([1,2,3])
ut.proto_fill_from_dict(self.msg, {'list_number32': [4,5,6]}, clear=False)
utp.proto_fill_from_dict(self.msg, {'list_number32': [4,5,6]}, clear=False)
self.assertEqual(self.msg.list_number32, [4,5,6])
@ -75,40 +77,40 @@ class Util_Proto(unittest.TestCase):
self.msg.messages.add(text='one')
self.msg.messages.add(text='two')
ut.proto_fill_from_dict(self.msg, {'messages': [{'text': 'three'}]}, clear=False)
utp.proto_fill_from_dict(self.msg, {'messages': [{'text': 'three'}]}, clear=False)
self.assertEqual(len(self.msg.messages), 1)
self.assertEqual(self.msg.messages[0].text, 'three')
def test_proto_from_dict__dict_insteadof_list(self):
with self.assertRaises(TypeError):
ut.proto_fill_from_dict(self.msg, {'list_number32': [{}, {}]})
utp.proto_fill_from_dict(self.msg, {'list_number32': [{}, {}]})
def test_proto_from_dict__list_insteadof_dict(self):
with self.assertRaises(TypeError):
ut.proto_fill_from_dict(self.msg, {'messages': [1,2,3]})
utp.proto_fill_from_dict(self.msg, {'messages': [1,2,3]})
def test_proto_fill_from_dict__list(self):
ut.proto_fill_from_dict(self.msg, {'list_number32': [1,2,3]})
utp.proto_fill_from_dict(self.msg, {'list_number32': [1,2,3]})
self.assertEqual(self.msg.list_number32, [1,2,3])
def test_proto_fill_from_dict__dict_list(self):
ut.proto_fill_from_dict(self.msg, {'messages': [{'text': 'one'}, {'text': 'two'}]})
utp.proto_fill_from_dict(self.msg, {'messages': [{'text': 'one'}, {'text': 'two'}]})
self.assertEqual(len(self.msg.messages), 2)
self.assertEqual(self.msg.messages[0].text, 'one')
self.assertEqual(self.msg.messages[1].text, 'two')
def test_proto_fill_from_dict__list(self):
ut.proto_fill_from_dict(self.msg, {'list_number32': range(10)})
utp.proto_fill_from_dict(self.msg, {'list_number32': range(10)})
self.assertEqual(self.msg.list_number32, list(range(10)))
def test_proto_fill_from_dict__generator(self):
ut.proto_fill_from_dict(self.msg, {'list_number32': (x for x in [1,2,3])})
utp.proto_fill_from_dict(self.msg, {'list_number32': (x for x in [1,2,3])})
self.assertEqual(self.msg.list_number32, [1,2,3])
def test_proto_fill_from_dict__dict_generator(self):
ut.proto_fill_from_dict(self.msg, {'messages': (x for x in [{'text': 'one'}, {'text': 'two'}])})
utp.proto_fill_from_dict(self.msg, {'messages': (x for x in [{'text': 'one'}, {'text': 'two'}])})
self.assertEqual(len(self.msg.messages), 2)
self.assertEqual(self.msg.messages[0].text, 'one')
self.assertEqual(self.msg.messages[1].text, 'two')
@ -119,7 +121,7 @@ class Util_Proto(unittest.TestCase):
yield 2
yield 3
ut.proto_fill_from_dict(self.msg, {'list_number32': number_gen()})
utp.proto_fill_from_dict(self.msg, {'list_number32': number_gen()})
self.assertEqual(self.msg.list_number32, [1,2,3])
def test_proto_fill_from_dict__dict_func_generator(self):
@ -127,28 +129,28 @@ class Util_Proto(unittest.TestCase):
yield {'text': 'one'}
yield {'text': 'two'}
ut.proto_fill_from_dict(self.msg, {'messages': dict_gen()})
utp.proto_fill_from_dict(self.msg, {'messages': dict_gen()})
self.assertEqual(len(self.msg.messages), 2)
self.assertEqual(self.msg.messages[0].text, 'one')
self.assertEqual(self.msg.messages[1].text, 'two')
def test_proto_fill_from_dict__map(self):
ut.proto_fill_from_dict(self.msg, {'list_number32': map(int, [1,2,3])})
utp.proto_fill_from_dict(self.msg, {'list_number32': map(int, [1,2,3])})
self.assertEqual(self.msg.list_number32, [1,2,3])
def test_proto_fill_from_dict__dict_map(self):
ut.proto_fill_from_dict(self.msg, {'messages': map(dict, [{'text': 'one'}, {'text': 'two'}])})
utp.proto_fill_from_dict(self.msg, {'messages': map(dict, [{'text': 'one'}, {'text': 'two'}])})
self.assertEqual(len(self.msg.messages), 2)
self.assertEqual(self.msg.messages[0].text, 'one')
self.assertEqual(self.msg.messages[1].text, 'two')
def test_proto_fill_from_dict__filter(self):
ut.proto_fill_from_dict(self.msg, {'list_number32': filter(lambda x: True, [1,2,3])})
utp.proto_fill_from_dict(self.msg, {'list_number32': filter(lambda x: True, [1,2,3])})
self.assertEqual(self.msg.list_number32, [1,2,3])
def test_proto_fill_from_dict__dict_filter(self):
ut.proto_fill_from_dict(self.msg, {'messages': filter(lambda x: True, [{'text': 'one'}, {'text': 'two'}])})
utp.proto_fill_from_dict(self.msg, {'messages': filter(lambda x: True, [{'text': 'one'}, {'text': 'two'}])})
self.assertEqual(len(self.msg.messages), 2)
self.assertEqual(self.msg.messages[0].text, 'one')
self.assertEqual(self.msg.messages[1].text, 'two')
Loading…
Cancel
Save