From 0a582e63b8ca2963da259c163ce05181e7eb5d7d Mon Sep 17 00:00:00 2001 From: Dov Shlachter Date: Wed, 20 Oct 2021 14:54:01 -0700 Subject: [PATCH] fix: setting 64bit fields from strings supported Due to limitations in certain browser/javascript combinations, fields that are 64 bit integer types are converted to strings when encoding a message to JSON or a dict. Decoding from JSON handles this explicitly, but it makes converting back from a dict an error. This fix adds support for setting these 64 bit fields explicitly from strings. E.g. class Squid(proto.Message): mass_kg = proto.Field(proto.INT64, number=1) s = Squid(mass_kg=10) s_dict = Squid.to_dict(s) assert s == Squid(**s_dict) # Supported assert s == Squid(s_dict) # NOT supported for performance reasons s.mass_kg = "20" # Supported --- docs/marshal.rst | 21 ++++++++ proto/marshal/marshal.py | 6 +++ proto/marshal/rules/stringy_numbers.py | 68 ++++++++++++++++++++++++++ tests/test_fields_int.py | 25 ++++++++++ 4 files changed, 120 insertions(+) create mode 100644 proto/marshal/rules/stringy_numbers.py diff --git a/docs/marshal.rst b/docs/marshal.rst index 1d2dcccc..2b5f0405 100644 --- a/docs/marshal.rst +++ b/docs/marshal.rst @@ -72,7 +72,28 @@ Protocol buffer type Python type Nullable assert msg == msg_pb == msg_two +.. warning:: + Due to certain browser/javascript limitations, 64 bit sized fields, e.g. INT64, UINT64, + are converted to strings when marshalling messages to dictionaries or JSON. + Decoding JSON handles this correctly, but dicts must be unpacked when reconstructing messages. This is necessary to trigger a special case workaround. + + .. code-block:: python + + import proto + + class MyMessage(proto.Message): + serial_id = proto.Field(proto.INT64, number=1) + + msg = MyMessage(serial_id=12345) + msg_dict = MyMessage.to_dict(msg) + + msg_2 = MyMessage(msg_dict) # Raises an exception + + msg_3 = MyMessage(**msg_dict) # Works without exception + assert msg == msg_3 + + Wrapper types ------------- diff --git a/proto/marshal/marshal.py b/proto/marshal/marshal.py index d0dc2ead..baac7adc 100644 --- a/proto/marshal/marshal.py +++ b/proto/marshal/marshal.py @@ -26,6 +26,7 @@ from proto.marshal.collections import Repeated from proto.marshal.collections import RepeatedComposite from proto.marshal.rules import bytes as pb_bytes +from proto.marshal.rules import stringy_numbers from proto.marshal.rules import dates from proto.marshal.rules import struct from proto.marshal.rules import wrappers @@ -147,6 +148,11 @@ def reset(self): # Special case for bytes to allow base64 encode/decode self.register(ProtoType.BYTES, pb_bytes.BytesRule()) + # Special case for int64 from strings because of dict round trip. + # See https://github.com/protocolbuffers/protobuf/issues/2679 + for rule_class in stringy_numbers.STRINGY_NUMBER_RULES: + self.register(rule_class._proto_type, rule_class()) + def to_python(self, proto_type, value, *, absent: bool = None): # Internal protobuf has its own special type for lists of values. # Return a view around it that implements MutableSequence. diff --git a/proto/marshal/rules/stringy_numbers.py b/proto/marshal/rules/stringy_numbers.py new file mode 100644 index 00000000..0d808cc2 --- /dev/null +++ b/proto/marshal/rules/stringy_numbers.py @@ -0,0 +1,68 @@ +# Copyright (C) 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from proto.primitives import ProtoType + + +class StringyNumberRule: + """A marshal between certain numeric types and strings + + This is a necessary hack to allow round trip conversion + from messages to dicts back to messages. + + See https://github.com/protocolbuffers/protobuf/issues/2679 + and + https://developers.google.com/protocol-buffers/docs/proto3#json + for more details. + """ + + def to_python(self, value, *, absent: bool = None): + return value + + def to_proto(self, value): + return self._python_type(value) + + +class Int64Rule(StringyNumberRule): + _python_type = int + _proto_type = ProtoType.INT64 + + +class UInt64Rule(StringyNumberRule): + _python_type = int + _proto_type = ProtoType.UINT64 + + +class SInt64Rule(StringyNumberRule): + _python_type = int + _proto_type = ProtoType.SINT64 + + +class Fixed64Rule(StringyNumberRule): + _python_type = int + _proto_type = ProtoType.FIXED64 + + +class SFixed64Rule(StringyNumberRule): + _python_type = int + _proto_type = ProtoType.SFIXED64 + + +STRINGY_NUMBER_RULES = [ + Int64Rule, + UInt64Rule, + SInt64Rule, + Fixed64Rule, + SFixed64Rule, +] diff --git a/tests/test_fields_int.py b/tests/test_fields_int.py index c3a979c0..45d5c96e 100644 --- a/tests/test_fields_int.py +++ b/tests/test_fields_int.py @@ -93,3 +93,28 @@ class Foo(proto.Message): bar_field = Foo.meta.fields["bar"] assert bar_field.descriptor is bar_field.descriptor + + +def test_int64_dict_round_trip(): + # When converting a message to other types, protobuf turns int64 fields + # into decimal coded strings. + # This is not a problem for round trip JSON, but it is a problem + # when doing a round trip conversion from a message to a dict to a message. + # See https://github.com/protocolbuffers/protobuf/issues/2679 + # and + # https://developers.google.com/protocol-buffers/docs/proto3#json + # for more details. + class Squid(proto.Message): + mass_kg = proto.Field(proto.INT64, number=1) + length_cm = proto.Field(proto.UINT64, number=2) + age_s = proto.Field(proto.FIXED64, number=3) + depth_m = proto.Field(proto.SFIXED64, number=4) + serial_num = proto.Field(proto.SINT64, number=5) + + s = Squid(mass_kg=10, length_cm=20, age_s=30, depth_m=40, serial_num=50) + + s_dict = Squid.to_dict(s) + + s2 = Squid(**s_dict) + + assert s == s2