Support unknown json data and add internal set/clear field methods for generated accessors. (#918)

Add support for unknown fields to JSON (de-)serialized proto messages.
This allows data to be roundtripped from a server via this format
without dropping fields when the client doesn't contain metadata for
some fields.

Store the data as decoded but unparsed json and append it back to the
message when re-serializing a message.

Ignore unknown json data for equality/hashing to preserve existing
behavior for users. Taking this data into account would change the
result of existing `==` and `hashCode` calls.

By separating out `$_setField` and `$_clearField` this allows one to
differentiate reflective uses of `setField` and `clearField` (i.e. from
user code, usually with the result of `getTagNumber`) vs. internal uses
from the generated accessors.

Fixes #49.

---

cl/611522109
cl/612283849

---------

Co-authored-by: Nate Biggs <[email protected]>
Co-authored-by: Ömer Sinan Ağacan <[email protected]>

Author: 3 people

protobuf/CHANGELOG.md

@@ -25,9 +25,20 @@
   To migrate, use `PbMap.unmodifiable(map.keyFieldType, map.valueFieldType)`
   instead of `PbMap.unmodifiable(map)`. ([#902])
 
+* Messages deserialized from JSON now generate the unknown fields when
+  serialized as JSON.
+
+  Note that, as before, unknown fields in JSON messages are not stored in the
+  `unknownFields` of the message. They are only used by the JSON serializers to
+  support roundtripping.
+
+  ([#49], [#918])
+
 [#738]: https://github.com/google/protobuf.dart/issues/738
 [#896]: https://github.com/google/protobuf.dart/issues/896
 [#902]: https://github.com/google/protobuf.dart/issues/902
+[#49]: https://github.com/google/protobuf.dart/issues/49
+[#918]: https://github.com/google/protobuf.dart/pulls/918
 
 ## 3.1.0
 

protobuf/lib/meta.dart

@@ -56,6 +56,7 @@ const GeneratedMessage_reservedNames = <String>[
   'writeToCodedBufferWriter',
   'writeToJson',
   'writeToJsonMap',
+  r'$_clearField',
   r'$_ensure',
   r'$_get',
   r'$_getI64',
@@ -72,6 +73,7 @@ const GeneratedMessage_reservedNames = <String>[
   r'$_setBool',
   r'$_setBytes',
   r'$_setDouble',
+  r'$_setField',
   r'$_setFloat',
   r'$_setInt64',
   r'$_setSignedInt32',

protobuf/lib/src/protobuf/field_set.dart

@@ -33,6 +33,9 @@ class _FieldSet {
   /// Contains all the unknown fields, or null if there aren't any.
   UnknownFieldSet? _unknownFields;
 
+  /// Contains unknown data for messages deserialized from json.
+  Map<String, dynamic>? _unknownJsonData;
+
   /// Encodes whether `this` has been frozen, and if so, also memoizes the
   /// hash code.
   ///
@@ -105,6 +108,7 @@ class _FieldSet {
       if (_isReadOnly) return UnknownFieldSet.emptyUnknownFieldSet;
       _unknownFields = UnknownFieldSet();
     }
+    _unknownJsonData = null;
     return _unknownFields!;
   }
 
@@ -531,6 +535,8 @@ class _FieldSet {
       if (_unknownFields != o._unknownFields) return false;
     }
 
+    // Ignore _unknownJsonData to preserve existing equality behavior.
+
     return true;
   }
 
@@ -597,6 +603,8 @@ class _FieldSet {
     // Hash with unknown fields.
     hash = _HashUtils._combine(hash, _unknownFields?.hashCode ?? 0);
 
+    // Ignore _unknownJsonData to preserve existing hashing behavior.
+
     if (_isReadOnly) {
       _frozenState = hash;
     }
@@ -682,6 +690,11 @@ class _FieldSet {
     } else {
       out.write(UnknownFieldSet().toString());
     }
+
+    final unknownJsonData = _unknownJsonData;
+    if (unknownJsonData != null) {
+      out.write(unknownJsonData.toString());
+    }
   }
 
   /// Merges the contents of the [other] into this message.
@@ -713,6 +726,15 @@ class _FieldSet {
     if (otherUnknownFields != null) {
       _ensureUnknownFields().mergeFromUnknownFieldSet(otherUnknownFields);
     }
+
+    final otherUnknownJsonData = other._unknownJsonData;
+    if (otherUnknownJsonData != null) {
+      final newUnknownJsonData =
+          Map<String, dynamic>.from(_unknownJsonData ?? {});
+      otherUnknownJsonData
+          .forEach((key, value) => newUnknownJsonData[key] = value);
+      _unknownJsonData = newUnknownJsonData.isEmpty ? null : newUnknownJsonData;
+    }
   }
 
   void _mergeField(FieldInfo otherFi, fieldValue, {required bool isExtension}) {
@@ -868,6 +890,11 @@ class _FieldSet {
       _ensureUnknownFields()._fields.addAll(originalUnknownFields._fields);
     }
 
+    final unknownJsonData = original._unknownJsonData;
+    if (unknownJsonData != null) {
+      _unknownJsonData = Map.from(unknownJsonData);
+    }
+
     _oneofCases?.addAll(original._oneofCases!);
   }
 }

protobuf/lib/src/protobuf/generated_message.dart

@@ -173,6 +173,10 @@ abstract class GeneratedMessage {
   }
 
   /// Serialize the message as the protobuf binary format.
+  ///
+  /// Unknown field data, data for which there is no metadata for the associated
+  /// field, will only be included if this message was deserialized from the
+  /// same wire format.
   Uint8List writeToBuffer() {
     final out = CodedBufferWriter();
     writeToCodedBufferWriter(out);
@@ -211,6 +215,10 @@ abstract class GeneratedMessage {
   /// Returns the JSON encoding of this message as a Dart [Map].
   ///
   /// The encoding is described in [GeneratedMessage.writeToJson].
+  ///
+  /// Unknown field data, data for which there is no metadata for the associated
+  /// field, will only be included if this message was deserialized from the
+  /// same wire format.
   Map<String, dynamic> writeToJsonMap() => _writeToJsonMap(_fieldSet);
 
   /// Returns a JSON string that encodes this message.
@@ -226,6 +234,10 @@ abstract class GeneratedMessage {
   /// represented as their integer value.
   ///
   /// For the proto3 JSON format use: [toProto3Json].
+  ///
+  /// Unknown field data, data for which there is no metadata for the associated
+  /// field, will only be included if this message was deserialized from the
+  /// same wire format.
   String writeToJson() => jsonEncode(writeToJsonMap());
 
   /// Returns an Object representing Proto3 JSON serialization of `this`.
@@ -241,6 +253,9 @@ abstract class GeneratedMessage {
   /// The [typeRegistry] is be used for encoding `Any` messages. If an `Any`
   /// message encoding a type not in [typeRegistry] is encountered, an
   /// error is thrown.
+  ///
+  /// Unknown field data, data for which there is no metadata for the associated
+  /// field, will not be included.
   Object? toProto3Json(
           {TypeRegistry typeRegistry = const TypeRegistry.empty()}) =>
       _writeToProto3Json(_fieldSet, typeRegistry);
@@ -509,6 +524,17 @@ abstract class GeneratedMessage {
   /// @nodoc
   void $_setInt64(int index, Int64 value) => _fieldSet._$set(index, value);
 
+  /// For generated code only. Separate from [setField] to distinguish
+  /// reflective accesses.
+  /// @nodoc
+  void $_setField(int tagNumber, Object value) =>
+      _fieldSet._setField(tagNumber, value);
+
+  /// For generated code only. Separate from [clearField] to distinguish
+  /// reflective accesses.
+  /// @nodoc
+  void $_clearField(int tagNumber) => _fieldSet._clearField(tagNumber);
+
   // Support for generating a read-only default singleton instance.
 
   static final Map<Function?, _SingletonMaker<GeneratedMessage>>

protobuf/lib/src/protobuf/json.dart

@@ -89,6 +89,12 @@ Map<String, dynamic> _writeToJsonMap(_FieldSet fs) {
       result['$tagNumber'] = convertToMap(value, fi.type);
     }
   }
+  final unknownJsonData = fs._unknownJsonData;
+  if (unknownJsonData != null) {
+    unknownJsonData.forEach((key, value) {
+      result[key] = value;
+    });
+  }
   return result;
 }
 
@@ -102,9 +108,11 @@ void _mergeFromJsonMap(
   for (final key in keys) {
     var fi = meta.byTagAsString[key];
     if (fi == null) {
-      if (registry == null) continue; // Unknown tag; skip
-      fi = registry.getExtension(fs._messageName, int.parse(key));
-      if (fi == null) continue; // Unknown tag; skip
+      fi = registry?.getExtension(fs._messageName, int.parse(key));
+      if (fi == null) {
+        (fs._unknownJsonData ??= {})[key] = json[key];
+        continue;
+      }
     }
     if (fi.isMapField) {
       _appendJsonMap(

protobuf/test/json_test.dart

@@ -87,7 +87,7 @@ void main() {
     checkJsonMap(m);
   });
 
-  test('testMergeFromJson', () {
+  test('testWriteToJsonMap', () {
     final t = T();
     t.mergeFromJson('''{"1": 123, "2": "hello"}''');
     checkMessage(t);
@@ -118,13 +118,21 @@ void main() {
     final decoded = T()..mergeFromJsonMap(encoded);
     expect(decoded.int64, value);
   });
+
+  test('testJsonMapWithUnknown', () {
+    final m = example.writeToJsonMap();
+    m['9999'] = 'world';
+    final t = T()..mergeFromJsonMap(m);
+    checkJsonMap(t.writeToJsonMap(), unknownFields: {'9999': 'world'});
+  });
 }
 
-void checkJsonMap(Map m) {
-  expect(m.length, 3);
+void checkJsonMap(Map m, {Map<String, dynamic>? unknownFields}) {
+  expect(m.length, 3 + (unknownFields?.length ?? 0));
   expect(m['1'], 123);
   expect(m['2'], 'hello');
   expect(m['4'], [1, 2, 3]);
+  unknownFields?.forEach((k, v) => expect(m[k], v));
 }
 
 void checkMessage(T t) {

protoc_plugin/lib/src/generated/dart_options.pb.dart

@@ -82,7 +82,7 @@ class DartMixin extends $pb.GeneratedMessage {
   @$pb.TagNumber(1)
   $core.bool hasName() => $_has(0);
   @$pb.TagNumber(1)
-  void clearName() => clearField(1);
+  void clearName() => $_clearField(1);
 
   /// A URI pointing to the Dart library that defines the mixin.
   /// The generated Dart code will use this in an import statement.
@@ -96,7 +96,7 @@ class DartMixin extends $pb.GeneratedMessage {
   @$pb.TagNumber(2)
   $core.bool hasImportFrom() => $_has(1);
   @$pb.TagNumber(2)
-  void clearImportFrom() => clearField(2);
+  void clearImportFrom() => $_clearField(2);
 
   /// The name of another mixin to be applied ahead of this one.
   /// The generated class for the message will inherit from all mixins
@@ -111,7 +111,7 @@ class DartMixin extends $pb.GeneratedMessage {
   @$pb.TagNumber(3)
   $core.bool hasParent() => $_has(2);
   @$pb.TagNumber(3)
-  void clearParent() => clearField(3);
+  void clearParent() => $_clearField(3);
 }
 
 /// Defines additional Dart imports to be used with messages in this file.