fix: resolve unit test failure caused by differences in protobuf runtimes

gapic/ads-templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2

@@ -1048,15 +1048,23 @@ def test_{{ method.name|snake_case }}_rest(request_type):
         # Wrap the value into a proper Response obj
         response_value = Response()
         response_value.status_code = 200
-    {% if method.void %}
+        {% if method.void %}
         json_return_value = ''
         {% elif method.lro %}
         json_return_value = json_format.MessageToJson(return_value)
-    {% elif method.server_streaming %}
-        json_return_value = "[{}]".format({{ method.output.ident }}.to_json(return_value))
-    {% else %}
-        json_return_value = {{ method.output.ident }}.to_json(return_value)
-    {% endif %}
+        {% else %}
+
+        {% if method.output.ident.is_proto_plus_type %}
+        pb_return_value = {{ method.output.ident }}.pb(return_value)
+        {% else %}
+        pb_return_value = return_value
+        {% endif %}
+        json_return_value = json_format.MessageToJson(pb_return_value)
+        {% endif %}
+        {% if method.server_streaming %}
+        json_return_value = "[{}]".format(json_return_value)
+        {% endif %}
+
         response_value._content = json_return_value.encode('UTF-8')
         req.return_value = response_value
         {% if method.client_streaming %}
@@ -1391,16 +1399,22 @@ def test_{{ method.name|snake_case }}_rest_flattened():
         # Wrap the value into a proper Response obj
         response_value = Response()
         response_value.status_code = 200
-    {% if method.void %}
+        {% if method.void %}
         json_return_value = ''
         {% elif method.lro %}
         json_return_value = json_format.MessageToJson(return_value)
-    {% elif method.server_streaming %}
-        json_return_value = "[{}]".format({{ method.output.ident }}.to_json(return_value))
-    {% else %}
-        json_return_value = {{ method.output.ident }}.to_json(return_value)
-    {% endif %}
+        {% else %}
 
+        {% if method.output.ident.is_proto_plus_type %}
+        pb_return_value = {{ method.output.ident }}.pb(return_value)
+        {% else %}
+        pb_return_value = return_value
+        {% endif %}
+        json_return_value = json_format.MessageToJson(pb_return_value)
+        {% endif %}
+        {% if method.server_streaming %}
+        json_return_value = "[{}]".format(json_return_value)
+        {% endif %}
         response_value._content = json_return_value.encode('UTF-8')
         req.return_value = response_value
 

gapic/templates/tests/unit/gapic/%name_%version/%sub/test_macros.j2

@@ -887,6 +887,50 @@ def test_{{ method_name }}_rest(request_type):
     {% if not field.oneof or field.proto3_optional %}
     {# ignore oneof fields that might conflict with sample_request #}
     request_init["{{ field.name }}"] = {{ field.merged_mock_value(method.http_options[0].sample_request(method).get(field.name)) }}
+    # The version of a generated dependency at runtime may differ compared to the one at time of generation
+    # Delete any keys which are not present in the current runtime dependency
+    # See https://github.com/googleapis/gapic-generator-python/issues/1748
+    if hasattr({{ method.input.ident }}.meta.fields["{{ field.name }}"].message, "DESCRIPTOR"):
+        keys_to_delete = []
+
+        # Get all subfields for the message
+        subfield_names = [
+            (field.name, subfield.name)
+            for field in {{ method.input.ident }}.meta.fields["{{ field.name }}"].message.DESCRIPTOR.fields
+            if field.message_type
+            for subfield in field.message_type.fields
+        ]
+
+        # For each item in the sample request, create a list of sub fields which are not present at runtime
+        for key, value in request_init["{{ field.name }}"].items():
+            result = None
+            is_repeated = False
+            # For repeated fields
+            if isinstance(value, list) and len(value):
+                is_repeated = True
+                result = value[0]
+            # For fields where the type is another message
+            if isinstance(value, dict):
+                result = value
+
+            if result:
+                for nested_key in result.keys():
+                    if (key, nested_key) not in subfield_names:
+                        keys_to_delete.append(
+                            {"key": key, "nested_key": nested_key, "is_repeated": is_repeated}
+                        )
+
+        # Remove fields from the sample request which are not present in the runtime version of the dependency
+        for key_to_delete in keys_to_delete:
+            if key_to_delete.get("nested_key"):
+                if key_to_delete.get("is_repeated"):
+                    del request_init["{{ field.name }}"][key_to_delete.get("key")][0][
+                        key_to_delete.get("nested_key")
+                    ]
+                else:
+                    del request_init["{{ field.name }}"][key_to_delete.get("key")][
+                        key_to_delete.get("nested_key")
+                    ]
     {% endif %}
     {% endfor %}
     request = request_type(**request_init)
@@ -1242,12 +1286,6 @@ def test_{{ method_name }}_rest_bad_request(transport: str = 'rest', request_typ
 
     # send a request that will satisfy transcoding
     request_init = {{ method.http_options[0].sample_request(method) }}
-    {% for field in method.body_fields.values() %}
-    {% if not field.oneof or field.proto3_optional %}
-    {# ignore oneof fields that might conflict with sample_request #}
-    request_init["{{ field.name }}"] = {{ field.merged_mock_value(method.http_options[0].sample_request(method).get(field.name)) }}
-    {% endif %}
-    {% endfor %}
     request = request_type(**request_init)
     {% if method.client_streaming %}
     requests = [request]

noxfile.py

@@ -110,9 +110,20 @@ def __call__(self, frag):
             )
 
             # Install the generated fragment library.
-            # Note: install into the tempdir to prevent issues
-            # with running pip concurrently.
-            self.session.install(tmp_dir, "-e", ".", "-t", tmp_dir, "-qqq")
+            if self.use_ads_templates:
+                self.session.install(tmp_dir, "-e", ".", "-qqq")
+            else:
+                # Use the constraints file for the specific python runtime version
+                # We do this to make sure that we're testing against the lowest
+                # supported version of a dependency.
+                # This is needed to recreate the issue reported in
+                # https://github.com/googleapis/gapic-generator-python/issues/1748
+                # The ads templates do not have constraints files.
+                constraints_path = str(
+                f"{tmp_dir}/testing/constraints-{self.session.python}.txt"
+                )
+                self.session.install(tmp_dir, "-e", ".", "-qqq", "-r", constraints_path)
+
             # Run the fragment's generated unit tests.
             # Don't bother parallelizing them: we already parallelize
             # the fragments, and there usually aren't too many tests per fragment.

setup.py

@@ -25,12 +25,14 @@
 version = "1.11.5"
 release_status = "Development Status :: 5 - Production/Stable"
 dependencies = [
+    # Esnure that the lower bounds of these dependencies match what we have in the
+    # templated setup.py.j2 here https://github.com/googleapis/gapic-generator-python/blob/main/gapic/templates/setup.py.j2
     "click >= 6.7",
-    "google-api-core >= 2.8.0",
+    "google-api-core[grpc] >= 1.34.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*",
     "googleapis-common-protos >= 1.55.0",
     "grpcio >= 1.24.3",
     "jinja2 >= 2.10",
-    "protobuf >= 3.18.0, < 5.0.0dev",
+    "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
     "pypandoc >= 1.4",
     "PyYAML >= 5.1.1",
     "grpc-google-iam-v1 >= 0.12.4, < 1.0.0dev",