From a00d7ded69c944ab680564a4a8915029ac268ad5 Mon Sep 17 00:00:00 2001 From: Chase Sterling Date: Wed, 4 Jan 2023 01:20:53 -0500 Subject: [PATCH 1/7] Fix caching documents when the root object is a reference. --- jsonref.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/jsonref.py b/jsonref.py index 4f75ae8..253fe3e 100644 --- a/jsonref.py +++ b/jsonref.py @@ -362,13 +362,8 @@ def _replace_refs( base_uri = urlparse.urljoin(base_uri, id_) store_uri = base_uri - try: - if not isinstance(obj["$ref"], str): - raise TypeError - except (TypeError, LookupError): - pass - else: - return JsonRef( + if isinstance(obj, Mapping) and isinstance(obj.get("$ref"), str): + obj = JsonRef( obj, base_uri=base_uri, loader=loader, @@ -378,10 +373,9 @@ def _replace_refs( _path=path, _store=store, ) - # If our obj was not a json reference object, iterate through it, # replacing children with JsonRefs - if isinstance(obj, Mapping): + elif isinstance(obj, Mapping): obj = { k: _replace_refs( v, From f58b487ae0267ee81414d06e0ef5901fa12a4ea9 Mon Sep 17 00:00:00 2001 From: Chase Sterling Date: Wed, 4 Jan 2023 01:24:11 -0500 Subject: [PATCH 2/7] Allow reference objects to point within themselves. --- jsonref.py | 19 +++++++++++-------- tests.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/jsonref.py b/jsonref.py index 253fe3e..9a27f30 100644 --- a/jsonref.py +++ b/jsonref.py @@ -124,9 +124,7 @@ def callback(self): uri, fragment = urlparse.urldefrag(self.full_uri) # If we already looked this up, return a reference to the same object - if uri in self.store: - result = self.resolve_pointer(self.store[uri], fragment) - else: + if uri not in self.store: # Remote ref try: base_doc = self.loader(uri) @@ -134,11 +132,16 @@ def callback(self): raise self._error( "%s: %s" % (e.__class__.__name__, str(e)), cause=e ) from e - - kwargs = self._ref_kwargs - kwargs["base_uri"] = uri - kwargs["recursing"] = False - base_doc = _replace_refs(base_doc, **kwargs) + base_doc = _replace_refs( + base_doc, **{**self._ref_kwargs, "base_uri": uri, "recursing": False} + ) + else: + base_doc = self.store[uri] + if base_doc is self: + # A reference pointing to a property within itself is dubious, + # but we'll allow it. Issue #51 + result = self.resolve_pointer(self.__reference__, fragment) + else: result = self.resolve_pointer(base_doc, fragment) if result is self: raise self._error("Reference refers directly to itself.") diff --git a/tests.py b/tests.py index 7cf9289..07e5cd3 100644 --- a/tests.py +++ b/tests.py @@ -234,6 +234,16 @@ def test_recursive_data_structures_remote_fragment(self): result = replace_refs(json1, base_uri="/json1", loader=loader) assert result["a"].__subject__ is result + def test_self_referent_reference(self): + json = {"$ref": "#/sub", "sub": [1, 2]} + result = replace_refs(json) + assert result == json["sub"] + + def test_self_referent_reference_w_merge(self): + json = {"$ref": "#/sub", "extra": "aoeu", "sub": {"main": "aoeu"}} + result = replace_refs(json, merge_props=True) + assert result == {"main": "aoeu", "extra": "aoeu", "sub": {"main": "aoeu"}} + def test_custom_loader(self): data = {"$ref": "foo"} loader = mock.Mock(return_value=42) From 690384cfa2c1d6ed89ed6b583a619aee669a85ef Mon Sep 17 00:00:00 2001 From: Chase Sterling Date: Wed, 4 Jan 2023 01:24:46 -0500 Subject: [PATCH 3/7] Allow extra properties with merge_props to resolve references within. --- jsonref.py | 6 +++++- tests.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/jsonref.py b/jsonref.py index 9a27f30..fd22c0b 100644 --- a/jsonref.py +++ b/jsonref.py @@ -152,9 +152,13 @@ def callback(self): and isinstance(result, Mapping) and len(self.__reference__) > 1 ): + extra_props = {k: v for k, v in self.__reference__.items() if k != "$ref"} + extra_props = _replace_refs( + extra_props, **{**self._ref_kwargs, "recursing": True} + ) result = { **result, - **{k: v for k, v in self.__reference__.items() if k != "$ref"}, + **extra_props, } return result diff --git a/tests.py b/tests.py index 07e5cd3..6f07741 100644 --- a/tests.py +++ b/tests.py @@ -113,6 +113,18 @@ def test_extra_ref_attributes(self, parametrized_replace_refs): } } + def test_refs_inside_extra_props(self): + """This seems really dubious... but OpenAPI 3.1 spec does it.""" + docs = { + "a.json": { + "file": "a", + "b": {"$ref": "b.json#/ba", "extra": {"$ref": "b.json#/bb"}}, + }, + "b.json": {"ba": {"a": 1}, "bb": {"b": 2}}, + } + result = replace_refs(docs["a.json"], loader=docs.get, merge_props=True) + assert result == {"file": "a", "b": {"a": 1, "extra": {"b": 2}}} + def test_recursive_extra(self, parametrized_replace_refs): json = {"a": {"$ref": "#", "extra": "foo"}} result = parametrized_replace_refs(json, merge_props=True) From 7a2b347dea3cbc8d694c45085c03039c3f3d36ab Mon Sep 17 00:00:00 2001 From: Chase Sterling Date: Wed, 4 Jan 2023 10:54:28 -0500 Subject: [PATCH 4/7] Remove some of the special casing for latest features --- jsonref.py | 54 ++++++++++++++++++++++++++---------------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/jsonref.py b/jsonref.py index fd22c0b..86c6147 100644 --- a/jsonref.py +++ b/jsonref.py @@ -137,12 +137,7 @@ def callback(self): ) else: base_doc = self.store[uri] - if base_doc is self: - # A reference pointing to a property within itself is dubious, - # but we'll allow it. Issue #51 - result = self.resolve_pointer(self.__reference__, fragment) - else: - result = self.resolve_pointer(base_doc, fragment) + result = self.resolve_pointer(base_doc, fragment) if result is self: raise self._error("Reference refers directly to itself.") if hasattr(result, "__subject__"): @@ -152,13 +147,9 @@ def callback(self): and isinstance(result, Mapping) and len(self.__reference__) > 1 ): - extra_props = {k: v for k, v in self.__reference__.items() if k != "$ref"} - extra_props = _replace_refs( - extra_props, **{**self._ref_kwargs, "recursing": True} - ) result = { **result, - **extra_props, + **{k: v for k, v in self.__reference__.items() if k != "$ref"}, } return result @@ -181,6 +172,9 @@ def resolve_pointer(self, document, pointer): part = int(part) except ValueError: pass + # If a reference points inside itself, it must mean inside reference object, not the referent data + if document is self: + document = self.__reference__ try: document = document[part] except (TypeError, LookupError) as e: @@ -369,20 +363,8 @@ def _replace_refs( base_uri = urlparse.urljoin(base_uri, id_) store_uri = base_uri - if isinstance(obj, Mapping) and isinstance(obj.get("$ref"), str): - obj = JsonRef( - obj, - base_uri=base_uri, - loader=loader, - jsonschema=jsonschema, - load_on_repr=load_on_repr, - merge_props=merge_props, - _path=path, - _store=store, - ) - # If our obj was not a json reference object, iterate through it, - # replacing children with JsonRefs - elif isinstance(obj, Mapping): + # First recursively iterate through our object, replacing children with JsonRefs + if isinstance(obj, Mapping): obj = { k: _replace_refs( v, @@ -412,8 +394,24 @@ def _replace_refs( ) for i, v in enumerate(obj) ] + + # If this object itself was a reference, replace it with a JsonRef + if isinstance(obj, Mapping) and isinstance(obj.get("$ref"), str): + obj = JsonRef( + obj, + base_uri=base_uri, + loader=loader, + jsonschema=jsonschema, + load_on_repr=load_on_repr, + merge_props=merge_props, + _path=path, + _store=store, + ) + + # Store the document with all references replaced in our cache if store_uri is not None: store[store_uri] = obj + return obj @@ -433,7 +431,7 @@ def load( proxied to their referent data. :param fp: File-like object containing JSON document - :param kwargs: This function takes any of the keyword arguments from + :param **kwargs: This function takes any of the keyword arguments from :func:`replace_refs`. Any other keyword arguments will be passed to :func:`json.load` @@ -470,7 +468,7 @@ def loads( proxied to their referent data. :param s: String containing JSON document - :param kwargs: This function takes any of the keyword arguments from + :param **kwargs: This function takes any of the keyword arguments from :func:`replace_refs`. Any other keyword arguments will be passed to :func:`json.loads` @@ -506,7 +504,7 @@ def load_uri( data. :param uri: URI to fetch the JSON from - :param kwargs: This function takes any of the keyword arguments from + :param **kwargs: This function takes any of the keyword arguments from :func:`replace_refs` """ From 9c7a4e99a8442521ff3c7b047073ad09c7223766 Mon Sep 17 00:00:00 2001 From: Chase Sterling Date: Wed, 4 Jan 2023 10:55:08 -0500 Subject: [PATCH 5/7] Parameterize some of the new tests --- tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests.py b/tests.py index 6f07741..1cb9c94 100644 --- a/tests.py +++ b/tests.py @@ -113,8 +113,8 @@ def test_extra_ref_attributes(self, parametrized_replace_refs): } } - def test_refs_inside_extra_props(self): - """This seems really dubious... but OpenAPI 3.1 spec does it.""" + def test_refs_inside_extra_props(self, parametrized_replace_refs): + """This seems really dubious per the spec... but OpenAPI 3.1 spec does it.""" docs = { "a.json": { "file": "a", @@ -122,7 +122,7 @@ def test_refs_inside_extra_props(self): }, "b.json": {"ba": {"a": 1}, "bb": {"b": 2}}, } - result = replace_refs(docs["a.json"], loader=docs.get, merge_props=True) + result = parametrized_replace_refs(docs["a.json"], loader=docs.get, merge_props=True) assert result == {"file": "a", "b": {"a": 1, "extra": {"b": 2}}} def test_recursive_extra(self, parametrized_replace_refs): @@ -246,14 +246,14 @@ def test_recursive_data_structures_remote_fragment(self): result = replace_refs(json1, base_uri="/json1", loader=loader) assert result["a"].__subject__ is result - def test_self_referent_reference(self): + def test_self_referent_reference(self, parametrized_replace_refs): json = {"$ref": "#/sub", "sub": [1, 2]} - result = replace_refs(json) + result = parametrized_replace_refs(json) assert result == json["sub"] - def test_self_referent_reference_w_merge(self): + def test_self_referent_reference_w_merge(self, parametrized_replace_refs): json = {"$ref": "#/sub", "extra": "aoeu", "sub": {"main": "aoeu"}} - result = replace_refs(json, merge_props=True) + result = parametrized_replace_refs(json, merge_props=True) assert result == {"main": "aoeu", "extra": "aoeu", "sub": {"main": "aoeu"}} def test_custom_loader(self): From 4a77dcb054ff911d991734cd13cffb5714cf5ad9 Mon Sep 17 00:00:00 2001 From: Chase Sterling Date: Wed, 4 Jan 2023 10:57:11 -0500 Subject: [PATCH 6/7] Bump version, and update declared supported python to versions we actuall test --- README.md | 2 +- docs/index.rst | 2 +- jsonref.py | 2 +- pyproject.toml | 3 +-- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c300c07..abb635e 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ `jsonref` is a library for automatic dereferencing of [JSON Reference](https://datatracker.ietf.org/doc/html/draft-pbryan-zyp-json-ref-03) -objects for Python (supporting Python 3.3+). +objects for Python (supporting Python 3.7+). This library lets you use a data structure with JSON reference objects, as if the references had been replaced with the referent data. diff --git a/docs/index.rst b/docs/index.rst index c99bc98..f0b39d5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,7 +8,7 @@ jsonref ``jsonref`` is a library for automatic dereferencing of `JSON Reference `_ -objects for Python (supporting Python 3.3+). +objects for Python (supporting Python 3.7+). .. testcode:: diff --git a/jsonref.py b/jsonref.py index 86c6147..10451a8 100644 --- a/jsonref.py +++ b/jsonref.py @@ -17,7 +17,7 @@ from proxytypes import LazyProxy -__version__ = "1.0.1" +__version__ = "1.1.0" class JsonRefError(Exception): diff --git a/pyproject.toml b/pyproject.toml index 2b91b51..9ab76c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ authors = [ license = {text = "MIT"} readme = "README.md" dynamic = ["version"] -requires-python = ">=3.3" +requires-python = ">=3.7" dependencies = [] [project.urls] @@ -28,4 +28,3 @@ build-backend = "pdm.pep517.api" [tool.isort] profile = "black" - From 92846565ce118e20a7bebd8d3d215cebd3a7a93d Mon Sep 17 00:00:00 2001 From: Chase Sterling Date: Wed, 4 Jan 2023 11:00:58 -0500 Subject: [PATCH 7/7] Add python 3.11 to test matrix --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9afd614..24ee9ad 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,6 +15,7 @@ jobs: - "3.8" - "3.9" - "3.10" + - "3.11" steps: - uses: actions/checkout@v3 @@ -35,4 +36,4 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | - pytest tests.py \ No newline at end of file + pytest tests.py