From e5d9b18f5438f59a80ff752ed808e46d7f71a65b Mon Sep 17 00:00:00 2001 From: Lawson Lewis Date: Mon, 22 Jul 2024 22:11:03 +1000 Subject: [PATCH 1/4] lint and format lint and format codebase with ruff and black --- poetry.lock | 29 +++++++++++++++++++- prez/app.py | 5 ++-- prez/models/__init__.py | 2 +- prez/models/model_exceptions.py | 2 +- prez/models/object_item.py | 11 ++------ prez/models/profiles_item.py | 1 - prez/queries/vocprez.py | 42 ++++++++++++++--------------- prez/renderers/json_renderer.py | 4 +-- prez/renderers/renderer.py | 2 +- prez/routers/management.py | 2 +- prez/routers/object.py | 28 +++---------------- prez/routers/profiles.py | 6 +++-- prez/routers/spaceprez.py | 36 ++++++++++++++++--------- prez/routers/sparql.py | 24 ++++++++++++----- prez/routers/vocprez.py | 19 +++++-------- prez/services/app_service.py | 22 +++++++-------- prez/services/connegp_service.py | 2 -- prez/services/cql_search.py | 4 +-- prez/services/curie_functions.py | 2 +- prez/services/exception_catchers.py | 9 ++++--- prez/services/generate_profiles.py | 2 +- prez/services/link_generation.py | 6 ++--- prez/services/objects.py | 3 --- prez/services/search_methods.py | 2 +- prez/sparql/methods.py | 18 +++++++++---- prez/sparql/objects_listings.py | 21 +++++++-------- pyproject.toml | 1 + tests/test_count.py | 2 +- tests/test_endpoints_catprez.py | 8 +++--- tests/test_endpoints_management.py | 2 +- tests/test_endpoints_object.py | 4 +-- tests/test_endpoints_spaceprez.py | 6 ++--- tests/test_endpoints_vocprez.py | 6 ++--- tests/test_remote_prefixes.py | 2 +- tests/test_sparql.py | 12 ++++----- 35 files changed, 180 insertions(+), 167 deletions(-) diff --git a/poetry.lock b/poetry.lock index 86505363..e7f6349f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1128,6 +1128,33 @@ pygments = ">=2.13.0,<3.0.0" [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] +[[package]] +name = "ruff" +version = "0.5.4" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.5.4-py3-none-linux_armv6l.whl", hash = "sha256:82acef724fc639699b4d3177ed5cc14c2a5aacd92edd578a9e846d5b5ec18ddf"}, + {file = "ruff-0.5.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:da62e87637c8838b325e65beee485f71eb36202ce8e3cdbc24b9fcb8b99a37be"}, + {file = "ruff-0.5.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e98ad088edfe2f3b85a925ee96da652028f093d6b9b56b76fc242d8abb8e2059"}, + {file = "ruff-0.5.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c55efbecc3152d614cfe6c2247a3054cfe358cefbf794f8c79c8575456efe19"}, + {file = "ruff-0.5.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f9b85eaa1f653abd0a70603b8b7008d9e00c9fa1bbd0bf40dad3f0c0bdd06793"}, + {file = "ruff-0.5.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cf497a47751be8c883059c4613ba2f50dd06ec672692de2811f039432875278"}, + {file = "ruff-0.5.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:09c14ed6a72af9ccc8d2e313d7acf7037f0faff43cde4b507e66f14e812e37f7"}, + {file = "ruff-0.5.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:628f6b8f97b8bad2490240aa84f3e68f390e13fabc9af5c0d3b96b485921cd60"}, + {file = "ruff-0.5.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3520a00c0563d7a7a7c324ad7e2cde2355733dafa9592c671fb2e9e3cd8194c1"}, + {file = "ruff-0.5.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93789f14ca2244fb91ed481456f6d0bb8af1f75a330e133b67d08f06ad85b516"}, + {file = "ruff-0.5.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:029454e2824eafa25b9df46882f7f7844d36fd8ce51c1b7f6d97e2615a57bbcc"}, + {file = "ruff-0.5.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:9492320eed573a13a0bc09a2957f17aa733fff9ce5bf00e66e6d4a88ec33813f"}, + {file = "ruff-0.5.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a6e1f62a92c645e2919b65c02e79d1f61e78a58eddaebca6c23659e7c7cb4ac7"}, + {file = "ruff-0.5.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:768fa9208df2bec4b2ce61dbc7c2ddd6b1be9fb48f1f8d3b78b3332c7d71c1ff"}, + {file = "ruff-0.5.4-py3-none-win32.whl", hash = "sha256:e1e7393e9c56128e870b233c82ceb42164966f25b30f68acbb24ed69ce9c3a4e"}, + {file = "ruff-0.5.4-py3-none-win_amd64.whl", hash = "sha256:58b54459221fd3f661a7329f177f091eb35cf7a603f01d9eb3eb11cc348d38c4"}, + {file = "ruff-0.5.4-py3-none-win_arm64.whl", hash = "sha256:bd53da65f1085fb5b307c38fd3c0829e76acf7b2a912d8d79cadcdb4875c1eb7"}, + {file = "ruff-0.5.4.tar.gz", hash = "sha256:2795726d5f71c4f4e70653273d1c23a8182f07dd8e48c12de5d867bfb7557eed"}, +] + [[package]] name = "scalene" version = "1.5.19" @@ -1539,4 +1566,4 @@ test = ["pytest (>=6.0.0)", "setuptools (>=65)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "4d8341b9f4f9d8333ec526655f69afd912430580909337a10b6fd69a1398fa91" +content-hash = "86b0f9acfaf67d345911d61953c65ad2b72ee546cdddde304676938c25844b37" diff --git a/prez/app.py b/prez/app.py index bc7ed9bb..ef793b60 100644 --- a/prez/app.py +++ b/prez/app.py @@ -20,7 +20,8 @@ from prez.models.model_exceptions import ( ClassNotFoundException, URINotFoundException, - NoProfilesException, InvalidSPARQLQueryException, + NoProfilesException, + InvalidSPARQLQueryException, ) from prez.routers.catprez import router as catprez_router from prez.routers.cql import router as cql_router @@ -168,7 +169,7 @@ def assemble_app( ClassNotFoundException: catch_class_not_found_exception, URINotFoundException: catch_uri_not_found_exception, NoProfilesException: catch_no_profiles_exception, - InvalidSPARQLQueryException: catch_invalid_sparql_query + InvalidSPARQLQueryException: catch_invalid_sparql_query, }, **kwargs ) diff --git a/prez/models/__init__.py b/prez/models/__init__.py index 680c5e73..49fbac67 100644 --- a/prez/models/__init__.py +++ b/prez/models/__init__.py @@ -1 +1 @@ -from prez.models.search_method import SearchMethod +from prez.models.search_method import SearchMethod as SearchMethod diff --git a/prez/models/model_exceptions.py b/prez/models/model_exceptions.py index 1f01e890..8e729247 100644 --- a/prez/models/model_exceptions.py +++ b/prez/models/model_exceptions.py @@ -44,4 +44,4 @@ class InvalidSPARQLQueryException(Exception): def __init__(self, error: str): self.message = f"Invalid SPARQL query: {error}" - super().__init__(self.message) \ No newline at end of file + super().__init__(self.message) diff --git a/prez/models/object_item.py b/prez/models/object_item.py index 6b19862c..001b5106 100644 --- a/prez/models/object_item.py +++ b/prez/models/object_item.py @@ -1,15 +1,8 @@ -from typing import Optional, FrozenSet, Tuple -from typing import Set +from typing import Optional, FrozenSet -from pydantic import BaseModel, root_validator +from pydantic import BaseModel from rdflib import URIRef, PROF -from prez.cache import endpoints_graph_cache -from prez.models.model_exceptions import ClassNotFoundException -from prez.reference_data.prez_ns import PREZ, ONT -from prez.services.curie_functions import get_uri_for_curie_id -from prez.services.model_methods import get_classes - class ObjectItem(BaseModel): uri: Optional[URIRef] = None diff --git a/prez/models/profiles_item.py b/prez/models/profiles_item.py index 7705233c..0379e8c7 100644 --- a/prez/models/profiles_item.py +++ b/prez/models/profiles_item.py @@ -7,7 +7,6 @@ from prez.cache import profiles_graph_cache from prez.config import settings from prez.services.curie_functions import get_uri_for_curie_id, get_curie_id_for_uri -from prez.services.model_methods import get_classes PREZ = Namespace("https://prez.dev/") diff --git a/prez/queries/vocprez.py b/prez/queries/vocprez.py index e7e5f11c..a7874d69 100644 --- a/prez/queries/vocprez.py +++ b/prez/queries/vocprez.py @@ -8,25 +8,25 @@ def get_concept_scheme_query(iri: str, bnode_depth: int) -> str: """ PREFIX prez: PREFIX skos: - - CONSTRUCT { + + CONSTRUCT { ?iri ?p ?o . - + {% if bnode_depth > 0 +%} ?iri ?p0 ?o0 . {% endif %} - + {% for i in range(bnode_depth) %} ?o{{ i }} ?p{{ i + 1 }} ?o{{ i + 1 }} . {% endfor %} - + ?iri prez:childrenCount ?childrenCount . } WHERE { BIND(<{{ iri }}> as ?iri) ?iri ?p ?o . FILTER (?p != skos:hasTopConcept) - + { SELECT (COUNT(?topConcept) AS ?childrenCount) WHERE { @@ -34,11 +34,11 @@ def get_concept_scheme_query(iri: str, bnode_depth: int) -> str: ?iri skos:hasTopConcept ?topConcept . } } - + {% if bnode_depth > 0 %} ?iri ?p0 ?o0 . {% endif %} - + {% for i in range(bnode_depth) %} ?o{{ i }} ?p{{ i + 1 }} ?o{{ i + 1 }} . FILTER (isBlank(?o0)) @@ -58,7 +58,7 @@ def get_concept_scheme_top_concepts_query(iri: str, page: int, per_page: int) -> PREFIX rdfs: PREFIX skos: PREFIX xsd: - + CONSTRUCT { ?concept skos:prefLabel ?label . ?concept prez:childrenCount ?narrowerChildrenCount . @@ -79,7 +79,7 @@ def get_concept_scheme_top_concepts_query(iri: str, page: int, per_page: int) -> } ?iri rdf:type ?type . ?concept rdf:type ?conceptType . - + { SELECT (COUNT(?childConcept) AS ?childrenCount) WHERE { @@ -87,19 +87,19 @@ def get_concept_scheme_top_concepts_query(iri: str, page: int, per_page: int) -> ?iri skos:hasTopConcept ?childConcept . } } - + { # Using two OPTIONAL clauses with a UNION causes ?narrowConcept to be duplicated. # Use DISTINCT to get an accurate count. SELECT ?concept ?label (COUNT(DISTINCT ?narrowerConcept) AS ?narrowerChildrenCount) WHERE { BIND(<{{ iri }}> as ?iri) - + { OPTIONAL { ?iri skos:hasTopConcept ?concept . ?concept skos:prefLabel ?label . - + OPTIONAL { ?narrowerConcept skos:broader ?concept . } @@ -112,7 +112,7 @@ def get_concept_scheme_top_concepts_query(iri: str, page: int, per_page: int) -> OPTIONAL { ?concept skos:topConceptOf ?iri . ?concept skos:prefLabel ?label . - + OPTIONAL { ?narrowerConcept skos:broader ?concept . } @@ -141,7 +141,7 @@ def get_concept_narrowers_query(iri: str, page: int, per_page: int) -> str: PREFIX rdf: PREFIX rdfs: PREFIX skos: - + CONSTRUCT { ?concept skos:prefLabel ?label . ?concept prez:childrenCount ?narrowerChildrenCount . @@ -161,7 +161,7 @@ def get_concept_narrowers_query(iri: str, page: int, per_page: int) -> str: ?concept skos:prefLabel ?label . } ?iri rdf:type ?type . - + { SELECT (COUNT(?childConcept) AS ?childrenCount) WHERE { @@ -169,17 +169,17 @@ def get_concept_narrowers_query(iri: str, page: int, per_page: int) -> str: ?childConcept skos:broader ?iri . } } - + { SELECT ?concept ?label (skos:Concept AS ?conceptType) (COUNT(?narrowerConcept) AS ?narrowerChildrenCount) WHERE { BIND(<{{ iri }}> as ?iri) - + { OPTIONAL { ?concept skos:broader ?iri . ?concept skos:prefLabel ?label . - + OPTIONAL { ?narrowerConcept skos:broader ?concept . } @@ -192,7 +192,7 @@ def get_concept_narrowers_query(iri: str, page: int, per_page: int) -> str: OPTIONAL { ?iri skos:narrower ?concept . ?concept skos:prefLabel ?label . - + OPTIONAL { ?narrowerConcept skos:broader ?concept . } @@ -201,7 +201,7 @@ def get_concept_narrowers_query(iri: str, page: int, per_page: int) -> str: } } } - + # Filter out any unbound ?concept rows which are invalid and may contain # a count of 0. This is possible because both paths within the select # query are using the OPTIONAL clause. diff --git a/prez/renderers/json_renderer.py b/prez/renderers/json_renderer.py index 602c15f9..ed3e81b8 100644 --- a/prez/renderers/json_renderer.py +++ b/prez/renderers/json_renderer.py @@ -1,6 +1,4 @@ -from itertools import chain - -from rdflib import Graph, URIRef, RDF, SH, Literal +from rdflib import Graph, URIRef, RDF, SH from rdflib.term import Node from prez.cache import profiles_graph_cache diff --git a/prez/renderers/renderer.py b/prez/renderers/renderer.py index 5a4faf00..80030425 100644 --- a/prez/renderers/renderer.py +++ b/prez/renderers/renderer.py @@ -70,7 +70,7 @@ async def return_from_graph( else: if "anot+" in mediatype: non_anot_mediatype = mediatype.replace("anot+", "") - profile_headers['Content-Type'] = non_anot_mediatype + profile_headers["Content-Type"] = non_anot_mediatype graph = await return_annotated_rdf(graph, profile, repo) content = io.BytesIO( graph.serialize(format=non_anot_mediatype, encoding="utf-8") diff --git a/prez/routers/management.py b/prez/routers/management.py index 98478136..fe67faaa 100644 --- a/prez/routers/management.py +++ b/prez/routers/management.py @@ -28,7 +28,7 @@ async def index(): g.add((URIRef(settings.system_uri), PREZ.version, Literal(settings.prez_version))) g += endpoints_graph_cache g += await return_annotation_predicates() - log.info(f"Populated API info") + log.info("Populated API info") return await return_rdf(g, "text/turtle", profile_headers={}) diff --git a/prez/routers/object.py b/prez/routers/object.py index 96728780..76b5bf7b 100644 --- a/prez/routers/object.py +++ b/prez/routers/object.py @@ -1,35 +1,11 @@ -from string import Template -from typing import FrozenSet, Optional - from fastapi import APIRouter, Request, HTTPException, status, Query from fastapi import Depends -from rdflib import Graph, Literal, URIRef, PROF, DCTERMS from starlette.responses import PlainTextResponse -from prez.cache import ( - endpoints_graph_cache, - profiles_graph_cache, - links_ids_graph_cache, -) from prez.dependencies import get_repo, get_system_repo -from prez.models.listing import ListingModel -from prez.models.object_item import ObjectItem -from prez.models.profiles_and_mediatypes import ProfilesMediatypesInfo from prez.queries.object import object_inbound_query, object_outbound_query -from prez.reference_data.prez_ns import PREZ -from prez.renderers.renderer import return_from_graph, return_profiles from prez.routers.identifier import get_iri_route -from prez.services.curie_functions import get_curie_id_for_uri, get_uri_for_curie_id -from prez.services.model_methods import get_classes from prez.services.objects import object_function -from prez.sparql.methods import Repo -from prez.sparql.objects_listings import ( - get_endpoint_template_queries, - generate_relationship_query, - generate_item_construct, - generate_listing_construct, - generate_listing_count_construct, -) router = APIRouter(tags=["Object"]) @@ -93,5 +69,7 @@ async def count_route( @router.get("/object", summary="Object", name="https://prez.dev/endpoint/object") -async def object_route(request: Request, repo=Depends(get_repo), system_repo=Depends(get_system_repo)): +async def object_route( + request: Request, repo=Depends(get_repo), system_repo=Depends(get_system_repo) +): return await object_function(request, repo=repo, system_repo=system_repo) diff --git a/prez/routers/profiles.py b/prez/routers/profiles.py index cbf5ba69..bddef39e 100644 --- a/prez/routers/profiles.py +++ b/prez/routers/profiles.py @@ -1,6 +1,6 @@ from fastapi import APIRouter, Request, Depends -from prez.dependencies import get_repo, get_system_repo +from prez.dependencies import get_system_repo from prez.services.objects import object_function from prez.services.listings import listing_function @@ -44,4 +44,6 @@ async def profiles( name="https://prez.dev/endpoint/profile", ) async def profile(request: Request, profile_curie: str, repo=Depends(get_system_repo)): - return await object_function(request, object_curie=profile_curie, repo=repo, system_repo=repo) + return await object_function( + request, object_curie=profile_curie, repo=repo, system_repo=repo + ) diff --git a/prez/routers/spaceprez.py b/prez/routers/spaceprez.py index 31e475dd..5cfe2b7d 100644 --- a/prez/routers/spaceprez.py +++ b/prez/routers/spaceprez.py @@ -25,12 +25,16 @@ async def spaceprez_profiles(): async def list_datasets( request: Request, repo: Repo = Depends(get_repo), - system_repo: Repo = Depends(get_system_repo), - page: Optional[int] = 1, + system_repo: Repo = Depends(get_system_repo), + page: Optional[int] = 1, per_page: Optional[int] = 20, ): return await listing_function( - request=request, page=page, per_page=per_page, repo=repo, system_repo=system_repo + request=request, + page=page, + per_page=per_page, + repo=repo, + system_repo=system_repo, ) @@ -43,8 +47,8 @@ async def list_feature_collections( request: Request, dataset_curie: str, repo: Repo = Depends(get_repo), - system_repo: Repo = Depends(get_system_repo), - page: Optional[int] = 1, + system_repo: Repo = Depends(get_system_repo), + page: Optional[int] = 1, per_page: Optional[int] = 20, ): dataset_uri = get_uri_for_curie_id(dataset_curie) @@ -68,8 +72,8 @@ async def list_features( dataset_curie: str, collection_curie: str, repo: Repo = Depends(get_repo), - system_repo: Repo = Depends(get_system_repo), - page: Optional[int] = 1, + system_repo: Repo = Depends(get_system_repo), + page: Optional[int] = 1, per_page: Optional[int] = 20, ): collection_uri = get_uri_for_curie_id(collection_curie) @@ -92,9 +96,11 @@ async def dataset_item( request: Request, dataset_curie: str, repo: Repo = Depends(get_repo), - system_repo: Repo = Depends(get_system_repo), + system_repo: Repo = Depends(get_system_repo), ): - return await object_function(request, object_curie=dataset_curie, repo=repo, system_repo=system_repo) + return await object_function( + request, object_curie=dataset_curie, repo=repo, system_repo=system_repo + ) @router.get( @@ -107,9 +113,11 @@ async def feature_collection_item( dataset_curie: str, collection_curie: str, repo: Repo = Depends(get_repo), - system_repo: Repo = Depends(get_system_repo), + system_repo: Repo = Depends(get_system_repo), ): - return await object_function(request, object_curie=collection_curie, repo=repo, system_repo=system_repo) + return await object_function( + request, object_curie=collection_curie, repo=repo, system_repo=system_repo + ) @router.get( @@ -123,6 +131,8 @@ async def feature_item( collection_curie: str, feature_curie: str, repo: Repo = Depends(get_repo), - system_repo: Repo = Depends(get_system_repo), + system_repo: Repo = Depends(get_system_repo), ): - return await object_function(request=request, object_curie=feature_curie, repo=repo, system_repo=system_repo) + return await object_function( + request=request, object_curie=feature_curie, repo=repo, system_repo=system_repo + ) diff --git a/prez/routers/sparql.py b/prez/routers/sparql.py index 6986c8e5..7db30d8e 100644 --- a/prez/routers/sparql.py +++ b/prez/routers/sparql.py @@ -1,9 +1,10 @@ import io from typing import Annotated +import httpx from fastapi import APIRouter, Depends, Form from fastapi.responses import JSONResponse, Response -from rdflib import Namespace, Graph +from rdflib import Graph, Namespace from starlette.background import BackgroundTask from starlette.datastructures import Headers from starlette.requests import Request @@ -24,7 +25,9 @@ async def sparql_post_passthrough( # To maintain compatibility with the other SPARQL endpoints, # /sparql POST endpoint is not a JSON API, it uses # values encoded with x-www-form-urlencoded - query: Annotated[str, Form()], # Pydantic validation prevents update queries (the Form would need to be "update") + query: Annotated[ + str, Form() + ], # Pydantic validation prevents update queries (the Form would need to be "update") request: Request, repo: Repo = Depends(get_repo), ): @@ -40,7 +43,9 @@ async def sparql_get_passthrough( return await sparql_endpoint_handler(query, request, repo, method="GET") -async def sparql_endpoint_handler(query: str, request: Request, repo: Repo, method="GET"): +async def sparql_endpoint_handler( + query: str, request: Request, repo: Repo, method="GET" +): request_mediatype = request.headers.get("accept").split(",")[0] # can't default the MT where not provided as it could be # graph (CONSTRUCT like queries) or tabular (SELECT queries) @@ -65,13 +70,14 @@ async def sparql_endpoint_handler(query: str, request: Request, repo: Repo, meth media_type=non_anot_mediatype, headers=prof_and_mt_info.profile_headers, ) - query_result: 'httpx.Response' = await repo.sparql(query, request.headers.raw, method=method) + query_result: httpx.Response = await repo.sparql( + query, request.headers.raw, method=method + ) if isinstance(query_result, dict): return JSONResponse(content=query_result) elif isinstance(query_result, Graph): return Response( - content=query_result.serialize(format="text/turtle"), - status_code=200 + content=query_result.serialize(format="text/turtle"), status_code=200 ) dispositions = query_result.headers.get_list("Content-Disposition") @@ -85,7 +91,11 @@ async def sparql_endpoint_handler(query: str, request: Request, repo: Repo, meth # remove transfer-encoding chunked, disposition=attachment, and content-length headers = dict() for k, v in query_result.headers.items(): - if k.lower() not in ("transfer-encoding", "content-disposition", "content-length"): + if k.lower() not in ( + "transfer-encoding", + "content-disposition", + "content-length", + ): headers[k] = v content = await query_result.aread() await query_result.aclose() diff --git a/prez/routers/vocprez.py b/prez/routers/vocprez.py index 61d56278..7b4f001f 100644 --- a/prez/routers/vocprez.py +++ b/prez/routers/vocprez.py @@ -1,28 +1,24 @@ import logging -from fastapi import APIRouter, Request -from fastapi import Depends +from fastapi import APIRouter, Depends, Request from fastapi.responses import RedirectResponse -from rdflib import URIRef, SKOS +from rdflib import SKOS from starlette.responses import PlainTextResponse from prez.bnode import get_bnode_depth from prez.dependencies import get_repo, get_system_repo from prez.models.profiles_and_mediatypes import ProfilesMediatypesInfo from prez.queries.vocprez import ( + get_concept_narrowers_query, get_concept_scheme_query, get_concept_scheme_top_concepts_query, - get_concept_narrowers_query, -) -from prez.renderers.renderer import ( - return_from_graph, ) +from prez.renderers.renderer import return_from_graph from prez.response import StreamingTurtleAnnotatedResponse from prez.routers.identifier import get_iri_route -from prez.services.objects import object_function -from prez.services.listings import listing_function from prez.services.link_generation import _add_prez_links -from prez.services.curie_functions import get_curie_id_for_uri +from prez.services.listings import listing_function +from prez.services.objects import object_function from prez.sparql.methods import Repo from prez.sparql.resource import get_resource @@ -178,9 +174,6 @@ async def concept_scheme_top_concepts_route( ) graph, _ = await repo.send_queries([concept_scheme_top_concepts_query], []) - for concept in graph.objects(iri, SKOS.hasTopConcept): - if isinstance(concept, URIRef): - concept_curie = get_curie_id_for_uri(concept) if "anot+" in profiles_mediatypes_info.mediatype: await _add_prez_links(graph, repo, system_repo) return await return_from_graph( diff --git a/prez/services/app_service.py b/prez/services/app_service.py index 59d4824c..1d0569ce 100644 --- a/prez/services/app_service.py +++ b/prez/services/app_service.py @@ -40,7 +40,7 @@ async def healthcheck_sparql_endpoints(): ) response.raise_for_status() if response.status_code == 200: - log.info(f"Successfully connected to triplestore SPARQL endpoint") + log.info("Successfully connected to triplestore SPARQL endpoint") connected_to_triplestore = True except httpx.HTTPError as exc: log.error(f"HTTP Exception for {exc.request.url} - {exc}") @@ -83,7 +83,7 @@ async def populate_api_info(): prez_system_graph.add( (URIRef(settings.system_uri), PREZ.version, Literal(settings.prez_version)) ) - log.info(f"Populated API info") + log.info("Populated API info") async def add_prefixes_to_prefix_graph(repo: Repo): @@ -91,9 +91,9 @@ async def add_prefixes_to_prefix_graph(repo: Repo): Adds prefixes to the prefix graph """ # look for remote prefixes - remote_prefix_query = f""" - CONSTRUCT WHERE {{ ?bn ?prefix; - ?namespace. }} + remote_prefix_query = """ + CONSTRUCT WHERE { ?bn ?prefix; + ?namespace. } """ remote_prefix_g, _ = await repo.send_queries([remote_prefix_query], []) if remote_prefix_g: @@ -175,20 +175,20 @@ async def create_endpoints_graph(repo) -> Graph: async def get_remote_endpoint_definitions(repo): - remote_endpoints_query = f""" + remote_endpoints_query = """ PREFIX ont: -CONSTRUCT {{ +CONSTRUCT { ?endpoint ?p ?o. -}} -WHERE {{ +} +WHERE { ?endpoint a ont:Endpoint; ?p ?o. -}} +} """ g, _ = await repo.send_queries([remote_endpoints_query], []) if len(g) > 0: endpoints_graph_cache.__iadd__(g) - log.info(f"Remote endpoint definition(s) found and added") + log.info("Remote endpoint definition(s) found and added") else: log.info("No remote endpoint definitions found") diff --git a/prez/services/connegp_service.py b/prez/services/connegp_service.py index ce1cdbff..2d80fb55 100644 --- a/prez/services/connegp_service.py +++ b/prez/services/connegp_service.py @@ -1,5 +1,3 @@ -import time - from connegp import Connegp from fastapi import Request diff --git a/prez/services/cql_search.py b/prez/services/cql_search.py index 17be1512..20ab0a05 100644 --- a/prez/services/cql_search.py +++ b/prez/services/cql_search.py @@ -3,10 +3,10 @@ from fastapi import HTTPException +from prez.config import settings -class CQLSearch(object): - from prez.config import settings +class CQLSearch(object): def __init__(self, cql_query: str, sparql_query: str) -> None: self.cql_query = cql_query self.sparql_query = sparql_query diff --git a/prez/services/curie_functions.py b/prez/services/curie_functions.py index d04a9afd..b0c4388a 100644 --- a/prez/services/curie_functions.py +++ b/prez/services/curie_functions.py @@ -40,7 +40,7 @@ def generate_new_prefix(uri): else: ns = f'{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path.rsplit("/", 1)[0]}/' - split_prefix_path = ns[:-1].rsplit('/', 1) + split_prefix_path = ns[:-1].rsplit("/", 1) if len(split_prefix_path) > 1: to_generate_prefix_from = split_prefix_path[-1].lower() # attempt to just use the last part of the path prior to the fragment or "identifier" diff --git a/prez/services/exception_catchers.py b/prez/services/exception_catchers.py index a474f5eb..c016ae03 100644 --- a/prez/services/exception_catchers.py +++ b/prez/services/exception_catchers.py @@ -5,7 +5,8 @@ from prez.models.model_exceptions import ( ClassNotFoundException, URINotFoundException, - NoProfilesException, InvalidSPARQLQueryException, + NoProfilesException, + InvalidSPARQLQueryException, ) @@ -53,11 +54,13 @@ async def catch_no_profiles_exception(request: Request, exc: NoProfilesException ) -async def catch_invalid_sparql_query(request: Request, exc: InvalidSPARQLQueryException): +async def catch_invalid_sparql_query( + request: Request, exc: InvalidSPARQLQueryException +): return JSONResponse( status_code=400, content={ "error": "Bad Request", "detail": exc.message, }, - ) \ No newline at end of file + ) diff --git a/prez/services/generate_profiles.py b/prez/services/generate_profiles.py index 3d17b0cf..5b00dd03 100644 --- a/prez/services/generate_profiles.py +++ b/prez/services/generate_profiles.py @@ -68,7 +68,7 @@ async def create_profiles_graph(repo) -> Graph: g, _ = await repo.send_queries([remote_profiles_query], []) if len(g) > 0: profiles_graph_cache.__iadd__(g) - log.info(f"Remote profile(s) found and added") + log.info("Remote profile(s) found and added") else: log.info("No remote profiles found") # add profiles internal links diff --git a/prez/services/link_generation.py b/prez/services/link_generation.py index 7f1b3ea6..b903ca25 100644 --- a/prez/services/link_generation.py +++ b/prez/services/link_generation.py @@ -1,11 +1,9 @@ from string import Template from typing import FrozenSet -from fastapi import Depends -from rdflib import Graph, Literal, URIRef, DCTERMS, BNode +from rdflib import Graph, Literal, URIRef, DCTERMS -from prez.cache import endpoints_graph_cache, links_ids_graph_cache -from prez.dependencies import get_system_repo +from prez.cache import links_ids_graph_cache from prez.reference_data.prez_ns import PREZ from prez.services.curie_functions import get_curie_id_for_uri from prez.services.model_methods import get_classes diff --git a/prez/services/objects.py b/prez/services/objects.py index 16decbd7..3e47010e 100644 --- a/prez/services/objects.py +++ b/prez/services/objects.py @@ -1,12 +1,9 @@ from typing import Optional -from fastapi import Depends from fastapi import Request, HTTPException from rdflib import URIRef from prez.cache import profiles_graph_cache -from prez.config import settings -from prez.dependencies import get_repo from prez.models.object_item import ObjectItem from prez.models.profiles_and_mediatypes import ProfilesMediatypesInfo from prez.reference_data.prez_ns import PREZ diff --git a/prez/services/search_methods.py b/prez/services/search_methods.py index c5d73117..c1fb929a 100644 --- a/prez/services/search_methods.py +++ b/prez/services/search_methods.py @@ -26,7 +26,7 @@ async def get_remote_search_methods(repo): graph, _ = await repo.send_queries([remote_search_methods_query], []) if len(graph) > 1: await generate_search_methods(graph) - log.info(f"Remote search methods found and added.") + log.info("Remote search methods found and added.") else: log.info("No remote search methods found.") diff --git a/prez/sparql/methods.py b/prez/sparql/methods.py index 7657e127..d1e2cfe4 100644 --- a/prez/sparql/methods.py +++ b/prez/sparql/methods.py @@ -28,7 +28,7 @@ async def tabular_query_to_table(self, query: str, context: URIRef = None): pass async def send_queries( - self, rdf_queries: List[str], tabular_queries: List[Tuple[URIRef, str]] = None + self, rdf_queries: List[str], tabular_queries: List[Tuple[URIRef, str]] = None ): # Common logic to send both query types in parallel results = await asyncio.gather( @@ -49,7 +49,9 @@ async def send_queries( return g, tabular_results @abstractmethod - def sparql(self, query: str, raw_headers: list[tuple[bytes, bytes]], method: str = "GET"): + def sparql( + self, query: str, raw_headers: list[tuple[bytes, bytes]], method: str = "GET" + ): pass @@ -114,7 +116,9 @@ async def sparql( content = query_escaped_as_bytes headers.append((b"host", str(url.host).encode("utf-8"))) - rp_req = self.async_client.build_request(method, url, headers=headers, content=content) + rp_req = self.async_client.build_request( + method, url, headers=headers, content=content + ) return await self.async_client.send(rp_req, stream=True) @@ -122,7 +126,9 @@ class PyoxigraphRepo(Repo): def __init__(self, pyoxi_store: pyoxigraph.Store): self.pyoxi_store = pyoxi_store - def _handle_query_solution_results(self, results: pyoxigraph.QuerySolutions) -> dict: + def _handle_query_solution_results( + self, results: pyoxigraph.QuerySolutions + ) -> dict: """Organise the query results into format serializable by FastAPIs JSONResponse.""" variables = results.variables results_dict = {"head": {"vars": [v.value for v in results.variables]}} @@ -188,7 +194,9 @@ async def tabular_query_to_table(self, query: str, context: URIRef = None) -> li self._sync_tabular_query_to_table, query, context ) - async def sparql(self, query: str, raw_headers: list[tuple[bytes, bytes]], method: str = "") -> list | Graph | bool: + async def sparql( + self, query: str, raw_headers: list[tuple[bytes, bytes]], method: str = "" + ) -> list | Graph | bool: return self._sparql(query) @staticmethod diff --git a/prez/sparql/objects_listings.py b/prez/sparql/objects_listings.py index c60f1d7d..8ad779ea 100644 --- a/prez/sparql/objects_listings.py +++ b/prez/sparql/objects_listings.py @@ -2,15 +2,14 @@ from functools import lru_cache from itertools import chain from textwrap import dedent -from typing import List, Optional, Tuple, Dict, FrozenSet +from typing import Dict, FrozenSet, List, Optional, Tuple -from rdflib import Graph, URIRef, Namespace, Literal +from rdflib import Graph, Literal, Namespace, URIRef -from prez.cache import endpoints_graph_cache, tbox_cache, profiles_graph_cache +from prez.cache import endpoints_graph_cache, profiles_graph_cache, tbox_cache from prez.config import settings from prez.models import SearchMethod from prez.models.listing import ListingModel -from prez.models.profiles_item import ProfileItem from prez.models.profiles_listings import ProfilesMembers from prez.reference_data.prez_ns import ONT from prez.services.curie_functions import get_uri_for_curie_id @@ -45,9 +44,7 @@ def generate_listing_construct( ) = get_item_predicates(profile, focus_item.selected_class) else: # for objects, this context is already included in the separate "generate_item_construct" function, so these # predicates are explicitly set to None here to avoid duplication. - include_predicates = ( - exclude_predicates - ) = inverse_predicates = sequence_predicates = None + include_predicates = sequence_predicates = None ( child_to_focus, parent_to_focus, @@ -196,7 +193,7 @@ def generate_item_construct(focus_item, profile: URIRef): def search_query_construct(): return dedent( - f"""?hashID a prez:SearchResult ; + """?hashID a prez:SearchResult ; prez:searchResultWeight ?weight ; prez:searchResultPredicate ?predicate ; prez:searchResultMatch ?match ; @@ -234,7 +231,7 @@ def generate_relative_properties( for k, v in kvs.items(): if v: if construct_select == "select": - rel_string += f"""OPTIONAL {{ """ + rel_string += """OPTIONAL { """ rel_string += f"""?{other_kvs[k]} ?rel_{k}_props ?rel_{k}_val .\n""" if construct_select == "select": rel_string += f"""VALUES ?rel_{k}_props {{ {" ".join('<' + str(pred) + '>' for pred in relative_properties)} }} }}\n""" @@ -899,6 +896,6 @@ def startup_count_objects(): """ Retrieves hardcoded counts for collections in the dataset (feature collections, datasets etc.) """ - return f"""PREFIX prez: -CONSTRUCT {{ ?collection prez:count ?count }} -WHERE {{ ?collection prez:count ?count }}""" + return """PREFIX prez: +CONSTRUCT { ?collection prez:count ?count } +WHERE { ?collection prez:count ?count }""" diff --git a/pyproject.toml b/pyproject.toml index 564d34e5..507cf30b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ scalene = "^1.5.18" python-dotenv = "^1.0.0" pyoxigraph = "^0.3.19" coverage = "^7.3.2" +ruff = "^0.5.4" [build-system] requires = ["poetry-core>=1.9.0"] diff --git a/tests/test_count.py b/tests/test_count.py index c8969ccd..b7aac662 100644 --- a/tests/test_count.py +++ b/tests/test_count.py @@ -82,5 +82,5 @@ def test_count( ): curie = get_curie(test_client, iri) params = {"curie": curie, "inbound": inbound, "outbound": outbound} - response = test_client.get(f"/count", params=params) + response = test_client.get("/count", params=params) assert int(response.text) == count diff --git a/tests/test_endpoints_catprez.py b/tests/test_endpoints_catprez.py index 5cb3487a..d9bc2969 100644 --- a/tests/test_endpoints_catprez.py +++ b/tests/test_endpoints_catprez.py @@ -51,7 +51,7 @@ def a_catalog_link(client): r = client.get("/c/catalogs") g = Graph().parse(data=r.text) member_uri = g.value(None, RDF.type, DCAT.Catalog) - link = g.value(member_uri, URIRef(f"https://prez.dev/link", None)) + link = g.value(member_uri, URIRef("https://prez.dev/link", None)) return link @@ -59,7 +59,7 @@ def a_catalog_link(client): def a_resource_link(client, a_catalog_link): r = client.get(a_catalog_link) g = Graph().parse(data=r.text) - links = g.objects(subject=None, predicate=URIRef(f"https://prez.dev/link")) + links = g.objects(subject=None, predicate=URIRef("https://prez.dev/link")) for link in links: if link != a_catalog_link: return link @@ -67,7 +67,7 @@ def a_resource_link(client, a_catalog_link): # @pytest.mark.xfail(reason="passes locally - setting to xfail pending test changes to pyoxigraph") def test_catalog_listing_anot(client): - r = client.get(f"/c/catalogs?_mediatype=text/anot+turtle") + r = client.get("/c/catalogs?_mediatype=text/anot+turtle") response_graph = Graph().parse(data=r.text) expected_graph = Graph().parse( Path(__file__).parent @@ -80,7 +80,7 @@ def test_catalog_listing_anot(client): def test_catalog_anot(client, a_catalog_link): - r = client.get(f"/c/catalogs/pd:democat?_mediatype=text/anot+turtle") + r = client.get("/c/catalogs/pd:democat?_mediatype=text/anot+turtle") response_graph = Graph().parse(data=r.text) expected_graph = Graph().parse( Path(__file__).parent diff --git a/tests/test_endpoints_management.py b/tests/test_endpoints_management.py index 08660035..56495dca 100644 --- a/tests/test_endpoints_management.py +++ b/tests/test_endpoints_management.py @@ -46,7 +46,7 @@ def override_get_repo(): def test_annotation_predicates(client): - r = client.get(f"/") + r = client.get("/") response_graph = Graph().parse(data=r.text) labelList = list( response_graph.objects( diff --git a/tests/test_endpoints_object.py b/tests/test_endpoints_object.py index 8f397ce1..a90bf770 100644 --- a/tests/test_endpoints_object.py +++ b/tests/test_endpoints_object.py @@ -59,7 +59,7 @@ def test_object_endpoint_sp_dataset(test_client, dataset_uri): def test_feature_collection(test_client): - r = test_client.get(f"/object?uri=https://test/feature-collection") + r = test_client.get("/object?uri=https://test/feature-collection") response_graph = Graph().parse(data=r.text) expected_graph = Graph().parse( Path(__file__).parent / "../tests/data/object/expected_responses/fc.ttl" @@ -72,7 +72,7 @@ def test_feature_collection(test_client): def test_feature(test_client): r = test_client.get( - f"/object?uri=https://linked.data.gov.au/datasets/geofabric/hydroid/102208962" + "/object?uri=https://linked.data.gov.au/datasets/geofabric/hydroid/102208962" ) response_graph = Graph().parse(data=r.text) expected_graph = Graph().parse( diff --git a/tests/test_endpoints_spaceprez.py b/tests/test_endpoints_spaceprez.py index c095c68f..c12052e5 100644 --- a/tests/test_endpoints_spaceprez.py +++ b/tests/test_endpoints_spaceprez.py @@ -50,7 +50,7 @@ def a_dataset_link(client): r = client.get("/s/datasets") g = Graph().parse(data=r.text) member_uri = g.value(None, RDF.type, DCAT.Dataset) - link = g.value(member_uri, URIRef(f"https://prez.dev/link", None)) + link = g.value(member_uri, URIRef("https://prez.dev/link", None)) return link @@ -61,7 +61,7 @@ def an_fc_link(client, a_dataset_link): member_uri = g.value( URIRef("http://example.com/datasets/sandgate"), RDFS.member, None ) - link = g.value(member_uri, URIRef(f"https://prez.dev/link", None)) + link = g.value(member_uri, URIRef("https://prez.dev/link", None)) return link @@ -72,7 +72,7 @@ def a_feature_link(client, an_fc_link): member_uri = g.value( URIRef("http://example.com/datasets/sandgate/catchments"), RDFS.member, None ) - link = g.value(member_uri, URIRef(f"https://prez.dev/link", None)) + link = g.value(member_uri, URIRef("https://prez.dev/link", None)) return link diff --git a/tests/test_endpoints_vocprez.py b/tests/test_endpoints_vocprez.py index d5090ef6..784ef01a 100644 --- a/tests/test_endpoints_vocprez.py +++ b/tests/test_endpoints_vocprez.py @@ -50,7 +50,7 @@ def links(test_client: TestClient): r = test_client.get("/v/collection") g = Graph().parse(data=r.text) vocab_uri = URIRef("http://resource.geosciml.org/classifier/cgi/contacttype") - vocab_link = g.value(vocab_uri, URIRef(f"https://prez.dev/link", None)) + vocab_link = g.value(vocab_uri, URIRef("https://prez.dev/link", None)) # vocab_uri = g.value(None, RDF.type, SKOS.ConceptScheme) # vocab_link = g.value(member_uri, URIRef(f"https://prez.dev/link", None)) return vocab_link @@ -64,7 +64,7 @@ def get_curie(test_client: TestClient, iri: str) -> str: def test_vocab_listing(test_client: TestClient): - response = test_client.get(f"/v/vocab?_mediatype=text/anot+turtle") + response = test_client.get("/v/vocab?_mediatype=text/anot+turtle") response_graph = Graph().parse(data=response.text) expected_graph = Graph().parse( Path(__file__).parent @@ -225,7 +225,7 @@ def test_concept( def test_collection_listing(test_client: TestClient): - response = test_client.get(f"/v/collection?_mediatype=text/anot+turtle") + response = test_client.get("/v/collection?_mediatype=text/anot+turtle") response_graph = Graph().parse(data=response.text, format="turtle") expected_graph = Graph().parse( Path(__file__).parent diff --git a/tests/test_remote_prefixes.py b/tests/test_remote_prefixes.py index 4357f001..69ccbae7 100644 --- a/tests/test_remote_prefixes.py +++ b/tests/test_remote_prefixes.py @@ -53,5 +53,5 @@ def test_catalog_link(client): r = client.get("/c/catalogs") g = Graph().parse(data=r.text) member_uri = g.value(None, RDF.type, DCAT.Catalog) - link = str(g.value(member_uri, URIRef(f"https://prez.dev/link", None))) + link = str(g.value(member_uri, URIRef("https://prez.dev/link", None))) assert link == "/c/catalogs/davo:bogusCatalogous" diff --git a/tests/test_sparql.py b/tests/test_sparql.py index c2b8059a..0a507491 100644 --- a/tests/test_sparql.py +++ b/tests/test_sparql.py @@ -6,7 +6,7 @@ from prez.app import assemble_app from prez.dependencies import get_repo -from prez.sparql.methods import Repo, PyoxigraphRepo +from prez.sparql.methods import PyoxigraphRepo, Repo @pytest.fixture(scope="session") @@ -48,7 +48,7 @@ def test_select(client): r = client.get( "/sparql?query=SELECT%20*%0AWHERE%20%7B%0A%20%20%3Fs%20%3Fp%20%3Fo%0A%7D%20LIMIT%201" ) - assert (r.status_code, 200) + assert r.status_code, 200 def test_construct(client): @@ -56,7 +56,7 @@ def test_construct(client): r = client.get( "/sparql?query=CONSTRUCT%20%7B%0A%20%20%3Fs%20%3Fp%20%3Fo%0A%7D%20WHERE%20%7B%0A%20%20%3Fs%20%3Fp%20%3Fo%0A%7D%20LIMIT%201" ) - assert (r.status_code, 200) + assert r.status_code, 200 def test_ask(client): @@ -64,7 +64,7 @@ def test_ask(client): r = client.get( "/sparql?query=PREFIX%20ex%3A%20%3Chttp%3A%2F%2Fexample.com%2Fdatasets%2F%3E%0APREFIX%20dcterms%3A%20%3Chttp%3A%2F%2Fpurl.org%2Fdc%2Fterms%2F%3E%0A%0AASK%0AWHERE%20%7B%0A%20%20%3Fsubject%20dcterms%3Atitle%20%3Ftitle%20.%0A%20%20FILTER%20CONTAINS(LCASE(%3Ftitle)%2C%20%22sandgate%22)%0A%7D" ) - assert (r.status_code, 200) + assert r.status_code, 200 def test_post(client): @@ -76,7 +76,7 @@ def test_post(client): "format": "application/x-www-form-urlencoded", }, ) - assert (r.status_code, 200) + assert r.status_code, 200 def test_post_invalid_data(client): @@ -102,4 +102,4 @@ def test_insert_as_query(client): "format": "application/x-www-form-urlencoded", }, ) - assert r.status_code == 400 \ No newline at end of file + assert r.status_code == 400 From bd4e13f580cd6fe747f24d356a30652df26ac734 Mon Sep 17 00:00:00 2001 From: Lawson Lewis Date: Mon, 29 Jul 2024 14:17:44 +1000 Subject: [PATCH 2/4] compute better filenames for returned rdf As per issue https://github.com/idn-au/catalogue-data/issues/35 The returned RDF files did not have appropriate filenames. --- prez/renderers/renderer.py | 50 +++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/prez/renderers/renderer.py b/prez/renderers/renderer.py index 80030425..99562d11 100644 --- a/prez/renderers/renderer.py +++ b/prez/renderers/renderer.py @@ -7,20 +7,18 @@ from fastapi import status from fastapi.exceptions import HTTPException from fastapi.responses import StreamingResponse -from rdflib import Graph, URIRef, Namespace, RDF +from rdflib import RDF, Graph, Namespace, URIRef from starlette.requests import Request from starlette.responses import Response from prez.models.profiles_and_mediatypes import ProfilesMediatypesInfo from prez.models.profiles_item import ProfileItem from prez.renderers.csv_renderer import render_csv_dropdown -from prez.renderers.json_renderer import render_json_dropdown, NotFoundError +from prez.renderers.json_renderer import NotFoundError, render_json_dropdown from prez.services.curie_functions import get_curie_id_for_uri from prez.sparql.methods import Repo -from prez.sparql.objects_listings import ( - generate_item_construct, - get_annotation_properties, -) +from prez.sparql.objects_listings import (generate_item_construct, + get_annotation_properties) log = logging.getLogger(__name__) @@ -33,40 +31,38 @@ async def return_from_graph( selected_class: URIRef, repo: Repo, ): - profile_headers["Content-Disposition"] = "inline" + # set content-disposition + profile_headers["Content-Disposition"] = ( + "attachment;" if str(mediatype) == "text/csv" else "inline;" + ) + iri = graph.value(None, RDF.type, selected_class) + if iri: + profile_headers[ + "Content-Disposition" + ] += f" filename={get_curie_id_for_uri(URIRef(str(iri)))}" + elif selected_class: + profile_headers[ + "Content-Disposition" + ] += f" filename={selected_class.split('#')[-1].split('/')[-1]}" if str(mediatype) in RDF_MEDIATYPES: return await return_rdf(graph, mediatype, profile_headers) - elif profile == URIRef("https://w3id.org/profile/dd"): graph = await return_annotated_rdf(graph, profile, repo) - try: # TODO: Currently, data is generated in memory, instead of in a streaming manner. # Not possible to do a streaming response yet since we are reading the RDF # data into an in-memory graph. jsonld_data = await render_json_dropdown(graph, profile, selected_class) - if str(mediatype) == "text/csv": - iri = graph.value(None, RDF.type, selected_class) - if iri: - filename = get_curie_id_for_uri(URIRef(str(iri))) - else: - filename = selected_class.split("#")[-1].split("/")[-1] stream = render_csv_dropdown(jsonld_data["@graph"]) - response = StreamingResponse(stream, media_type=mediatype) - response.headers[ - "Content-Disposition" - ] = f"attachment;filename={filename}.csv" - return response - - # application/json - stream = io.StringIO(json.dumps(jsonld_data)) - return StreamingResponse(stream, media_type=mediatype) - + else: + stream = io.StringIO(json.dumps(jsonld_data)) + return StreamingResponse( + stream, media_type=mediatype, headers=profile_headers + ) except NotFoundError as err: raise HTTPException(status.HTTP_404_NOT_FOUND, str(err)) - else: if "anot+" in mediatype: non_anot_mediatype = mediatype.replace("anot+", "") @@ -78,7 +74,6 @@ async def return_from_graph( return StreamingResponse( content=content, media_type=non_anot_mediatype, headers=profile_headers ) - raise HTTPException( status.HTTP_400_BAD_REQUEST, f"Unsupported mediatype: {mediatype}." ) @@ -91,7 +86,6 @@ async def return_rdf(graph, mediatype, profile_headers): format=RDF_SERIALIZER_TYPES_MAP[str(mediatype)], encoding="utf-8" ) ) - profile_headers["Content-Disposition"] = "inline" return StreamingResponse(content=obj, media_type=mediatype, headers=profile_headers) From 61c56ae46642fb23551f45aff79dfae01a86e232 Mon Sep 17 00:00:00 2001 From: Lawson Lewis Date: Tue, 30 Jul 2024 13:02:51 +1000 Subject: [PATCH 3/4] add tests for content-disposition and mediatype tests that the returned headers are as expected for object and listing endpoints --- tests/test_pmt_headers.py | 133 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 tests/test_pmt_headers.py diff --git a/tests/test_pmt_headers.py b/tests/test_pmt_headers.py new file mode 100644 index 00000000..6e9c57e7 --- /dev/null +++ b/tests/test_pmt_headers.py @@ -0,0 +1,133 @@ +"""test_pmt_headers + +A set of tests to confirm that the Profile and Media Type information in the response headers are +as expected for object and listing endpoints. + +Also checks the content-disposition header +""" + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from pyoxigraph.pyoxigraph import Store + +from prez.app import assemble_app +from prez.dependencies import get_repo +from prez.services.curie_functions import get_curie_id_for_uri +from prez.sparql.methods import PyoxigraphRepo, Repo + + +@pytest.fixture(scope="session") +def test_store() -> Store: + # Create a new pyoxigraph Store + store = Store() + + for file in Path(__file__).parent.glob("../tests/data/*/input/*.ttl"): + store.load(file.read_bytes(), "text/turtle") + + return store + + +@pytest.fixture(scope="session") +def test_repo(test_store: Store) -> Repo: + # Create a PyoxigraphQuerySender using the test_store + return PyoxigraphRepo(test_store) + + +@pytest.fixture(scope="session") +def test_client(test_repo: Repo) -> TestClient: + # Override the dependency to use the test_repo + def override_get_repo(): + return test_repo + + app = assemble_app() + + app.dependency_overrides[get_repo] = override_get_repo + + with TestClient(app) as c: + yield c + + # Remove the override to ensure subsequent tests are unaffected + app.dependency_overrides.clear() + + +@pytest.mark.parametrize( + "endpoint, mediatype, filename", + [ + ("/v/vocab", "text/turtle", "SchemesList"), + ("/s/datasets", "text/turtle", "DatasetList"), + ("/c/catalogs", "text/turtle", "CatalogList"), + ("/v/vocab", "application/ld+json", "SchemesList"), + ("/s/datasets", "application/ld+json", "DatasetList"), + ("/c/catalogs", "application/ld+json", "CatalogList"), + ], +) +def test_listing_endpoint( + endpoint: str, mediatype: str, filename: str, test_client: TestClient +): + """Assert that response headers are returned correctly for a listing endpoint. + + i.e that they specify the + + - Content-Type, and + - Content-Disposition. + + headers. And that the headers have an appropriate value. + """ + headers = {"accept": mediatype} + expected_headers = { + "content-type": mediatype, + "content-disposition": f"inline; filename={filename}", + } + response = test_client.get(endpoint, headers=headers) + assert all( + header in response.headers.keys() for header in expected_headers.keys() + ), f"Response must specify the {expected_headers.keys()} headers." + assert all( + response.headers[header] == expected_headers[header] + for header in expected_headers.keys() + ), "Required headers do not have the expected values." + + +@pytest.mark.parametrize( + "endpoint, mediatype, object_uri", + [ + ("/v/vocab", "text/turtle", "https://linked.data.gov.au/def/vocdermods"), + ("/s/datasets", "text/turtle", "http://example.com/datasets/sandgate"), + ("/c/catalogs", "text/turtle", "https://data.idnau.org/pid/democat"), + ( + "/v/vocab", + "application/ld+json", + "https://linked.data.gov.au/def/vocdermods", + ), + ("/s/datasets", "application/ld+json", "http://example.com/datasets/sandgate"), + ("/c/catalogs", "application/ld+json", "https://data.idnau.org/pid/democat"), + ], +) +def test_object_endpoint( + endpoint: str, mediatype: str, object_uri: str, test_client: TestClient +): + """Assert that response headers are returned correctly for an object endpoint. + + i.e that they specify the + + - Content-Type, and + - Content-Disposition. + + headers. And that the headers have an appropriate value. + """ + curie = get_curie_id_for_uri(object_uri) + headers = {"accept": mediatype} + expected_headers = { + "content-type": mediatype, + "content-disposition": f"inline; filename={curie}", + } + response = test_client.get(endpoint + "/" + curie, headers=headers) + assert all( + header in response.headers.keys() for header in expected_headers.keys() + ), f"Response must specify the {expected_headers.keys()} headers." + assert all( + response.headers[header] == expected_headers[header] + for header in expected_headers.keys() + ), "Required headers do not have the expected values." From e54253a6ebb0b60adf1044807547c6cee1a7bd3c Mon Sep 17 00:00:00 2001 From: Lawson Lewis Date: Thu, 1 Aug 2024 22:05:38 +1000 Subject: [PATCH 4/4] update pre-commit hooks --- .pre-commit-config.yaml | 23 +++++++++----- poetry.lock | 66 +++++++++++++++++++++++++---------------- pyproject.toml | 1 + 3 files changed, 56 insertions(+), 34 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 81619f4c..98d5073a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,19 +2,26 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.6.0 hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - id: check-yaml - id: check-json - id: check-added-large-files - - repo: https://github.com/psf/black - rev: 22.3.0 - hooks: - - id: black - repo: https://github.com/codespell-project/codespell - rev: v2.1.0 + rev: v2.3.0 hooks: - id: codespell args: [--skip, "*.ttl"] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.5.5 + hooks: + - id: ruff + args: [ --fix ] + - repo: https://github.com/psf/black + rev: 24.4.2 + hooks: + - id: black + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort diff --git a/poetry.lock b/poetry.lock index e7f6349f..3c1af0be 100644 --- a/poetry.lock +++ b/poetry.lock @@ -571,6 +571,20 @@ files = [ [package.dependencies] six = "*" +[[package]] +name = "isort" +version = "5.13.2" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, + {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, +] + +[package.extras] +colors = ["colorama (>=0.4.6)"] + [[package]] name = "jinja2" version = "3.1.4" @@ -875,13 +889,13 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pynvml" -version = "11.5.2" +version = "11.5.3" description = "Python utilities for the NVIDIA Management Library" optional = false python-versions = ">=3.6" files = [ - {file = "pynvml-11.5.2-py3-none-any.whl", hash = "sha256:227ccc843f8fa37ff50c73b272b15c073ce210c0e2da8dd329b86f5522072181"}, - {file = "pynvml-11.5.2.tar.gz", hash = "sha256:45bb3d935b4ac5d7f62022e1a8595718169195b306d65428cf749826d30710cd"}, + {file = "pynvml-11.5.3-py3-none-any.whl", hash = "sha256:a5fba3ab14febda50d19dbda012ef62ae0aed45b7ccc07af0bc5be79223e450c"}, + {file = "pynvml-11.5.3.tar.gz", hash = "sha256:183d223ae487e5f00402d8da06c68c978ef8a9295793ee75559839c6ade7b229"}, ] [[package]] @@ -1130,29 +1144,29 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "ruff" -version = "0.5.4" +version = "0.5.5" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.5.4-py3-none-linux_armv6l.whl", hash = "sha256:82acef724fc639699b4d3177ed5cc14c2a5aacd92edd578a9e846d5b5ec18ddf"}, - {file = "ruff-0.5.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:da62e87637c8838b325e65beee485f71eb36202ce8e3cdbc24b9fcb8b99a37be"}, - {file = "ruff-0.5.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e98ad088edfe2f3b85a925ee96da652028f093d6b9b56b76fc242d8abb8e2059"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c55efbecc3152d614cfe6c2247a3054cfe358cefbf794f8c79c8575456efe19"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f9b85eaa1f653abd0a70603b8b7008d9e00c9fa1bbd0bf40dad3f0c0bdd06793"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cf497a47751be8c883059c4613ba2f50dd06ec672692de2811f039432875278"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:09c14ed6a72af9ccc8d2e313d7acf7037f0faff43cde4b507e66f14e812e37f7"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:628f6b8f97b8bad2490240aa84f3e68f390e13fabc9af5c0d3b96b485921cd60"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3520a00c0563d7a7a7c324ad7e2cde2355733dafa9592c671fb2e9e3cd8194c1"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93789f14ca2244fb91ed481456f6d0bb8af1f75a330e133b67d08f06ad85b516"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:029454e2824eafa25b9df46882f7f7844d36fd8ce51c1b7f6d97e2615a57bbcc"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:9492320eed573a13a0bc09a2957f17aa733fff9ce5bf00e66e6d4a88ec33813f"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a6e1f62a92c645e2919b65c02e79d1f61e78a58eddaebca6c23659e7c7cb4ac7"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:768fa9208df2bec4b2ce61dbc7c2ddd6b1be9fb48f1f8d3b78b3332c7d71c1ff"}, - {file = "ruff-0.5.4-py3-none-win32.whl", hash = "sha256:e1e7393e9c56128e870b233c82ceb42164966f25b30f68acbb24ed69ce9c3a4e"}, - {file = "ruff-0.5.4-py3-none-win_amd64.whl", hash = "sha256:58b54459221fd3f661a7329f177f091eb35cf7a603f01d9eb3eb11cc348d38c4"}, - {file = "ruff-0.5.4-py3-none-win_arm64.whl", hash = "sha256:bd53da65f1085fb5b307c38fd3c0829e76acf7b2a912d8d79cadcdb4875c1eb7"}, - {file = "ruff-0.5.4.tar.gz", hash = "sha256:2795726d5f71c4f4e70653273d1c23a8182f07dd8e48c12de5d867bfb7557eed"}, + {file = "ruff-0.5.5-py3-none-linux_armv6l.whl", hash = "sha256:605d589ec35d1da9213a9d4d7e7a9c761d90bba78fc8790d1c5e65026c1b9eaf"}, + {file = "ruff-0.5.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:00817603822a3e42b80f7c3298c8269e09f889ee94640cd1fc7f9329788d7bf8"}, + {file = "ruff-0.5.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:187a60f555e9f865a2ff2c6984b9afeffa7158ba6e1eab56cb830404c942b0f3"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe26fc46fa8c6e0ae3f47ddccfbb136253c831c3289bba044befe68f467bfb16"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4ad25dd9c5faac95c8e9efb13e15803cd8bbf7f4600645a60ffe17c73f60779b"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f70737c157d7edf749bcb952d13854e8f745cec695a01bdc6e29c29c288fc36e"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:cfd7de17cef6ab559e9f5ab859f0d3296393bc78f69030967ca4d87a541b97a0"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a09b43e02f76ac0145f86a08e045e2ea452066f7ba064fd6b0cdccb486f7c3e7"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d0b856cb19c60cd40198be5d8d4b556228e3dcd545b4f423d1ad812bfdca5884"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3687d002f911e8a5faf977e619a034d159a8373514a587249cc00f211c67a091"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ac9dc814e510436e30d0ba535f435a7f3dc97f895f844f5b3f347ec8c228a523"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:af9bdf6c389b5add40d89b201425b531e0a5cceb3cfdcc69f04d3d531c6be74f"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d40a8533ed545390ef8315b8e25c4bb85739b90bd0f3fe1280a29ae364cc55d8"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:cab904683bf9e2ecbbe9ff235bfe056f0eba754d0168ad5407832928d579e7ab"}, + {file = "ruff-0.5.5-py3-none-win32.whl", hash = "sha256:696f18463b47a94575db635ebb4c178188645636f05e934fdf361b74edf1bb2d"}, + {file = "ruff-0.5.5-py3-none-win_amd64.whl", hash = "sha256:50f36d77f52d4c9c2f1361ccbfbd09099a1b2ea5d2b2222c586ab08885cf3445"}, + {file = "ruff-0.5.5-py3-none-win_arm64.whl", hash = "sha256:3191317d967af701f1b73a31ed5788795936e423b7acce82a2b63e26eb3e89d6"}, + {file = "ruff-0.5.5.tar.gz", hash = "sha256:cc5516bdb4858d972fbc31d246bdb390eab8df1a26e2353be2dbc0c2d7f5421a"}, ] [[package]] @@ -1294,13 +1308,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "uvicorn" -version = "0.30.1" +version = "0.30.4" description = "The lightning-fast ASGI server." optional = false python-versions = ">=3.8" files = [ - {file = "uvicorn-0.30.1-py3-none-any.whl", hash = "sha256:cd17daa7f3b9d7a24de3617820e634d0933b69eed8e33a516071174427238c81"}, - {file = "uvicorn-0.30.1.tar.gz", hash = "sha256:d46cd8e0fd80240baffbcd9ec1012a712938754afcf81bce56c024c1656aece8"}, + {file = "uvicorn-0.30.4-py3-none-any.whl", hash = "sha256:06b00e3087e58c6865c284143c0c42f810b32ff4f265ab19d08c566f74a08728"}, + {file = "uvicorn-0.30.4.tar.gz", hash = "sha256:00db9a9e3711a5fa59866e2b02fac69d8dc70ce0814aaec9a66d1d9e5c832a30"}, ] [package.dependencies] @@ -1566,4 +1580,4 @@ test = ["pytest (>=6.0.0)", "setuptools (>=65)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "86b0f9acfaf67d345911d61953c65ad2b72ee546cdddde304676938c25844b37" +content-hash = "3699ccfc737d5ad2f8433652417a01afad73ec95a7cf0ed075158cda6b4fe5b9" diff --git a/pyproject.toml b/pyproject.toml index 507cf30b..a6c4dff4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ python-dotenv = "^1.0.0" pyoxigraph = "^0.3.19" coverage = "^7.3.2" ruff = "^0.5.4" +isort = "^5.13.2" [build-system] requires = ["poetry-core>=1.9.0"]