From b0b814718fabb3f536ac223392f1043b7034e101 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 8 Aug 2024 10:08:42 -0500 Subject: [PATCH 01/38] add functionality for requester pays buckets --- fence/blueprints/data/indexd.py | 42 ++- fence/blueprints/data/multipart_upload.py | 30 ++- poetry.lock | 299 ++++++++++------------ pyproject.toml | 5 +- 4 files changed, 186 insertions(+), 190 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 380fcd43e..274e7f416 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1,16 +1,17 @@ import re import time import json -from urllib.parse import urlparse, ParseResult, urlunparse +import boto3 +from urllib.parse import urlparse, ParseResult, urlunparse, quote from datetime import datetime, timedelta from sqlalchemy.sql.functions import user from cached_property import cached_property import gen3cirrus from gen3cirrus import GoogleCloudManager +from gen3cirrus import AwsService from cdislogging import get_logger from cdispyutils.config import get_value -from cdispyutils.hmac4 import generate_aws_presigned_url import flask from flask import current_app import requests @@ -1041,6 +1042,8 @@ def get_signed_url( bucket_name = self.bucket_name() bucket = s3_buckets.get(bucket_name) + object_id = self.parsed_url.path.strip("/") + if bucket and bucket.get("endpoint_url"): http_url = bucket["endpoint_url"].strip("/") + "/{}/{}".format( self.parsed_url.netloc, self.parsed_url.path.strip("/") @@ -1073,19 +1076,34 @@ def get_signed_url( self.parsed_url.netloc, credential ) - auth_info = _get_auth_info_for_id_or_from_request(user=authorized_user) - - url = generate_aws_presigned_url( - http_url, - ACTION_DICT["s3"][action], - credential, + client = boto3.client( "s3", - region, - expires_in, - auth_info, + aws_access_key_id=credential["aws_access_key_id"], + aws_secret_access_key=credential["aws_secret_access_key"], ) - return url + cirrus_aws = AwsService(client) + auth_info = _get_auth_info_for_id_or_from_request(user=authorized_user) + + action = ACTION_DICT["s3"][action] + if action == "PUT": # get presigned url for upload + url = cirrus_aws.uploadPresignedURL(bucket_name, object_id, expires_in) + else: # get presigned url for download + print(f"---------- DEBUG ------------------ Bucket config {bucket}") + if bucket.get("requester_pays", False) == True: + url = cirrus_aws.requesterPaysDownloadPresignedURL( + bucket_name, object_id, expires_in + ) + else: + url = cirrus_aws.downloadPresignedURL( + bucket_name, object_id, expires_in + ) + + canonical_qs = "" + for key in sorted(auth_info.keys()): + canonical_qs += "&" + key + "=" + quote(auth_info[key], safe="") + + return url + "?" + canonical_qs def init_multipart_upload(self, expires_in): """ diff --git a/fence/blueprints/data/multipart_upload.py b/fence/blueprints/data/multipart_upload.py index 96c3d6227..eda49c9e2 100644 --- a/fence/blueprints/data/multipart_upload.py +++ b/fence/blueprints/data/multipart_upload.py @@ -2,7 +2,6 @@ from botocore.exceptions import ClientError from retry.api import retry_call -from cdispyutils.hmac4 import generate_aws_presigned_url from cdispyutils.config import get_value from cdislogging import get_logger from fence.config import config @@ -146,18 +145,29 @@ def generate_presigned_url_for_uploading_part( ) bucket = s3_buckets.get(bucket_name) - if bucket.get("endpoint_url"): - url = bucket["endpoint_url"].strip("/") + "/{}/{}".format( - bucket_name, key.strip("/") - ) - else: - url = "https://{}.s3.amazonaws.com/{}".format(bucket_name, key) - additional_signed_qs = {"partNumber": str(partNumber), "uploadId": uploadId} + # if bucket.get("endpoint_url"): + # url = bucket["endpoint_url"].strip("/") + "/{}/{}".format( + # bucket_name, key.strip("/") + # ) + # else: + # url = "https://{}.s3.amazonaws.com/{}".format(bucket_name, key) + # additional_signed_qs = {"partNumber": str(partNumber), "uploadId": uploadId} try: - presigned_url = generate_aws_presigned_url( - url, "PUT", credentials, "s3", region, expires, additional_signed_qs + s3client = boto3.client( + "s3", + aws_access_key_id=credentials["aws_access_key_id"], + aws_secret_access_key=credentials["aws_secret_access_key"], ) + cirrus_aws = AwsService(client) + + presigned_url = cirrus_aws.multipartUploadPresignedURL( + bucket, key, expires, uploadId, partNumber + ) + + # presigned_url = generate_aws_presigned_url( + # url, "PUT", credentials, "s3", region, expires, additional_signed_qs + # ) return presigned_url except Exception as e: raise InternalError( diff --git a/poetry.lock b/poetry.lock index b3ea2a211..bb79b9772 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "addict" @@ -78,22 +78,22 @@ files = [ [[package]] name = "attrs" -version = "23.2.0" +version = "24.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" files = [ - {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, - {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, + {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, + {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, ] [package.extras] -cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[tests]", "pre-commit"] -docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] -tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] -tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] +benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] [[package]] name = "authlib" @@ -250,17 +250,17 @@ files = [ [[package]] name = "boto3" -version = "1.34.147" +version = "1.34.154" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.147-py3-none-any.whl", hash = "sha256:e1cef9a1a301866bcdee32ae0c699465eb2345f9a8e613a5835821430165ff6d"}, - {file = "boto3-1.34.147.tar.gz", hash = "sha256:9ec1c6ab22588242a47549f51a63dfc7c21fdf95a94820fc6e629ab060c38bd9"}, + {file = "boto3-1.34.154-py3-none-any.whl", hash = "sha256:7ca22adef4c77ee128e1e1dc7d48bc9512a87cc6fe3d771b3f913d5ecd41c057"}, + {file = "boto3-1.34.154.tar.gz", hash = "sha256:864f06528c583dc7b02adf12db395ecfadbf9cb0da90e907e848ffb27128ce19"}, ] [package.dependencies] -botocore = ">=1.34.147,<1.35.0" +botocore = ">=1.34.154,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -269,13 +269,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.147" +version = "1.34.154" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.147-py3-none-any.whl", hash = "sha256:be94a2f4874b1d1705cae2bd512c475047497379651678593acb6c61c50d91de"}, - {file = "botocore-1.34.147.tar.gz", hash = "sha256:2e8f000b77e4ca345146cb2edab6403769a517b564f627bb084ab335417f3dbe"}, + {file = "botocore-1.34.154-py3-none-any.whl", hash = "sha256:4eef4b1bb809b382ba9dc9c88f5fcc4a133f221a1acb693ee6bee4de9f325979"}, + {file = "botocore-1.34.154.tar.gz", hash = "sha256:64d9b4c85a504d77cb56dabb2ad717cd8e1717424a88edb458b01d1e5797262a"}, ] [package.dependencies] @@ -351,18 +351,18 @@ files = [ [[package]] name = "cdispyutils" -version = "2.0.1" +version = "2.1.0" description = "This package includes several utility Python tools for the Gen3 stack." optional = false -python-versions = ">=3.6,<4.0" +python-versions = "<4.0,>=3.9" files = [ - {file = "cdispyutils-2.0.1-py3-none-any.whl", hash = "sha256:9a269014c657c87830e00d9b581280bfbe57a8708bbf3e0cf21a141d3810ab06"}, - {file = "cdispyutils-2.0.1.tar.gz", hash = "sha256:b6bfef5b3c77afe1d7705124d021eb579b500f9fcc07a66dc0f8fe8d130e6c23"}, + {file = "cdispyutils-2.1.0-py3-none-any.whl", hash = "sha256:ca1310ebb7e1b971d183823a8294e1dc9d8d55f862aa3c83e0feac7125364308"}, + {file = "cdispyutils-2.1.0.tar.gz", hash = "sha256:1c87830ea1d537f8479364b9473cca037f5ac8906e874471920f69622e3a4431"}, ] [package.dependencies] -cdiserrors = ">=1.0.0,<2.0.0" -cryptography = ">=3.2" +cdiserrors = "*" +cryptography = "*" Flask = "*" PyJWT = "*" requests = "*" @@ -949,14 +949,13 @@ name = "gen3cirrus" version = "3.0.1" description = "" optional = false -python-versions = ">=3.9,<4.0" -files = [ - {file = "gen3cirrus-3.0.1-py3-none-any.whl", hash = "sha256:74628faca3b1cbe65c78e08eb567e1ac0cb8ae52e1bfc603f904af0277e3cb52"}, - {file = "gen3cirrus-3.0.1.tar.gz", hash = "sha256:0ae0ddc0ee7df870603457fe186245f3c8124d989254276e5011a23e1139a6c8"}, -] +python-versions = "^3.9" +files = [] +develop = false [package.dependencies] backoff = "*" +boto3 = "*" cdislogging = "*" google-api-python-client = "*" google-auth = "*" @@ -964,6 +963,12 @@ google-auth-httplib2 = "*" google-cloud-storage = "*" oauth2client = "*" +[package.source] +type = "git" +url = "https://github.com/uc-cdis/cirrus" +reference = "feat/s3-functionality" +resolved_reference = "b648ed107386409e5c91daeda548b0f036689c53" + [[package]] name = "gen3config" version = "1.1.0" @@ -1023,13 +1028,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-api-python-client" -version = "2.137.0" +version = "2.139.0" description = "Google API Client Library for Python" optional = false python-versions = ">=3.7" files = [ - {file = "google_api_python_client-2.137.0-py2.py3-none-any.whl", hash = "sha256:a8b5c5724885e5be9f5368739aa0ccf416627da4ebd914b410a090c18f84d692"}, - {file = "google_api_python_client-2.137.0.tar.gz", hash = "sha256:e739cb74aac8258b1886cb853b0722d47c81fe07ad649d7f2206f06530513c04"}, + {file = "google_api_python_client-2.139.0-py2.py3-none-any.whl", hash = "sha256:1850a92505d91a82e2ca1635ab2b8dff179f4b67082c2651e1db332e8039840c"}, + {file = "google_api_python_client-2.139.0.tar.gz", hash = "sha256:ed4bc3abe2c060a87412465b4e8254620bbbc548eefc5388e2c5ff912d36a68b"}, ] [package.dependencies] @@ -1387,13 +1392,13 @@ files = [ [[package]] name = "importlib-metadata" -version = "8.1.0" +version = "8.2.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_metadata-8.1.0-py3-none-any.whl", hash = "sha256:3cd29f739ed65973840b068e3132135ce954c254d48b5b640484467ef7ab3c8c"}, - {file = "importlib_metadata-8.1.0.tar.gz", hash = "sha256:fcdcb1d5ead7bdf3dd32657bb94ebe9d2aabfe89a19782ddc32da5041d6ebfb4"}, + {file = "importlib_metadata-8.2.0-py3-none-any.whl", hash = "sha256:11901fa0c2f97919b288679932bb64febaeacf289d18ac84dd68cb2e74213369"}, + {file = "importlib_metadata-8.2.0.tar.gz", hash = "sha256:72e8d4399996132204f9a16dcc751af254a48f8d1b20b9ff0f98d4a8f901e73d"}, ] [package.dependencies] @@ -1404,6 +1409,17 @@ doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linke perf = ["ipython"] test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "isodate" version = "0.6.1" @@ -1608,17 +1624,6 @@ six = ">=1.9" docs = ["Pygments (<2)", "jinja2 (<2.7)", "sphinx", "sphinx (<1.3)"] test = ["unittest2 (>=1.1.0)"] -[[package]] -name = "more-itertools" -version = "10.3.0" -description = "More routines for operating on iterables, beyond itertools" -optional = false -python-versions = ">=3.8" -files = [ - {file = "more-itertools-10.3.0.tar.gz", hash = "sha256:e5d93ef411224fbcef366a6e8ddc4c5781bc6359d43412a65dd5964e46111463"}, - {file = "more_itertools-10.3.0-py3-none-any.whl", hash = "sha256:ea6a02e24a9161e51faad17a8782b92a0df82c12c1c8886fec7f0c3fa1a1b320"}, -] - [[package]] name = "moto" version = "1.3.7" @@ -1716,17 +1721,18 @@ files = [ [[package]] name = "pluggy" -version = "0.13.1" +version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.8" files = [ - {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, - {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] [package.extras] dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "prometheus-client" @@ -1761,44 +1767,22 @@ testing = ["google-api-core (>=1.31.5)"] [[package]] name = "protobuf" -version = "5.27.2" +version = "5.27.3" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "protobuf-5.27.2-cp310-abi3-win32.whl", hash = "sha256:354d84fac2b0d76062e9b3221f4abbbacdfd2a4d8af36bab0474f3a0bb30ab38"}, - {file = "protobuf-5.27.2-cp310-abi3-win_amd64.whl", hash = "sha256:0e341109c609749d501986b835f667c6e1e24531096cff9d34ae411595e26505"}, - {file = "protobuf-5.27.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a109916aaac42bff84702fb5187f3edadbc7c97fc2c99c5ff81dd15dcce0d1e5"}, - {file = "protobuf-5.27.2-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:176c12b1f1c880bf7a76d9f7c75822b6a2bc3db2d28baa4d300e8ce4cde7409b"}, - {file = "protobuf-5.27.2-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:b848dbe1d57ed7c191dfc4ea64b8b004a3f9ece4bf4d0d80a367b76df20bf36e"}, - {file = "protobuf-5.27.2-cp38-cp38-win32.whl", hash = "sha256:4fadd8d83e1992eed0248bc50a4a6361dc31bcccc84388c54c86e530b7f58863"}, - {file = "protobuf-5.27.2-cp38-cp38-win_amd64.whl", hash = "sha256:610e700f02469c4a997e58e328cac6f305f649826853813177e6290416e846c6"}, - {file = "protobuf-5.27.2-cp39-cp39-win32.whl", hash = "sha256:9e8f199bf7f97bd7ecebffcae45ebf9527603549b2b562df0fbc6d4d688f14ca"}, - {file = "protobuf-5.27.2-cp39-cp39-win_amd64.whl", hash = "sha256:7fc3add9e6003e026da5fc9e59b131b8f22b428b991ccd53e2af8071687b4fce"}, - {file = "protobuf-5.27.2-py3-none-any.whl", hash = "sha256:54330f07e4949d09614707c48b06d1a22f8ffb5763c159efd5c0928326a91470"}, - {file = "protobuf-5.27.2.tar.gz", hash = "sha256:f3ecdef226b9af856075f28227ff2c90ce3a594d092c39bee5513573f25e2714"}, -] - -[[package]] -name = "psycopg2" -version = "2.9.9" -description = "psycopg2 - Python-PostgreSQL Database Adapter" -optional = false -python-versions = ">=3.7" -files = [ - {file = "psycopg2-2.9.9-cp310-cp310-win32.whl", hash = "sha256:38a8dcc6856f569068b47de286b472b7c473ac7977243593a288ebce0dc89516"}, - {file = "psycopg2-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3"}, - {file = "psycopg2-2.9.9-cp311-cp311-win32.whl", hash = "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372"}, - {file = "psycopg2-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981"}, - {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, - {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, - {file = "psycopg2-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa"}, - {file = "psycopg2-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a"}, - {file = "psycopg2-2.9.9-cp38-cp38-win32.whl", hash = "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c"}, - {file = "psycopg2-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:bac58c024c9922c23550af2a581998624d6e02350f4ae9c5f0bc642c633a2d5e"}, - {file = "psycopg2-2.9.9-cp39-cp39-win32.whl", hash = "sha256:c92811b2d4c9b6ea0285942b2e7cac98a59e166d59c588fe5cfe1eda58e72d59"}, - {file = "psycopg2-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:de80739447af31525feddeb8effd640782cf5998e1a4e9192ebdf829717e3913"}, - {file = "psycopg2-2.9.9.tar.gz", hash = "sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156"}, + {file = "protobuf-5.27.3-cp310-abi3-win32.whl", hash = "sha256:dcb307cd4ef8fec0cf52cb9105a03d06fbb5275ce6d84a6ae33bc6cf84e0a07b"}, + {file = "protobuf-5.27.3-cp310-abi3-win_amd64.whl", hash = "sha256:16ddf3f8c6c41e1e803da7abea17b1793a97ef079a912e42351eabb19b2cffe7"}, + {file = "protobuf-5.27.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:68248c60d53f6168f565a8c76dc58ba4fa2ade31c2d1ebdae6d80f969cdc2d4f"}, + {file = "protobuf-5.27.3-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:b8a994fb3d1c11156e7d1e427186662b64694a62b55936b2b9348f0a7c6625ce"}, + {file = "protobuf-5.27.3-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:a55c48f2a2092d8e213bd143474df33a6ae751b781dd1d1f4d953c128a415b25"}, + {file = "protobuf-5.27.3-cp38-cp38-win32.whl", hash = "sha256:043853dcb55cc262bf2e116215ad43fa0859caab79bb0b2d31b708f128ece035"}, + {file = "protobuf-5.27.3-cp38-cp38-win_amd64.whl", hash = "sha256:c2a105c24f08b1e53d6c7ffe69cb09d0031512f0b72f812dd4005b8112dbe91e"}, + {file = "protobuf-5.27.3-cp39-cp39-win32.whl", hash = "sha256:c84eee2c71ed83704f1afbf1a85c3171eab0fd1ade3b399b3fad0884cbcca8bf"}, + {file = "protobuf-5.27.3-cp39-cp39-win_amd64.whl", hash = "sha256:af7c0b7cfbbb649ad26132e53faa348580f844d9ca46fd3ec7ca48a1ea5db8a1"}, + {file = "protobuf-5.27.3-py3-none-any.whl", hash = "sha256:8572c6533e544ebf6899c360e91d6bcbbee2549251643d32c52cf8a5de295ba5"}, + {file = "protobuf-5.27.3.tar.gz", hash = "sha256:82460903e640f2b7e34ee81a947fdaad89de796d324bcbc38ff5430bcdead82c"}, ] [[package]] @@ -1908,13 +1892,13 @@ files = [ [[package]] name = "pyjwt" -version = "2.8.0" +version = "2.9.0" description = "JSON Web Token implementation in Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, - {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, + {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, + {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, ] [package.dependencies] @@ -1922,8 +1906,8 @@ cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"cryp [package.extras] crypto = ["cryptography (>=3.4.0)"] -dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] -docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] [[package]] @@ -1968,27 +1952,26 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "5.4.3" +version = "6.2.5" description = "pytest: simple powerful testing with Python" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" files = [ - {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"}, - {file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"}, + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, ] [package.dependencies] atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=17.4.0" +attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} -more-itertools = ">=4.0.0" +iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<1.0" -py = ">=1.5.0" -wcwidth = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" [package.extras] -checkqa-mypy = ["mypy (==v0.761)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] [[package]] @@ -2123,7 +2106,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -2268,57 +2250,55 @@ files = [ [[package]] name = "sqlalchemy" -version = "1.4.52" +version = "1.4.53" description = "Database Abstraction Library" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "SQLAlchemy-1.4.52-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:f68016f9a5713684c1507cc37133c28035f29925c75c0df2f9d0f7571e23720a"}, - {file = "SQLAlchemy-1.4.52-cp310-cp310-manylinux1_x86_64.manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_5_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24bb0f81fbbb13d737b7f76d1821ec0b117ce8cbb8ee5e8641ad2de41aa916d3"}, - {file = "SQLAlchemy-1.4.52-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e93983cc0d2edae253b3f2141b0a3fb07e41c76cd79c2ad743fc27eb79c3f6db"}, - {file = "SQLAlchemy-1.4.52-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:84e10772cfc333eb08d0b7ef808cd76e4a9a30a725fb62a0495877a57ee41d81"}, - {file = "SQLAlchemy-1.4.52-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:427988398d2902de042093d17f2b9619a5ebc605bf6372f7d70e29bde6736842"}, - {file = "SQLAlchemy-1.4.52-cp310-cp310-win32.whl", hash = "sha256:1296f2cdd6db09b98ceb3c93025f0da4835303b8ac46c15c2136e27ee4d18d94"}, - {file = "SQLAlchemy-1.4.52-cp310-cp310-win_amd64.whl", hash = "sha256:80e7f697bccc56ac6eac9e2df5c98b47de57e7006d2e46e1a3c17c546254f6ef"}, - {file = "SQLAlchemy-1.4.52-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2f251af4c75a675ea42766880ff430ac33291c8d0057acca79710f9e5a77383d"}, - {file = "SQLAlchemy-1.4.52-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb8f9e4c4718f111d7b530c4e6fb4d28f9f110eb82e7961412955b3875b66de0"}, - {file = "SQLAlchemy-1.4.52-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afb1672b57f58c0318ad2cff80b384e816735ffc7e848d8aa51e0b0fc2f4b7bb"}, - {file = "SQLAlchemy-1.4.52-cp311-cp311-win32.whl", hash = "sha256:6e41cb5cda641f3754568d2ed8962f772a7f2b59403b95c60c89f3e0bd25f15e"}, - {file = "SQLAlchemy-1.4.52-cp311-cp311-win_amd64.whl", hash = "sha256:5bed4f8c3b69779de9d99eb03fd9ab67a850d74ab0243d1be9d4080e77b6af12"}, - {file = "SQLAlchemy-1.4.52-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:49e3772eb3380ac88d35495843daf3c03f094b713e66c7d017e322144a5c6b7c"}, - {file = "SQLAlchemy-1.4.52-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:618827c1a1c243d2540314c6e100aee7af09a709bd005bae971686fab6723554"}, - {file = "SQLAlchemy-1.4.52-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de9acf369aaadb71a725b7e83a5ef40ca3de1cf4cdc93fa847df6b12d3cd924b"}, - {file = "SQLAlchemy-1.4.52-cp312-cp312-win32.whl", hash = "sha256:763bd97c4ebc74136ecf3526b34808c58945023a59927b416acebcd68d1fc126"}, - {file = "SQLAlchemy-1.4.52-cp312-cp312-win_amd64.whl", hash = "sha256:f12aaf94f4d9679ca475975578739e12cc5b461172e04d66f7a3c39dd14ffc64"}, - {file = "SQLAlchemy-1.4.52-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:853fcfd1f54224ea7aabcf34b227d2b64a08cbac116ecf376907968b29b8e763"}, - {file = "SQLAlchemy-1.4.52-cp36-cp36m-manylinux1_x86_64.manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_5_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f98dbb8fcc6d1c03ae8ec735d3c62110949a3b8bc6e215053aa27096857afb45"}, - {file = "SQLAlchemy-1.4.52-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e135fff2e84103bc15c07edd8569612ce317d64bdb391f49ce57124a73f45c5"}, - {file = "SQLAlchemy-1.4.52-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5b5de6af8852500d01398f5047d62ca3431d1e29a331d0b56c3e14cb03f8094c"}, - {file = "SQLAlchemy-1.4.52-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3491c85df263a5c2157c594f54a1a9c72265b75d3777e61ee13c556d9e43ffc9"}, - {file = "SQLAlchemy-1.4.52-cp36-cp36m-win32.whl", hash = "sha256:427c282dd0deba1f07bcbf499cbcc9fe9a626743f5d4989bfdfd3ed3513003dd"}, - {file = "SQLAlchemy-1.4.52-cp36-cp36m-win_amd64.whl", hash = "sha256:ca5ce82b11731492204cff8845c5e8ca1a4bd1ade85e3b8fcf86e7601bfc6a39"}, - {file = "SQLAlchemy-1.4.52-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:29d4247313abb2015f8979137fe65f4eaceead5247d39603cc4b4a610936cd2b"}, - {file = "SQLAlchemy-1.4.52-cp37-cp37m-manylinux1_x86_64.manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_5_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a752bff4796bf22803d052d4841ebc3c55c26fb65551f2c96e90ac7c62be763a"}, - {file = "SQLAlchemy-1.4.52-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7ea11727feb2861deaa293c7971a4df57ef1c90e42cb53f0da40c3468388000"}, - {file = "SQLAlchemy-1.4.52-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d913f8953e098ca931ad7f58797f91deed26b435ec3756478b75c608aa80d139"}, - {file = "SQLAlchemy-1.4.52-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a251146b921725547ea1735b060a11e1be705017b568c9f8067ca61e6ef85f20"}, - {file = "SQLAlchemy-1.4.52-cp37-cp37m-win32.whl", hash = "sha256:1f8e1c6a6b7f8e9407ad9afc0ea41c1f65225ce505b79bc0342159de9c890782"}, - {file = "SQLAlchemy-1.4.52-cp37-cp37m-win_amd64.whl", hash = "sha256:346ed50cb2c30f5d7a03d888e25744154ceac6f0e6e1ab3bc7b5b77138d37710"}, - {file = "SQLAlchemy-1.4.52-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:4dae6001457d4497736e3bc422165f107ecdd70b0d651fab7f731276e8b9e12d"}, - {file = "SQLAlchemy-1.4.52-cp38-cp38-manylinux1_x86_64.manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_5_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5d2e08d79f5bf250afb4a61426b41026e448da446b55e4770c2afdc1e200fce"}, - {file = "SQLAlchemy-1.4.52-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bbce5dd7c7735e01d24f5a60177f3e589078f83c8a29e124a6521b76d825b85"}, - {file = "SQLAlchemy-1.4.52-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:bdb7b4d889631a3b2a81a3347c4c3f031812eb4adeaa3ee4e6b0d028ad1852b5"}, - {file = "SQLAlchemy-1.4.52-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c294ae4e6bbd060dd79e2bd5bba8b6274d08ffd65b58d106394cb6abbf35cf45"}, - {file = "SQLAlchemy-1.4.52-cp38-cp38-win32.whl", hash = "sha256:bcdfb4b47fe04967669874fb1ce782a006756fdbebe7263f6a000e1db969120e"}, - {file = "SQLAlchemy-1.4.52-cp38-cp38-win_amd64.whl", hash = "sha256:7d0dbc56cb6af5088f3658982d3d8c1d6a82691f31f7b0da682c7b98fa914e91"}, - {file = "SQLAlchemy-1.4.52-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:a551d5f3dc63f096ed41775ceec72fdf91462bb95abdc179010dc95a93957800"}, - {file = "SQLAlchemy-1.4.52-cp39-cp39-manylinux1_x86_64.manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_5_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ab773f9ad848118df7a9bbabca53e3f1002387cdbb6ee81693db808b82aaab0"}, - {file = "SQLAlchemy-1.4.52-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2de46f5d5396d5331127cfa71f837cca945f9a2b04f7cb5a01949cf676db7d1"}, - {file = "SQLAlchemy-1.4.52-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7027be7930a90d18a386b25ee8af30514c61f3852c7268899f23fdfbd3107181"}, - {file = "SQLAlchemy-1.4.52-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99224d621affbb3c1a4f72b631f8393045f4ce647dd3262f12fe3576918f8bf3"}, - {file = "SQLAlchemy-1.4.52-cp39-cp39-win32.whl", hash = "sha256:c124912fd4e1bb9d1e7dc193ed482a9f812769cb1e69363ab68e01801e859821"}, - {file = "SQLAlchemy-1.4.52-cp39-cp39-win_amd64.whl", hash = "sha256:2c286fab42e49db23c46ab02479f328b8bdb837d3e281cae546cc4085c83b680"}, - {file = "SQLAlchemy-1.4.52.tar.gz", hash = "sha256:80e63bbdc5217dad3485059bdf6f65a7d43f33c8bde619df5c220edf03d87296"}, + {file = "SQLAlchemy-1.4.53-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:b61ac5457d91b5629a3dea2b258deb4cdd35ac8f6fa2031d2b9b2fff5b3396da"}, + {file = "SQLAlchemy-1.4.53-cp310-cp310-manylinux1_x86_64.manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_5_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a96aa8d425047551676b0e178ddb0683421e78eda879ab55775128b2e612cae"}, + {file = "SQLAlchemy-1.4.53-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e10ac36f0b994235c13388b39598bf27219ec8bdea5be99bdac612b01cbe525"}, + {file = "SQLAlchemy-1.4.53-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:437592b341a3229dd0443c9c803b0bf0a466f8f539014fef6cdb9c06b7edb7f9"}, + {file = "SQLAlchemy-1.4.53-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:784272ceb5eb71421fea9568749bcbe8bd019261a0e2e710a7efa76057af2499"}, + {file = "SQLAlchemy-1.4.53-cp310-cp310-win32.whl", hash = "sha256:122d7b5722df1a24402c6748bbb04687ef981493bb559d0cc0beffe722e0e6ed"}, + {file = "SQLAlchemy-1.4.53-cp310-cp310-win_amd64.whl", hash = "sha256:4604d42b2abccba266d3f5bbe883684b5df93e74054024c70d3fbb5eea45e530"}, + {file = "SQLAlchemy-1.4.53-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:fb8e15dfa47f5de11ab073e12aadd6b502cfb7ac4bafd18bd18cfd1c7d13dbbc"}, + {file = "SQLAlchemy-1.4.53-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc8be4df55e8fde3006d9cb1f6b3df2ba26db613855dc4df2c0fcd5ec15cb3b7"}, + {file = "SQLAlchemy-1.4.53-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86b11640251f9a9789fd96cd6e5d176b1c230230c70ad40299bcbcc568451b4c"}, + {file = "SQLAlchemy-1.4.53-cp311-cp311-win32.whl", hash = "sha256:cd534c716f86bdf95b7b984a34ee278c91d1b1d7d183e7e5ff878600b1696046"}, + {file = "SQLAlchemy-1.4.53-cp311-cp311-win_amd64.whl", hash = "sha256:6dd06572872ca13ef5a90306a3e5af787498ddaa17fb00109b1243642646cd69"}, + {file = "SQLAlchemy-1.4.53-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:2774c24c405136c3ef472e2352bdca7330659d481fbf2283f996c0ef9eb90f22"}, + {file = "SQLAlchemy-1.4.53-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68a614765197b3d13a730d631a78c3bb9b3b72ba58ed7ab295d58d517464e315"}, + {file = "SQLAlchemy-1.4.53-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d13d4dfbc6e52363886b47cf02cf68c5d2a37c468626694dc210d7e97d4ad330"}, + {file = "SQLAlchemy-1.4.53-cp312-cp312-win32.whl", hash = "sha256:197065b91456574d70b6459bfa62bc0b52a4960a29ef923c375ec427274a3e05"}, + {file = "SQLAlchemy-1.4.53-cp312-cp312-win_amd64.whl", hash = "sha256:421306c4b936b0271a3ce2dc074928d5ece4a36f9c482daa5770f44ecfc3a883"}, + {file = "SQLAlchemy-1.4.53-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:13fc34b35d8ddb3fbe3f8fcfdf6c2546e676187f0fb20f5774da362ddaf8fa2d"}, + {file = "SQLAlchemy-1.4.53-cp36-cp36m-manylinux1_x86_64.manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_5_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626be971ff89541cfd3e70b54be00b57a7f8557204decb6223ce0428fec058f3"}, + {file = "SQLAlchemy-1.4.53-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:991e42fdfec561ebc6a4fae7161a86d129d6069fa14210b96b8dd752afa7059c"}, + {file = "SQLAlchemy-1.4.53-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:95123f3a1e0e8020848fd32ba751db889a01a44e4e4fef7e58c87ddd0b2fca59"}, + {file = "SQLAlchemy-1.4.53-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c58e011e9e6373b3a091d83f20601fb335a3b4bace80bfcb914ac168aad3b70d"}, + {file = "SQLAlchemy-1.4.53-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:670c7769bf5dcae9aff331247b5d82fe635c63731088a46ce68ba2ba519ef36e"}, + {file = "SQLAlchemy-1.4.53-cp37-cp37m-manylinux1_x86_64.manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_5_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07ba54f09033d387ae9df8d62cbe211ed7304e0bfbece1f8c55e21db9fae5c11"}, + {file = "SQLAlchemy-1.4.53-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a38834b4c183c33daf58544281395aad2e985f0b47cca1e88ea5ada88344e63"}, + {file = "SQLAlchemy-1.4.53-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:616492f5315128a847f293a7c552f3561ac7e996d2aa5dc46bef4fb0d3781f1d"}, + {file = "SQLAlchemy-1.4.53-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0cf8c0af9563892c6632f7343bc393dfce6eeef8e4d10c5fadba9c0390520bd"}, + {file = "SQLAlchemy-1.4.53-cp37-cp37m-win32.whl", hash = "sha256:c05fe05941424c2f3747a8952381b7725e24cba2ca00141380e54789d5b616b6"}, + {file = "SQLAlchemy-1.4.53-cp37-cp37m-win_amd64.whl", hash = "sha256:93e90aa3e3b2f8e8cbae4d5509f8e0cf82972378d323c740a8df1c1e9f484172"}, + {file = "SQLAlchemy-1.4.53-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:9d7368df54d3ed45a18955f6cec38ebe075290594ac0d5c87a8ddaff7e10de27"}, + {file = "SQLAlchemy-1.4.53-cp38-cp38-manylinux1_x86_64.manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_5_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89d8ac4158ef68eea8bb0f6dd0583127d9aa8720606964ba8eee20b254f9c83a"}, + {file = "SQLAlchemy-1.4.53-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16bb9fa4d00b4581b14d9f0e2224dc7745b854aa4687738279af0f48f7056c98"}, + {file = "SQLAlchemy-1.4.53-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4fe5168d0249c23f537950b6d75935ff2709365a113e29938a979aec36668ecf"}, + {file = "SQLAlchemy-1.4.53-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b8608d162d3bd29d807aab32c3fb6e2f8e225a43d1c54c917fed38513785380"}, + {file = "SQLAlchemy-1.4.53-cp38-cp38-win32.whl", hash = "sha256:a9d4d132198844bd6828047135ce7b887687c92925049a2468a605fc775c7a1a"}, + {file = "SQLAlchemy-1.4.53-cp38-cp38-win_amd64.whl", hash = "sha256:c15d1f1fcf1f9bec0499ae1d9132b950fcc7730f2d26d10484c8808b4e077816"}, + {file = "SQLAlchemy-1.4.53-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:edf094a20a386ff2ec73de65ef18014b250259cb860edc61741e240ca22d6981"}, + {file = "SQLAlchemy-1.4.53-cp39-cp39-manylinux1_x86_64.manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_5_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83a9c3514ff19d9d30d8a8d378b24cd1dfa5528d20891481cb5f196117db6a48"}, + {file = "SQLAlchemy-1.4.53-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eaaeedbceb4dfd688fff2faf25a9a87a391f548811494f7bff7fa701b639abc3"}, + {file = "SQLAlchemy-1.4.53-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d021699b9007deb7aa715629078830c99a5fec2753d9bdd5ff33290d363ef755"}, + {file = "SQLAlchemy-1.4.53-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0465b8a68f8f4de754c1966c45b187ac784ad97bc9747736f913130f0e1adea0"}, + {file = "SQLAlchemy-1.4.53-cp39-cp39-win32.whl", hash = "sha256:5f67b9e9dcac3241781e96575468d55a42332157dee04bdbf781df573dff5f85"}, + {file = "SQLAlchemy-1.4.53-cp39-cp39-win_amd64.whl", hash = "sha256:a8c2f2a0b2c4e3b86eb58c9b6bb98548205eea2fba9dae4edfd29dc6aebbe95a"}, + {file = "SQLAlchemy-1.4.53.tar.gz", hash = "sha256:5e6ab710c4c064755fd92d1a417bef360228a19bdf0eee32b03aa0f5f8e9fe0d"}, ] [package.dependencies] @@ -2329,17 +2309,17 @@ aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] -mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)", "mariadb (>=1.0.1,!=1.1.2)"] mssql = ["pyodbc"] -mssql-pymssql = ["pymssql"] -mssql-pyodbc = ["pyodbc"] +mssql-pymssql = ["pymssql", "pymssql"] +mssql-pyodbc = ["pyodbc", "pyodbc"] mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] -mysql-connector = ["mysql-connector-python"] +mysql-connector = ["mysql-connector-python", "mysql-connector-python"] oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"] postgresql = ["psycopg2 (>=2.7)"] -postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] -postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] +postgresql-asyncpg = ["asyncpg", "asyncpg", "greenlet (!=0.4.17)", "greenlet (!=0.4.17)"] +postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)", "pg8000 (>=1.16.6,!=1.29.0)"] postgresql-psycopg2binary = ["psycopg2-binary"] postgresql-psycopg2cffi = ["psycopg2cffi"] pymysql = ["pymysql", "pymysql (<1)"] @@ -2425,17 +2405,6 @@ files = [ cdislogging = "*" sqlalchemy = ">=1.3.3" -[[package]] -name = "wcwidth" -version = "0.2.13" -description = "Measures the displayed width of unicode strings in a terminal" -optional = false -python-versions = "*" -files = [ - {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, - {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, -] - [[package]] name = "werkzeug" version = "3.0.3" @@ -2578,4 +2547,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0.0" -content-hash = "d003418dcc0d68257a215186d21776941f8739bd3b4f898762a93e7895d7c89e" +content-hash = "325bef0fbcf6d82845a31d349ac9b40b908143db6876b92ec3d1361a7d041261" diff --git a/pyproject.toml b/pyproject.toml index 143ca2940..ded3e5da8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ flask-cors = ">=3.0.3" flask-restful = ">=0.3.8" email_validator = "^1.1.1" gen3authz = "^1.5.1" -gen3cirrus = ">=3.0.1" +gen3cirrus = {git = "https://github.com/uc-cdis/cirrus", rev = "feat/s3-functionality"} gen3config = ">=1.1.0" gen3users = "^1.0.2" idna = "^3.7" @@ -39,7 +39,6 @@ markupsafe = "^2.0.1" paramiko = ">=2.6.0" prometheus-client = "<1" -psycopg2 = "^2.8.3" PyJWT = "^2.4.0" python_dateutil = "^2.6.1" python-jose = "^2.0.2" @@ -64,7 +63,7 @@ codacy-coverage = "^1.3.11" coveralls = "^2.1.1" mock = "^2.0.0" moto = "^1.1.24" -pytest = "^5.2.0" +pytest = "^6.2.5" pytest-cov = "^2.5.1" pytest-flask = ">=1.3.0" From 9dbf6f185b8be4e6924e86fb1f81905c11291cfd Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 8 Aug 2024 10:30:25 -0500 Subject: [PATCH 02/38] add removed dependency --- fence/blueprints/data/indexd.py | 2 +- poetry.lock | 298 ++++++++++++++++++-------------- pyproject.toml | 1 + 3 files changed, 171 insertions(+), 130 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 274e7f416..bdc338cd1 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1090,7 +1090,7 @@ def get_signed_url( url = cirrus_aws.uploadPresignedURL(bucket_name, object_id, expires_in) else: # get presigned url for download print(f"---------- DEBUG ------------------ Bucket config {bucket}") - if bucket.get("requester_pays", False) == True: + if bucket.get("requester_pays") == True: url = cirrus_aws.requesterPaysDownloadPresignedURL( bucket_name, object_id, expires_in ) diff --git a/poetry.lock b/poetry.lock index bb79b9772..1df701742 100644 --- a/poetry.lock +++ b/poetry.lock @@ -170,13 +170,13 @@ aio = ["aiohttp (>=3.0)"] [[package]] name = "azure-storage-blob" -version = "12.21.0" +version = "12.22.0" description = "Microsoft Azure Blob Storage Client Library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "azure-storage-blob-12.21.0.tar.gz", hash = "sha256:b9722725072f5b7373c0f4dd6d78fbae2bb37bffc5c3e01731ab8c750ee8dd7e"}, - {file = "azure_storage_blob-12.21.0-py3-none-any.whl", hash = "sha256:f9ede187dd5a0ef296b583a7c1861c6938ddd6708d6e70f4203a163c2ab42d43"}, + {file = "azure-storage-blob-12.22.0.tar.gz", hash = "sha256:b3804bb4fe8ab1c32771fa464053da772a682c2737b19da438a3f4e5e3b3736e"}, + {file = "azure_storage_blob-12.22.0-py3-none-any.whl", hash = "sha256:bb7d2d824ce3f11f14a27ee7d9281289f7e072ac8311c52e3652672455b7d5e8"}, ] [package.dependencies] @@ -250,17 +250,17 @@ files = [ [[package]] name = "boto3" -version = "1.34.154" +version = "1.34.156" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.154-py3-none-any.whl", hash = "sha256:7ca22adef4c77ee128e1e1dc7d48bc9512a87cc6fe3d771b3f913d5ecd41c057"}, - {file = "boto3-1.34.154.tar.gz", hash = "sha256:864f06528c583dc7b02adf12db395ecfadbf9cb0da90e907e848ffb27128ce19"}, + {file = "boto3-1.34.156-py3-none-any.whl", hash = "sha256:cbbd453270b8ce94ef9da60dfbb6f9ceeb3eeee226b635aa9ec44b1def98cc96"}, + {file = "boto3-1.34.156.tar.gz", hash = "sha256:b33e9a8f8be80d3053b8418836a7c1900410b23a30c7cb040927d601a1082e68"}, ] [package.dependencies] -botocore = ">=1.34.154,<1.35.0" +botocore = ">=1.34.156,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -269,13 +269,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.154" +version = "1.34.156" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.154-py3-none-any.whl", hash = "sha256:4eef4b1bb809b382ba9dc9c88f5fcc4a133f221a1acb693ee6bee4de9f325979"}, - {file = "botocore-1.34.154.tar.gz", hash = "sha256:64d9b4c85a504d77cb56dabb2ad717cd8e1717424a88edb458b01d1e5797262a"}, + {file = "botocore-1.34.156-py3-none-any.whl", hash = "sha256:c48f8c8996216dfdeeb0aa6d3c0f2c7ae25234766434a2ea3e57bdc08494bdda"}, + {file = "botocore-1.34.156.tar.gz", hash = "sha256:5d1478c41ab9681e660b3322432fe09c4055759c317984b7b8d3af9557ff769a"}, ] [package.dependencies] @@ -287,7 +287,7 @@ urllib3 = [ ] [package.extras] -crt = ["awscrt (==0.20.11)"] +crt = ["awscrt (==0.21.2)"] [[package]] name = "cached-property" @@ -395,63 +395,78 @@ files = [ [[package]] name = "cffi" -version = "1.16.0" +version = "1.17.0" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.8" files = [ - {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, - {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, - {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, - {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, - {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, - {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, - {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, - {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, - {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, - {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, - {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, - {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, - {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, - {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, - {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, - {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, - {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, - {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, - {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, - {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, - {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, - {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, - {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, - {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, - {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, - {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, - {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, + {file = "cffi-1.17.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f9338cc05451f1942d0d8203ec2c346c830f8e86469903d5126c1f0a13a2bcbb"}, + {file = "cffi-1.17.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a0ce71725cacc9ebf839630772b07eeec220cbb5f03be1399e0457a1464f8e1a"}, + {file = "cffi-1.17.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c815270206f983309915a6844fe994b2fa47e5d05c4c4cef267c3b30e34dbe42"}, + {file = "cffi-1.17.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6bdcd415ba87846fd317bee0774e412e8792832e7805938987e4ede1d13046d"}, + {file = "cffi-1.17.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8a98748ed1a1df4ee1d6f927e151ed6c1a09d5ec21684de879c7ea6aa96f58f2"}, + {file = "cffi-1.17.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a048d4f6630113e54bb4b77e315e1ba32a5a31512c31a273807d0027a7e69ab"}, + {file = "cffi-1.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24aa705a5f5bd3a8bcfa4d123f03413de5d86e497435693b638cbffb7d5d8a1b"}, + {file = "cffi-1.17.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:856bf0924d24e7f93b8aee12a3a1095c34085600aa805693fb7f5d1962393206"}, + {file = "cffi-1.17.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:4304d4416ff032ed50ad6bb87416d802e67139e31c0bde4628f36a47a3164bfa"}, + {file = "cffi-1.17.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:331ad15c39c9fe9186ceaf87203a9ecf5ae0ba2538c9e898e3a6967e8ad3db6f"}, + {file = "cffi-1.17.0-cp310-cp310-win32.whl", hash = "sha256:669b29a9eca6146465cc574659058ed949748f0809a2582d1f1a324eb91054dc"}, + {file = "cffi-1.17.0-cp310-cp310-win_amd64.whl", hash = "sha256:48b389b1fd5144603d61d752afd7167dfd205973a43151ae5045b35793232aa2"}, + {file = "cffi-1.17.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c5d97162c196ce54af6700949ddf9409e9833ef1003b4741c2b39ef46f1d9720"}, + {file = "cffi-1.17.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5ba5c243f4004c750836f81606a9fcb7841f8874ad8f3bf204ff5e56332b72b9"}, + {file = "cffi-1.17.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bb9333f58fc3a2296fb1d54576138d4cf5d496a2cc118422bd77835e6ae0b9cb"}, + {file = "cffi-1.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:435a22d00ec7d7ea533db494da8581b05977f9c37338c80bc86314bec2619424"}, + {file = "cffi-1.17.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1df34588123fcc88c872f5acb6f74ae59e9d182a2707097f9e28275ec26a12d"}, + {file = "cffi-1.17.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df8bb0010fdd0a743b7542589223a2816bdde4d94bb5ad67884348fa2c1c67e8"}, + {file = "cffi-1.17.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8b5b9712783415695663bd463990e2f00c6750562e6ad1d28e072a611c5f2a6"}, + {file = "cffi-1.17.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ffef8fd58a36fb5f1196919638f73dd3ae0db1a878982b27a9a5a176ede4ba91"}, + {file = "cffi-1.17.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e67d26532bfd8b7f7c05d5a766d6f437b362c1bf203a3a5ce3593a645e870b8"}, + {file = "cffi-1.17.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:45f7cd36186db767d803b1473b3c659d57a23b5fa491ad83c6d40f2af58e4dbb"}, + {file = "cffi-1.17.0-cp311-cp311-win32.whl", hash = "sha256:a9015f5b8af1bb6837a3fcb0cdf3b874fe3385ff6274e8b7925d81ccaec3c5c9"}, + {file = "cffi-1.17.0-cp311-cp311-win_amd64.whl", hash = "sha256:b50aaac7d05c2c26dfd50c3321199f019ba76bb650e346a6ef3616306eed67b0"}, + {file = "cffi-1.17.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aec510255ce690d240f7cb23d7114f6b351c733a74c279a84def763660a2c3bc"}, + {file = "cffi-1.17.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2770bb0d5e3cc0e31e7318db06efcbcdb7b31bcb1a70086d3177692a02256f59"}, + {file = "cffi-1.17.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:db9a30ec064129d605d0f1aedc93e00894b9334ec74ba9c6bdd08147434b33eb"}, + {file = "cffi-1.17.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a47eef975d2b8b721775a0fa286f50eab535b9d56c70a6e62842134cf7841195"}, + {file = "cffi-1.17.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f3e0992f23bbb0be00a921eae5363329253c3b86287db27092461c887b791e5e"}, + {file = "cffi-1.17.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6107e445faf057c118d5050560695e46d272e5301feffda3c41849641222a828"}, + {file = "cffi-1.17.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb862356ee9391dc5a0b3cbc00f416b48c1b9a52d252d898e5b7696a5f9fe150"}, + {file = "cffi-1.17.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c1c13185b90bbd3f8b5963cd8ce7ad4ff441924c31e23c975cb150e27c2bf67a"}, + {file = "cffi-1.17.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17c6d6d3260c7f2d94f657e6872591fe8733872a86ed1345bda872cfc8c74885"}, + {file = "cffi-1.17.0-cp312-cp312-win32.whl", hash = "sha256:c3b8bd3133cd50f6b637bb4322822c94c5ce4bf0d724ed5ae70afce62187c492"}, + {file = "cffi-1.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:dca802c8db0720ce1c49cce1149ff7b06e91ba15fa84b1d59144fef1a1bc7ac2"}, + {file = "cffi-1.17.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6ce01337d23884b21c03869d2f68c5523d43174d4fc405490eb0091057943118"}, + {file = "cffi-1.17.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cab2eba3830bf4f6d91e2d6718e0e1c14a2f5ad1af68a89d24ace0c6b17cced7"}, + {file = "cffi-1.17.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:14b9cbc8f7ac98a739558eb86fabc283d4d564dafed50216e7f7ee62d0d25377"}, + {file = "cffi-1.17.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b00e7bcd71caa0282cbe3c90966f738e2db91e64092a877c3ff7f19a1628fdcb"}, + {file = "cffi-1.17.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:41f4915e09218744d8bae14759f983e466ab69b178de38066f7579892ff2a555"}, + {file = "cffi-1.17.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4760a68cab57bfaa628938e9c2971137e05ce48e762a9cb53b76c9b569f1204"}, + {file = "cffi-1.17.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:011aff3524d578a9412c8b3cfaa50f2c0bd78e03eb7af7aa5e0df59b158efb2f"}, + {file = "cffi-1.17.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:a003ac9edc22d99ae1286b0875c460351f4e101f8c9d9d2576e78d7e048f64e0"}, + {file = "cffi-1.17.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ef9528915df81b8f4c7612b19b8628214c65c9b7f74db2e34a646a0a2a0da2d4"}, + {file = "cffi-1.17.0-cp313-cp313-win32.whl", hash = "sha256:70d2aa9fb00cf52034feac4b913181a6e10356019b18ef89bc7c12a283bf5f5a"}, + {file = "cffi-1.17.0-cp313-cp313-win_amd64.whl", hash = "sha256:b7b6ea9e36d32582cda3465f54c4b454f62f23cb083ebc7a94e2ca6ef011c3a7"}, + {file = "cffi-1.17.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:964823b2fc77b55355999ade496c54dde161c621cb1f6eac61dc30ed1b63cd4c"}, + {file = "cffi-1.17.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:516a405f174fd3b88829eabfe4bb296ac602d6a0f68e0d64d5ac9456194a5b7e"}, + {file = "cffi-1.17.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dec6b307ce928e8e112a6bb9921a1cb00a0e14979bf28b98e084a4b8a742bd9b"}, + {file = "cffi-1.17.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e4094c7b464cf0a858e75cd14b03509e84789abf7b79f8537e6a72152109c76e"}, + {file = "cffi-1.17.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2404f3de742f47cb62d023f0ba7c5a916c9c653d5b368cc966382ae4e57da401"}, + {file = "cffi-1.17.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aa9d43b02a0c681f0bfbc12d476d47b2b2b6a3f9287f11ee42989a268a1833c"}, + {file = "cffi-1.17.0-cp38-cp38-win32.whl", hash = "sha256:0bb15e7acf8ab35ca8b24b90af52c8b391690ef5c4aec3d31f38f0d37d2cc499"}, + {file = "cffi-1.17.0-cp38-cp38-win_amd64.whl", hash = "sha256:93a7350f6706b31f457c1457d3a3259ff9071a66f312ae64dc024f049055f72c"}, + {file = "cffi-1.17.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1a2ddbac59dc3716bc79f27906c010406155031a1c801410f1bafff17ea304d2"}, + {file = "cffi-1.17.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6327b572f5770293fc062a7ec04160e89741e8552bf1c358d1a23eba68166759"}, + {file = "cffi-1.17.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbc183e7bef690c9abe5ea67b7b60fdbca81aa8da43468287dae7b5c046107d4"}, + {file = "cffi-1.17.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bdc0f1f610d067c70aa3737ed06e2726fd9d6f7bfee4a351f4c40b6831f4e82"}, + {file = "cffi-1.17.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6d872186c1617d143969defeadac5a904e6e374183e07977eedef9c07c8953bf"}, + {file = "cffi-1.17.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0d46ee4764b88b91f16661a8befc6bfb24806d885e27436fdc292ed7e6f6d058"}, + {file = "cffi-1.17.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f76a90c345796c01d85e6332e81cab6d70de83b829cf1d9762d0a3da59c7932"}, + {file = "cffi-1.17.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0e60821d312f99d3e1569202518dddf10ae547e799d75aef3bca3a2d9e8ee693"}, + {file = "cffi-1.17.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:eb09b82377233b902d4c3fbeeb7ad731cdab579c6c6fda1f763cd779139e47c3"}, + {file = "cffi-1.17.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:24658baf6224d8f280e827f0a50c46ad819ec8ba380a42448e24459daf809cf4"}, + {file = "cffi-1.17.0-cp39-cp39-win32.whl", hash = "sha256:0fdacad9e0d9fc23e519efd5ea24a70348305e8d7d85ecbb1a5fa66dc834e7fb"}, + {file = "cffi-1.17.0-cp39-cp39-win_amd64.whl", hash = "sha256:7cbc78dc018596315d4e7841c8c3a7ae31cc4d638c9b627f87d52e8abaaf2d29"}, + {file = "cffi-1.17.0.tar.gz", hash = "sha256:f3157624b7558b914cb039fd1af735e5e8049a87c817cc215109ad1c8779df76"}, ] [package.dependencies] @@ -1028,13 +1043,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-api-python-client" -version = "2.139.0" +version = "2.140.0" description = "Google API Client Library for Python" optional = false python-versions = ">=3.7" files = [ - {file = "google_api_python_client-2.139.0-py2.py3-none-any.whl", hash = "sha256:1850a92505d91a82e2ca1635ab2b8dff179f4b67082c2651e1db332e8039840c"}, - {file = "google_api_python_client-2.139.0.tar.gz", hash = "sha256:ed4bc3abe2c060a87412465b4e8254620bbbc548eefc5388e2c5ff912d36a68b"}, + {file = "google_api_python_client-2.140.0-py2.py3-none-any.whl", hash = "sha256:aeb4bb99e9fdd241473da5ff35464a0658fea0db76fe89c0f8c77ecfc3813404"}, + {file = "google_api_python_client-2.140.0.tar.gz", hash = "sha256:0bb973adccbe66a3d0a70abe4e49b3f2f004d849416bfec38d22b75649d389d8"}, ] [package.dependencies] @@ -1046,13 +1061,13 @@ uritemplate = ">=3.0.1,<5" [[package]] name = "google-auth" -version = "2.32.0" +version = "2.33.0" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google_auth-2.32.0-py2.py3-none-any.whl", hash = "sha256:53326ea2ebec768070a94bee4e1b9194c9646ea0c2bd72422785bd0f9abfad7b"}, - {file = "google_auth-2.32.0.tar.gz", hash = "sha256:49315be72c55a6a37d62819e3573f6b416aca00721f7e3e31a008d928bf64022"}, + {file = "google_auth-2.33.0-py2.py3-none-any.whl", hash = "sha256:8eff47d0d4a34ab6265c50a106a3362de6a9975bb08998700e389f857e4d39df"}, + {file = "google_auth-2.33.0.tar.gz", hash = "sha256:d6a52342160d7290e334b4d47ba390767e4438ad0d45b7630774533e82655b95"}, ] [package.dependencies] @@ -1102,13 +1117,13 @@ grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] [[package]] name = "google-cloud-storage" -version = "2.18.0" +version = "2.18.1" description = "Google Cloud Storage API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google_cloud_storage-2.18.0-py2.py3-none-any.whl", hash = "sha256:e8e1a9577952143c3fca8163005ecfadd2d70ec080fa158a8b305000e2c22fbb"}, - {file = "google_cloud_storage-2.18.0.tar.gz", hash = "sha256:0aa3f7c57f3632f81b455d91558d2b27ada96eee2de3aaa17f689db1470d9578"}, + {file = "google_cloud_storage-2.18.1-py2.py3-none-any.whl", hash = "sha256:9d8db6bde3a979cca7150511cd0e4cb363e5f69d31259d890ba1124fa109418c"}, + {file = "google_cloud_storage-2.18.1.tar.gz", hash = "sha256:6707a6f30a05aee36faca81296419ca2907ac750af1c0457f278bc9a6fb219ad"}, ] [package.dependencies] @@ -1205,13 +1220,13 @@ testing = ["pytest"] [[package]] name = "google-resumable-media" -version = "2.7.1" +version = "2.7.2" description = "Utilities for Google Media Downloads and Resumable Uploads" optional = false python-versions = ">=3.7" files = [ - {file = "google-resumable-media-2.7.1.tar.gz", hash = "sha256:eae451a7b2e2cdbaaa0fd2eb00cc8a1ee5e95e16b55597359cbc3d27d7d90e33"}, - {file = "google_resumable_media-2.7.1-py2.py3-none-any.whl", hash = "sha256:103ebc4ba331ab1bfdac0250f8033627a2cd7cde09e7ccff9181e31ba4315b2c"}, + {file = "google_resumable_media-2.7.2-py2.py3-none-any.whl", hash = "sha256:3ce7551e9fe6d99e9a126101d2536612bb73486721951e9562fee0f90c6ababa"}, + {file = "google_resumable_media-2.7.2.tar.gz", hash = "sha256:5280aed4629f2b60b847b0d42f9857fd4935c11af266744df33d8074cae92fe0"}, ] [package.dependencies] @@ -1785,6 +1800,28 @@ files = [ {file = "protobuf-5.27.3.tar.gz", hash = "sha256:82460903e640f2b7e34ee81a947fdaad89de796d324bcbc38ff5430bcdead82c"}, ] +[[package]] +name = "psycopg2" +version = "2.9.9" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "psycopg2-2.9.9-cp310-cp310-win32.whl", hash = "sha256:38a8dcc6856f569068b47de286b472b7c473ac7977243593a288ebce0dc89516"}, + {file = "psycopg2-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3"}, + {file = "psycopg2-2.9.9-cp311-cp311-win32.whl", hash = "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372"}, + {file = "psycopg2-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981"}, + {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, + {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, + {file = "psycopg2-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa"}, + {file = "psycopg2-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a"}, + {file = "psycopg2-2.9.9-cp38-cp38-win32.whl", hash = "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c"}, + {file = "psycopg2-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:bac58c024c9922c23550af2a581998624d6e02350f4ae9c5f0bc642c633a2d5e"}, + {file = "psycopg2-2.9.9-cp39-cp39-win32.whl", hash = "sha256:c92811b2d4c9b6ea0285942b2e7cac98a59e166d59c588fe5cfe1eda58e72d59"}, + {file = "psycopg2-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:de80739447af31525feddeb8effd640782cf5998e1a4e9192ebdf829717e3913"}, + {file = "psycopg2-2.9.9.tar.gz", hash = "sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156"}, +] + [[package]] name = "py" version = "1.11.0" @@ -2083,61 +2120,64 @@ files = [ [[package]] name = "pyyaml" -version = "6.0.1" +version = "6.0.2" description = "YAML parser and emitter for Python" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, - {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, - {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, - {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, - {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, - {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, - {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, - {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, - {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, - {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, - {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, - {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, - {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, - {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] [[package]] @@ -2547,4 +2587,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0.0" -content-hash = "325bef0fbcf6d82845a31d349ac9b40b908143db6876b92ec3d1361a7d041261" +content-hash = "597d2a7ac8980713561f7d16c82f045dcb4f8997da0cfba73d2299d791c9b58e" diff --git a/pyproject.toml b/pyproject.toml index ded3e5da8..c6451c0e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ markupsafe = "^2.0.1" paramiko = ">=2.6.0" prometheus-client = "<1" +psycopg2 = "*" PyJWT = "^2.4.0" python_dateutil = "^2.6.1" python-jose = "^2.0.2" From dd4a111cd4f30f4e38bcbc617e75125afcad3b0c Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 8 Aug 2024 12:07:11 -0500 Subject: [PATCH 03/38] add debug for requester pays --- fence/blueprints/data/indexd.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index bdc338cd1..da175d233 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1091,6 +1091,9 @@ def get_signed_url( else: # get presigned url for download print(f"---------- DEBUG ------------------ Bucket config {bucket}") if bucket.get("requester_pays") == True: + print( + f"---------- DEBUG ------------------ WE are generateing a presigned url for a requester pays bucket!!" + ) url = cirrus_aws.requesterPaysDownloadPresignedURL( bucket_name, object_id, expires_in ) From fa84b161d3e9a459006259ddbe053fb260e32d73 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 8 Aug 2024 17:52:01 -0500 Subject: [PATCH 04/38] remove debug statements and fix url concat --- fence/blueprints/data/indexd.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index da175d233..71b3fa285 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1086,14 +1086,11 @@ def get_signed_url( auth_info = _get_auth_info_for_id_or_from_request(user=authorized_user) action = ACTION_DICT["s3"][action] + if action == "PUT": # get presigned url for upload url = cirrus_aws.uploadPresignedURL(bucket_name, object_id, expires_in) else: # get presigned url for download - print(f"---------- DEBUG ------------------ Bucket config {bucket}") if bucket.get("requester_pays") == True: - print( - f"---------- DEBUG ------------------ WE are generateing a presigned url for a requester pays bucket!!" - ) url = cirrus_aws.requesterPaysDownloadPresignedURL( bucket_name, object_id, expires_in ) @@ -1106,7 +1103,7 @@ def get_signed_url( for key in sorted(auth_info.keys()): canonical_qs += "&" + key + "=" + quote(auth_info[key], safe="") - return url + "?" + canonical_qs + return url + canonical_qs def init_multipart_upload(self, expires_in): """ From 0532e4e0a23ffddcaf3a68f7d61d3576afbe2593 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 13 Aug 2024 11:49:05 -0500 Subject: [PATCH 05/38] fix presigned url tests and clean up --- .secrets.baseline | 6 ++--- fence/blueprints/data/multipart_upload.py | 11 --------- tests/conftest.py | 28 +++++++++++++++++++++++ tests/data/test_data.py | 20 ++++++++++++++++ 4 files changed, 51 insertions(+), 14 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 9aa531c19..b70519cdc 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -268,14 +268,14 @@ "filename": "tests/conftest.py", "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9", "is_verified": false, - "line_number": 1569 + "line_number": 1573 }, { "type": "Base64 High Entropy String", "filename": "tests/conftest.py", "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108", "is_verified": false, - "line_number": 1593 + "line_number": 1597 } ], "tests/credentials/google/test_credentials.py": [ @@ -422,5 +422,5 @@ } ] }, - "generated_at": "2024-07-25T17:19:58Z" + "generated_at": "2024-08-13T16:48:45Z" } diff --git a/fence/blueprints/data/multipart_upload.py b/fence/blueprints/data/multipart_upload.py index 5a56f482b..b8112f0c1 100644 --- a/fence/blueprints/data/multipart_upload.py +++ b/fence/blueprints/data/multipart_upload.py @@ -149,14 +149,6 @@ def generate_presigned_url_for_uploading_part( ) bucket = s3_buckets.get(bucket_name) - # if bucket.get("endpoint_url"): - # url = bucket["endpoint_url"].strip("/") + "/{}/{}".format( - # bucket_name, key.strip("/") - # ) - # else: - # url = "https://{}.s3.amazonaws.com/{}".format(bucket_name, key) - # additional_signed_qs = {"partNumber": str(partNumber), "uploadId": uploadId} - try: s3client = boto3.client( "s3", @@ -169,9 +161,6 @@ def generate_presigned_url_for_uploading_part( bucket, key, expires, uploadId, partNumber ) - # presigned_url = generate_aws_presigned_url( - # url, "PUT", credentials, "s3", region, expires, additional_signed_qs - # ) return presigned_url except Exception as e: raise InternalError( diff --git a/tests/conftest.py b/tests/conftest.py index 273f4a496..c3b106936 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,6 +33,10 @@ import pytest import requests from sqlalchemy.ext.compiler import compiles +import boto3 +import botocore +from gen3cirrus import AwsService +from botocore.stub import Stubber # Set FENCE_CONFIG_PATH *before* loading the configuration CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -1596,6 +1600,30 @@ def google_signed_url(): return manager +@pytest.fixture(scope="function") +def aws_signed_url(): + """ + Mock signed urls coming from AWS using a side effect function + """ + + def presigned_url_side_effect(*args, **kwargs): + print(args) + return f"https://{args[0]}/{args[1]}/?X-Amz-Expires={args[2]}" + + manager = MagicMock(side_effect=presigned_url_side_effect) + + patch( + "fence.blueprints.data.indexd.gen3cirrus.aws.services.AwsService.downloadPresignedURL", + manager, + ).start() + patch( + "fence.blueprints.data.indexd.gen3cirrus.aws.services.AwsService.uploadPresignedURL", + manager, + ).start() + + return manager + + @pytest.fixture(scope="function") def encoded_creds_jwt( kid, rsa_private_key, user_client, oauth_client, google_proxy_group diff --git a/tests/data/test_data.py b/tests/data/test_data.py index 568fb97bd..e6bccd839 100755 --- a/tests/data/test_data.py +++ b/tests/data/test_data.py @@ -72,6 +72,7 @@ def test_indexd_download_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -123,6 +124,7 @@ def test_indexd_upload_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -163,6 +165,7 @@ def test_indexd_upload_file_key_error( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test upload with a missing configuration key should fail @@ -211,6 +214,7 @@ def test_indexd_upload_file_filename( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, guid, file_name, ): @@ -255,6 +259,7 @@ def test_indexd_upload_file_filename_key_error( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1?file_name=`` with an example file name @@ -310,6 +315,7 @@ def test_indexd_upload_file_bucket( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, bucket, expected_status_code, ): @@ -353,6 +359,7 @@ def test_indexd_upload_file_doesnt_exist( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1`` when 1 doesn't exist. @@ -391,6 +398,7 @@ def test_indexd_download_file_no_protocol( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -443,6 +451,7 @@ def test_indexd_unauthorized_download_file( indexd_client, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -472,6 +481,7 @@ def test_unauthorized_indexd_download_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -536,6 +546,7 @@ def test_unauthorized_indexd_upload_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1``. @@ -600,6 +611,7 @@ def test_unavailable_indexd_upload_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1``. @@ -660,6 +672,7 @@ def test_public_object_download_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -686,6 +699,7 @@ def test_public_object_download_file_no_force_sign( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1?no_force_sign=True``. @@ -721,6 +735,7 @@ def test_public_bucket_download_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1`` with public bucket @@ -755,6 +770,7 @@ def test_public_bucket_download_file_no_force_sign( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1`` with public bucket with no_force_sign request @@ -777,6 +793,7 @@ def test_public_bucket_unsupported_protocol_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1`` with public bucket @@ -1271,6 +1288,7 @@ def test_assume_role_cache( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1`` with authorized user (user is the uploader). @@ -1380,6 +1398,7 @@ def test_indexd_download_with_uploader_unauthorized( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1`` with unauthorized user (user is not the uploader). @@ -1650,6 +1669,7 @@ def test_abac( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): mock_arborist_requests({"arborist/auth/request": {"POST": ({"auth": True}, 200)}}) indexd_client = indexd_client_with_arborist("test_abac") From fe8a5f2e211a403a1347b79d3d03e3dc600653a9 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 13 Aug 2024 11:50:41 -0500 Subject: [PATCH 06/38] resolve poetry.lock conflict --- poetry.lock | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/poetry.lock b/poetry.lock index 1df701742..f0630a08a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -250,17 +250,17 @@ files = [ [[package]] name = "boto3" -version = "1.34.156" +version = "1.34.159" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.156-py3-none-any.whl", hash = "sha256:cbbd453270b8ce94ef9da60dfbb6f9ceeb3eeee226b635aa9ec44b1def98cc96"}, - {file = "boto3-1.34.156.tar.gz", hash = "sha256:b33e9a8f8be80d3053b8418836a7c1900410b23a30c7cb040927d601a1082e68"}, + {file = "boto3-1.34.159-py3-none-any.whl", hash = "sha256:21120d23cc37c0e80dc4f64434bc5664d2a5645dcd9bf8a8fa97ed5c82164ca0"}, + {file = "boto3-1.34.159.tar.gz", hash = "sha256:ffe7bbb88ba81b5d54bc8fa0cfb2f3b7fe63a6cffa0f9207df2ef5c22a1c0587"}, ] [package.dependencies] -botocore = ">=1.34.156,<1.35.0" +botocore = ">=1.34.159,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -269,13 +269,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.156" +version = "1.34.159" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.156-py3-none-any.whl", hash = "sha256:c48f8c8996216dfdeeb0aa6d3c0f2c7ae25234766434a2ea3e57bdc08494bdda"}, - {file = "botocore-1.34.156.tar.gz", hash = "sha256:5d1478c41ab9681e660b3322432fe09c4055759c317984b7b8d3af9557ff769a"}, + {file = "botocore-1.34.159-py3-none-any.whl", hash = "sha256:7633062491457419a49f5860c014251ae85689f78266a3ce020c2c8688a76b97"}, + {file = "botocore-1.34.159.tar.gz", hash = "sha256:dc28806eb21e3c8d690c422530dff8b4b242ac033cbe98f160a9d37796c09cb1"}, ] [package.dependencies] @@ -1117,13 +1117,13 @@ grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] [[package]] name = "google-cloud-storage" -version = "2.18.1" +version = "2.18.2" description = "Google Cloud Storage API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google_cloud_storage-2.18.1-py2.py3-none-any.whl", hash = "sha256:9d8db6bde3a979cca7150511cd0e4cb363e5f69d31259d890ba1124fa109418c"}, - {file = "google_cloud_storage-2.18.1.tar.gz", hash = "sha256:6707a6f30a05aee36faca81296419ca2907ac750af1c0457f278bc9a6fb219ad"}, + {file = "google_cloud_storage-2.18.2-py2.py3-none-any.whl", hash = "sha256:97a4d45c368b7d401ed48c4fdfe86e1e1cb96401c9e199e419d289e2c0370166"}, + {file = "google_cloud_storage-2.18.2.tar.gz", hash = "sha256:aaf7acd70cdad9f274d29332673fcab98708d0e1f4dceb5a5356aaef06af4d99"}, ] [package.dependencies] @@ -1131,7 +1131,7 @@ google-api-core = ">=2.15.0,<3.0.0dev" google-auth = ">=2.26.1,<3.0dev" google-cloud-core = ">=2.3.0,<3.0dev" google-crc32c = ">=1.0,<2.0dev" -google-resumable-media = ">=2.6.0" +google-resumable-media = ">=2.7.2" requests = ">=2.18.0,<3.0.0dev" [package.extras] @@ -1704,13 +1704,13 @@ files = [ [[package]] name = "paramiko" -version = "3.4.0" +version = "3.4.1" description = "SSH2 protocol library" optional = false python-versions = ">=3.6" files = [ - {file = "paramiko-3.4.0-py3-none-any.whl", hash = "sha256:43f0b51115a896f9c00f59618023484cb3a14b98bbceab43394a39c6739b7ee7"}, - {file = "paramiko-3.4.0.tar.gz", hash = "sha256:aac08f26a31dc4dffd92821527d1682d99d52f9ef6851968114a8728f3c274d3"}, + {file = "paramiko-3.4.1-py3-none-any.whl", hash = "sha256:8e49fd2f82f84acf7ffd57c64311aa2b30e575370dc23bdb375b10262f7eac32"}, + {file = "paramiko-3.4.1.tar.gz", hash = "sha256:8b15302870af7f6652f2e038975c1d2973f06046cb5d7d65355668b3ecbece0c"}, ] [package.dependencies] @@ -2571,13 +2571,13 @@ files = [ [[package]] name = "zipp" -version = "3.19.2" +version = "3.20.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, - {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, + {file = "zipp-3.20.0-py3-none-any.whl", hash = "sha256:58da6168be89f0be59beb194da1250516fdaa062ccebd30127ac65d30045e10d"}, + {file = "zipp-3.20.0.tar.gz", hash = "sha256:0145e43d89664cfe1a2e533adc75adafed82fe2da404b4bbb6b026c0157bdb31"}, ] [package.extras] From b361663ae330f5111ddef370f8961820e77d1c78 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 13 Aug 2024 13:05:24 -0500 Subject: [PATCH 07/38] add mock presigned urls for blank tests --- tests/data/test_blank_index.py | 10 +++++++++- tests/data/test_data.py | 7 ++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/data/test_blank_index.py b/tests/data/test_blank_index.py index 7586d85be..f5f6bb400 100755 --- a/tests/data/test_blank_index.py +++ b/tests/data/test_blank_index.py @@ -41,7 +41,14 @@ def text(self): return self.data -def test_blank_index_upload(app, client, auth_client, encoded_creds_jwt, user_client): +def test_blank_index_upload( + app, + client, + auth_client, + encoded_creds_jwt, + user_client, + aws_signed_url, +): """ test BlankIndex upload POST /data/upload @@ -149,6 +156,7 @@ def test_blank_index_upload_bucket( user_client, bucket, expected_status_code, + aws_signed_url, ): """ Same test as above, except request a specific bucket to upload the file to diff --git a/tests/data/test_data.py b/tests/data/test_data.py index e6bccd839..f0a3a9852 100755 --- a/tests/data/test_data.py +++ b/tests/data/test_data.py @@ -1622,7 +1622,12 @@ def json(self): def test_blank_index_upload_unauthorized( - app, client, auth_client, encoded_creds_jwt, user_client + app, + client, + auth_client, + encoded_creds_jwt, + user_client, + aws_signed_url, ): class MockResponse(object): def __init__(self, data, status_code=200): From e4d64ee7e9c8b8ca80c204b7f0b324b260c3c0a4 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 8 Aug 2024 10:08:42 -0500 Subject: [PATCH 08/38] add functionality for requester pays buckets --- fence/blueprints/data/indexd.py | 42 +++++-- fence/blueprints/data/multipart_upload.py | 30 +++-- poetry.lock | 128 +++++++++------------- pyproject.toml | 5 +- 4 files changed, 102 insertions(+), 103 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index fe1383321..8f99d9e08 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1,16 +1,17 @@ import re import time import json -from urllib.parse import urlparse, ParseResult, urlunparse +import boto3 +from urllib.parse import urlparse, ParseResult, urlunparse, quote from datetime import datetime, timedelta from sqlalchemy.sql.functions import user from cached_property import cached_property import gen3cirrus from gen3cirrus import GoogleCloudManager +from gen3cirrus import AwsService from cdislogging import get_logger from cdispyutils.config import get_value -from cdispyutils.hmac4 import generate_aws_presigned_url import flask from flask import current_app import requests @@ -1061,6 +1062,8 @@ def get_signed_url( bucket_name = self.bucket_name() bucket = s3_buckets.get(bucket_name) + object_id = self.parsed_url.path.strip("/") + if bucket and bucket.get("endpoint_url"): http_url = bucket["endpoint_url"].strip("/") + "/{}/{}".format( self.parsed_url.netloc, self.parsed_url.path.strip("/") @@ -1093,19 +1096,34 @@ def get_signed_url( self.parsed_url.netloc, credential ) - auth_info = _get_auth_info_for_id_or_from_request(user=authorized_user) - - url = generate_aws_presigned_url( - http_url, - ACTION_DICT["s3"][action], - credential, + client = boto3.client( "s3", - region, - expires_in, - auth_info, + aws_access_key_id=credential["aws_access_key_id"], + aws_secret_access_key=credential["aws_secret_access_key"], ) - return url + cirrus_aws = AwsService(client) + auth_info = _get_auth_info_for_id_or_from_request(user=authorized_user) + + action = ACTION_DICT["s3"][action] + if action == "PUT": # get presigned url for upload + url = cirrus_aws.uploadPresignedURL(bucket_name, object_id, expires_in) + else: # get presigned url for download + print(f"---------- DEBUG ------------------ Bucket config {bucket}") + if bucket.get("requester_pays", False) == True: + url = cirrus_aws.requesterPaysDownloadPresignedURL( + bucket_name, object_id, expires_in + ) + else: + url = cirrus_aws.downloadPresignedURL( + bucket_name, object_id, expires_in + ) + + canonical_qs = "" + for key in sorted(auth_info.keys()): + canonical_qs += "&" + key + "=" + quote(auth_info[key], safe="") + + return url + "?" + canonical_qs def init_multipart_upload(self, expires_in): """ diff --git a/fence/blueprints/data/multipart_upload.py b/fence/blueprints/data/multipart_upload.py index 7352f66f1..5a56f482b 100644 --- a/fence/blueprints/data/multipart_upload.py +++ b/fence/blueprints/data/multipart_upload.py @@ -2,7 +2,6 @@ from botocore.exceptions import ClientError from retry.api import retry_call -from cdispyutils.hmac4 import generate_aws_presigned_url from cdispyutils.config import get_value from cdislogging import get_logger from fence.config import config @@ -150,18 +149,29 @@ def generate_presigned_url_for_uploading_part( ) bucket = s3_buckets.get(bucket_name) - if bucket.get("endpoint_url"): - url = bucket["endpoint_url"].strip("/") + "/{}/{}".format( - bucket_name, key.strip("/") - ) - else: - url = "https://{}.s3.amazonaws.com/{}".format(bucket_name, key) - additional_signed_qs = {"partNumber": str(partNumber), "uploadId": uploadId} + # if bucket.get("endpoint_url"): + # url = bucket["endpoint_url"].strip("/") + "/{}/{}".format( + # bucket_name, key.strip("/") + # ) + # else: + # url = "https://{}.s3.amazonaws.com/{}".format(bucket_name, key) + # additional_signed_qs = {"partNumber": str(partNumber), "uploadId": uploadId} try: - presigned_url = generate_aws_presigned_url( - url, "PUT", credentials, "s3", region, expires, additional_signed_qs + s3client = boto3.client( + "s3", + aws_access_key_id=credentials["aws_access_key_id"], + aws_secret_access_key=credentials["aws_secret_access_key"], ) + cirrus_aws = AwsService(client) + + presigned_url = cirrus_aws.multipartUploadPresignedURL( + bucket, key, expires, uploadId, partNumber + ) + + # presigned_url = generate_aws_presigned_url( + # url, "PUT", credentials, "s3", region, expires, additional_signed_qs + # ) return presigned_url except Exception as e: raise InternalError( diff --git a/poetry.lock b/poetry.lock index 29e85be0a..694b156cb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "addict" @@ -250,17 +250,17 @@ files = [ [[package]] name = "boto3" -version = "1.34.158" +version = "1.34.159" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.158-py3-none-any.whl", hash = "sha256:c29e9b7e1034e8734ccaffb9f2b3f3df2268022fd8a93d836604019f8759ce27"}, - {file = "boto3-1.34.158.tar.gz", hash = "sha256:5b7b2ce0ec1e498933f600d29f3e1c641f8c44dd7e468c26795359d23d81fa39"}, + {file = "boto3-1.34.159-py3-none-any.whl", hash = "sha256:21120d23cc37c0e80dc4f64434bc5664d2a5645dcd9bf8a8fa97ed5c82164ca0"}, + {file = "boto3-1.34.159.tar.gz", hash = "sha256:ffe7bbb88ba81b5d54bc8fa0cfb2f3b7fe63a6cffa0f9207df2ef5c22a1c0587"}, ] [package.dependencies] -botocore = ">=1.34.158,<1.35.0" +botocore = ">=1.34.159,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -269,13 +269,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.158" +version = "1.34.159" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.158-py3-none-any.whl", hash = "sha256:0e6fceba1e39bfa8feeba70ba3ac2af958b3387df4bd3b5f2db3f64c1754c756"}, - {file = "botocore-1.34.158.tar.gz", hash = "sha256:5934082e25ad726673afbf466092fb1223dafa250e6e756c819430ba6b1b3da5"}, + {file = "botocore-1.34.159-py3-none-any.whl", hash = "sha256:7633062491457419a49f5860c014251ae85689f78266a3ce020c2c8688a76b97"}, + {file = "botocore-1.34.159.tar.gz", hash = "sha256:dc28806eb21e3c8d690c422530dff8b4b242ac033cbe98f160a9d37796c09cb1"}, ] [package.dependencies] @@ -964,14 +964,13 @@ name = "gen3cirrus" version = "3.0.1" description = "" optional = false -python-versions = ">=3.9,<4.0" -files = [ - {file = "gen3cirrus-3.0.1-py3-none-any.whl", hash = "sha256:74628faca3b1cbe65c78e08eb567e1ac0cb8ae52e1bfc603f904af0277e3cb52"}, - {file = "gen3cirrus-3.0.1.tar.gz", hash = "sha256:0ae0ddc0ee7df870603457fe186245f3c8124d989254276e5011a23e1139a6c8"}, -] +python-versions = "^3.9" +files = [] +develop = false [package.dependencies] backoff = "*" +boto3 = "*" cdislogging = "*" google-api-python-client = "*" google-auth = "*" @@ -979,6 +978,12 @@ google-auth-httplib2 = "*" google-cloud-storage = "*" oauth2client = "*" +[package.source] +type = "git" +url = "https://github.com/uc-cdis/cirrus" +reference = "feat/s3-functionality" +resolved_reference = "b648ed107386409e5c91daeda548b0f036689c53" + [[package]] name = "gen3config" version = "1.1.0" @@ -1419,6 +1424,17 @@ doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linke perf = ["ipython"] test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "isodate" version = "0.6.1" @@ -1623,17 +1639,6 @@ six = ">=1.9" docs = ["Pygments (<2)", "jinja2 (<2.7)", "sphinx", "sphinx (<1.3)"] test = ["unittest2 (>=1.1.0)"] -[[package]] -name = "more-itertools" -version = "10.4.0" -description = "More routines for operating on iterables, beyond itertools" -optional = false -python-versions = ">=3.8" -files = [ - {file = "more-itertools-10.4.0.tar.gz", hash = "sha256:fe0e63c4ab068eac62410ab05cccca2dc71ec44ba8ef29916a0090df061cf923"}, - {file = "more_itertools-10.4.0-py3-none-any.whl", hash = "sha256:0f7d9f83a0a8dcfa8a2694a770590d98a67ea943e3d9f5298309a484758c4e27"}, -] - [[package]] name = "moto" version = "1.3.7" @@ -1699,13 +1704,13 @@ files = [ [[package]] name = "paramiko" -version = "3.4.0" +version = "3.4.1" description = "SSH2 protocol library" optional = false python-versions = ">=3.6" files = [ - {file = "paramiko-3.4.0-py3-none-any.whl", hash = "sha256:43f0b51115a896f9c00f59618023484cb3a14b98bbceab43394a39c6739b7ee7"}, - {file = "paramiko-3.4.0.tar.gz", hash = "sha256:aac08f26a31dc4dffd92821527d1682d99d52f9ef6851968114a8728f3c274d3"}, + {file = "paramiko-3.4.1-py3-none-any.whl", hash = "sha256:8e49fd2f82f84acf7ffd57c64311aa2b30e575370dc23bdb375b10262f7eac32"}, + {file = "paramiko-3.4.1.tar.gz", hash = "sha256:8b15302870af7f6652f2e038975c1d2973f06046cb5d7d65355668b3ecbece0c"}, ] [package.dependencies] @@ -1731,17 +1736,18 @@ files = [ [[package]] name = "pluggy" -version = "0.13.1" +version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.8" files = [ - {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, - {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] [package.extras] dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "prometheus-client" @@ -1794,28 +1800,6 @@ files = [ {file = "protobuf-5.27.3.tar.gz", hash = "sha256:82460903e640f2b7e34ee81a947fdaad89de796d324bcbc38ff5430bcdead82c"}, ] -[[package]] -name = "psycopg2" -version = "2.9.9" -description = "psycopg2 - Python-PostgreSQL Database Adapter" -optional = false -python-versions = ">=3.7" -files = [ - {file = "psycopg2-2.9.9-cp310-cp310-win32.whl", hash = "sha256:38a8dcc6856f569068b47de286b472b7c473ac7977243593a288ebce0dc89516"}, - {file = "psycopg2-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3"}, - {file = "psycopg2-2.9.9-cp311-cp311-win32.whl", hash = "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372"}, - {file = "psycopg2-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981"}, - {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, - {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, - {file = "psycopg2-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa"}, - {file = "psycopg2-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a"}, - {file = "psycopg2-2.9.9-cp38-cp38-win32.whl", hash = "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c"}, - {file = "psycopg2-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:bac58c024c9922c23550af2a581998624d6e02350f4ae9c5f0bc642c633a2d5e"}, - {file = "psycopg2-2.9.9-cp39-cp39-win32.whl", hash = "sha256:c92811b2d4c9b6ea0285942b2e7cac98a59e166d59c588fe5cfe1eda58e72d59"}, - {file = "psycopg2-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:de80739447af31525feddeb8effd640782cf5998e1a4e9192ebdf829717e3913"}, - {file = "psycopg2-2.9.9.tar.gz", hash = "sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156"}, -] - [[package]] name = "py" version = "1.11.0" @@ -1983,27 +1967,26 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "5.4.3" +version = "6.2.5" description = "pytest: simple powerful testing with Python" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" files = [ - {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"}, - {file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"}, + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, ] [package.dependencies] atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=17.4.0" +attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} -more-itertools = ">=4.0.0" +iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<1.0" -py = ">=1.5.0" -wcwidth = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" [package.extras] -checkqa-mypy = ["mypy (==v0.761)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] [[package]] @@ -2440,17 +2423,6 @@ files = [ cdislogging = "*" sqlalchemy = ">=1.3.3" -[[package]] -name = "wcwidth" -version = "0.2.13" -description = "Measures the displayed width of unicode strings in a terminal" -optional = false -python-versions = "*" -files = [ - {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, - {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, -] - [[package]] name = "werkzeug" version = "3.0.3" @@ -2577,13 +2549,13 @@ files = [ [[package]] name = "zipp" -version = "3.19.2" +version = "3.20.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, - {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, + {file = "zipp-3.20.0-py3-none-any.whl", hash = "sha256:58da6168be89f0be59beb194da1250516fdaa062ccebd30127ac65d30045e10d"}, + {file = "zipp-3.20.0.tar.gz", hash = "sha256:0145e43d89664cfe1a2e533adc75adafed82fe2da404b4bbb6b026c0157bdb31"}, ] [package.extras] @@ -2593,4 +2565,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0.0" -content-hash = "d003418dcc0d68257a215186d21776941f8739bd3b4f898762a93e7895d7c89e" +content-hash = "325bef0fbcf6d82845a31d349ac9b40b908143db6876b92ec3d1361a7d041261" diff --git a/pyproject.toml b/pyproject.toml index 2a8b74a29..02a705248 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ flask-cors = ">=3.0.3" flask-restful = ">=0.3.8" email_validator = "^1.1.1" gen3authz = "^1.5.1" -gen3cirrus = ">=3.0.1" +gen3cirrus = {git = "https://github.com/uc-cdis/cirrus", rev = "feat/s3-functionality"} gen3config = ">=1.1.0" gen3users = "^1.0.2" idna = "^3.7" @@ -39,7 +39,6 @@ markupsafe = "^2.0.1" paramiko = ">=2.6.0" prometheus-client = "<1" -psycopg2 = "^2.8.3" PyJWT = "^2.4.0" python_dateutil = "^2.6.1" python-jose = "^2.0.2" @@ -64,7 +63,7 @@ codacy-coverage = "^1.3.11" coveralls = "^2.1.1" mock = "^2.0.0" moto = "^1.1.24" -pytest = "^5.2.0" +pytest = "^6.2.5" pytest-cov = "^2.5.1" pytest-flask = ">=1.3.0" From eddf1dd22209a99af21c5ab55f84aba36e167d11 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 8 Aug 2024 10:30:25 -0500 Subject: [PATCH 09/38] add removed dependency --- fence/blueprints/data/indexd.py | 2 +- poetry.lock | 24 +++++++++++++++++++++++- pyproject.toml | 1 + 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 8f99d9e08..54941d5e3 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1110,7 +1110,7 @@ def get_signed_url( url = cirrus_aws.uploadPresignedURL(bucket_name, object_id, expires_in) else: # get presigned url for download print(f"---------- DEBUG ------------------ Bucket config {bucket}") - if bucket.get("requester_pays", False) == True: + if bucket.get("requester_pays") == True: url = cirrus_aws.requesterPaysDownloadPresignedURL( bucket_name, object_id, expires_in ) diff --git a/poetry.lock b/poetry.lock index 694b156cb..f0630a08a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1800,6 +1800,28 @@ files = [ {file = "protobuf-5.27.3.tar.gz", hash = "sha256:82460903e640f2b7e34ee81a947fdaad89de796d324bcbc38ff5430bcdead82c"}, ] +[[package]] +name = "psycopg2" +version = "2.9.9" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "psycopg2-2.9.9-cp310-cp310-win32.whl", hash = "sha256:38a8dcc6856f569068b47de286b472b7c473ac7977243593a288ebce0dc89516"}, + {file = "psycopg2-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3"}, + {file = "psycopg2-2.9.9-cp311-cp311-win32.whl", hash = "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372"}, + {file = "psycopg2-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981"}, + {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, + {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, + {file = "psycopg2-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa"}, + {file = "psycopg2-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a"}, + {file = "psycopg2-2.9.9-cp38-cp38-win32.whl", hash = "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c"}, + {file = "psycopg2-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:bac58c024c9922c23550af2a581998624d6e02350f4ae9c5f0bc642c633a2d5e"}, + {file = "psycopg2-2.9.9-cp39-cp39-win32.whl", hash = "sha256:c92811b2d4c9b6ea0285942b2e7cac98a59e166d59c588fe5cfe1eda58e72d59"}, + {file = "psycopg2-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:de80739447af31525feddeb8effd640782cf5998e1a4e9192ebdf829717e3913"}, + {file = "psycopg2-2.9.9.tar.gz", hash = "sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156"}, +] + [[package]] name = "py" version = "1.11.0" @@ -2565,4 +2587,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0.0" -content-hash = "325bef0fbcf6d82845a31d349ac9b40b908143db6876b92ec3d1361a7d041261" +content-hash = "597d2a7ac8980713561f7d16c82f045dcb4f8997da0cfba73d2299d791c9b58e" diff --git a/pyproject.toml b/pyproject.toml index 02a705248..2a157f090 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ markupsafe = "^2.0.1" paramiko = ">=2.6.0" prometheus-client = "<1" +psycopg2 = "*" PyJWT = "^2.4.0" python_dateutil = "^2.6.1" python-jose = "^2.0.2" From 01a1867b77a55e28bc8f5061b69f4d93a79b5af6 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 8 Aug 2024 12:07:11 -0500 Subject: [PATCH 10/38] add debug for requester pays --- fence/blueprints/data/indexd.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 54941d5e3..3d6851e48 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1111,6 +1111,9 @@ def get_signed_url( else: # get presigned url for download print(f"---------- DEBUG ------------------ Bucket config {bucket}") if bucket.get("requester_pays") == True: + print( + f"---------- DEBUG ------------------ WE are generateing a presigned url for a requester pays bucket!!" + ) url = cirrus_aws.requesterPaysDownloadPresignedURL( bucket_name, object_id, expires_in ) From 5f1a5e5f40c2ecce95afa71a474cf94b1bd74d62 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 8 Aug 2024 17:52:01 -0500 Subject: [PATCH 11/38] remove debug statements and fix url concat --- fence/blueprints/data/indexd.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 3d6851e48..411e158d9 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1106,14 +1106,11 @@ def get_signed_url( auth_info = _get_auth_info_for_id_or_from_request(user=authorized_user) action = ACTION_DICT["s3"][action] + if action == "PUT": # get presigned url for upload url = cirrus_aws.uploadPresignedURL(bucket_name, object_id, expires_in) else: # get presigned url for download - print(f"---------- DEBUG ------------------ Bucket config {bucket}") if bucket.get("requester_pays") == True: - print( - f"---------- DEBUG ------------------ WE are generateing a presigned url for a requester pays bucket!!" - ) url = cirrus_aws.requesterPaysDownloadPresignedURL( bucket_name, object_id, expires_in ) @@ -1126,7 +1123,7 @@ def get_signed_url( for key in sorted(auth_info.keys()): canonical_qs += "&" + key + "=" + quote(auth_info[key], safe="") - return url + "?" + canonical_qs + return url + canonical_qs def init_multipart_upload(self, expires_in): """ From 6ee194753e3126ed8bdc8d05ed444d096b6948dd Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 13 Aug 2024 11:49:05 -0500 Subject: [PATCH 12/38] fix presigned url tests and clean up --- .secrets.baseline | 6 ++--- fence/blueprints/data/multipart_upload.py | 11 --------- tests/conftest.py | 28 +++++++++++++++++++++++ tests/data/test_data.py | 20 ++++++++++++++++ 4 files changed, 51 insertions(+), 14 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 9aa531c19..b70519cdc 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -268,14 +268,14 @@ "filename": "tests/conftest.py", "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9", "is_verified": false, - "line_number": 1569 + "line_number": 1573 }, { "type": "Base64 High Entropy String", "filename": "tests/conftest.py", "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108", "is_verified": false, - "line_number": 1593 + "line_number": 1597 } ], "tests/credentials/google/test_credentials.py": [ @@ -422,5 +422,5 @@ } ] }, - "generated_at": "2024-07-25T17:19:58Z" + "generated_at": "2024-08-13T16:48:45Z" } diff --git a/fence/blueprints/data/multipart_upload.py b/fence/blueprints/data/multipart_upload.py index 5a56f482b..b8112f0c1 100644 --- a/fence/blueprints/data/multipart_upload.py +++ b/fence/blueprints/data/multipart_upload.py @@ -149,14 +149,6 @@ def generate_presigned_url_for_uploading_part( ) bucket = s3_buckets.get(bucket_name) - # if bucket.get("endpoint_url"): - # url = bucket["endpoint_url"].strip("/") + "/{}/{}".format( - # bucket_name, key.strip("/") - # ) - # else: - # url = "https://{}.s3.amazonaws.com/{}".format(bucket_name, key) - # additional_signed_qs = {"partNumber": str(partNumber), "uploadId": uploadId} - try: s3client = boto3.client( "s3", @@ -169,9 +161,6 @@ def generate_presigned_url_for_uploading_part( bucket, key, expires, uploadId, partNumber ) - # presigned_url = generate_aws_presigned_url( - # url, "PUT", credentials, "s3", region, expires, additional_signed_qs - # ) return presigned_url except Exception as e: raise InternalError( diff --git a/tests/conftest.py b/tests/conftest.py index 273f4a496..c3b106936 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,6 +33,10 @@ import pytest import requests from sqlalchemy.ext.compiler import compiles +import boto3 +import botocore +from gen3cirrus import AwsService +from botocore.stub import Stubber # Set FENCE_CONFIG_PATH *before* loading the configuration CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -1596,6 +1600,30 @@ def google_signed_url(): return manager +@pytest.fixture(scope="function") +def aws_signed_url(): + """ + Mock signed urls coming from AWS using a side effect function + """ + + def presigned_url_side_effect(*args, **kwargs): + print(args) + return f"https://{args[0]}/{args[1]}/?X-Amz-Expires={args[2]}" + + manager = MagicMock(side_effect=presigned_url_side_effect) + + patch( + "fence.blueprints.data.indexd.gen3cirrus.aws.services.AwsService.downloadPresignedURL", + manager, + ).start() + patch( + "fence.blueprints.data.indexd.gen3cirrus.aws.services.AwsService.uploadPresignedURL", + manager, + ).start() + + return manager + + @pytest.fixture(scope="function") def encoded_creds_jwt( kid, rsa_private_key, user_client, oauth_client, google_proxy_group diff --git a/tests/data/test_data.py b/tests/data/test_data.py index 568fb97bd..e6bccd839 100755 --- a/tests/data/test_data.py +++ b/tests/data/test_data.py @@ -72,6 +72,7 @@ def test_indexd_download_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -123,6 +124,7 @@ def test_indexd_upload_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -163,6 +165,7 @@ def test_indexd_upload_file_key_error( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test upload with a missing configuration key should fail @@ -211,6 +214,7 @@ def test_indexd_upload_file_filename( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, guid, file_name, ): @@ -255,6 +259,7 @@ def test_indexd_upload_file_filename_key_error( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1?file_name=`` with an example file name @@ -310,6 +315,7 @@ def test_indexd_upload_file_bucket( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, bucket, expected_status_code, ): @@ -353,6 +359,7 @@ def test_indexd_upload_file_doesnt_exist( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1`` when 1 doesn't exist. @@ -391,6 +398,7 @@ def test_indexd_download_file_no_protocol( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -443,6 +451,7 @@ def test_indexd_unauthorized_download_file( indexd_client, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -472,6 +481,7 @@ def test_unauthorized_indexd_download_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -536,6 +546,7 @@ def test_unauthorized_indexd_upload_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1``. @@ -600,6 +611,7 @@ def test_unavailable_indexd_upload_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1``. @@ -660,6 +672,7 @@ def test_public_object_download_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1``. @@ -686,6 +699,7 @@ def test_public_object_download_file_no_force_sign( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1?no_force_sign=True``. @@ -721,6 +735,7 @@ def test_public_bucket_download_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1`` with public bucket @@ -755,6 +770,7 @@ def test_public_bucket_download_file_no_force_sign( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1`` with public bucket with no_force_sign request @@ -777,6 +793,7 @@ def test_public_bucket_unsupported_protocol_file( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/upload/1`` with public bucket @@ -1271,6 +1288,7 @@ def test_assume_role_cache( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1`` with authorized user (user is the uploader). @@ -1380,6 +1398,7 @@ def test_indexd_download_with_uploader_unauthorized( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): """ Test ``GET /data/download/1`` with unauthorized user (user is not the uploader). @@ -1650,6 +1669,7 @@ def test_abac( primary_google_service_account, cloud_manager, google_signed_url, + aws_signed_url, ): mock_arborist_requests({"arborist/auth/request": {"POST": ({"auth": True}, 200)}}) indexd_client = indexd_client_with_arborist("test_abac") From a31cab2381d3e20cc9f58d22645371aecf972dad Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 13 Aug 2024 13:05:24 -0500 Subject: [PATCH 13/38] add mock presigned urls for blank tests --- tests/data/test_blank_index.py | 10 +++++++++- tests/data/test_data.py | 7 ++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/data/test_blank_index.py b/tests/data/test_blank_index.py index 7586d85be..f5f6bb400 100755 --- a/tests/data/test_blank_index.py +++ b/tests/data/test_blank_index.py @@ -41,7 +41,14 @@ def text(self): return self.data -def test_blank_index_upload(app, client, auth_client, encoded_creds_jwt, user_client): +def test_blank_index_upload( + app, + client, + auth_client, + encoded_creds_jwt, + user_client, + aws_signed_url, +): """ test BlankIndex upload POST /data/upload @@ -149,6 +156,7 @@ def test_blank_index_upload_bucket( user_client, bucket, expected_status_code, + aws_signed_url, ): """ Same test as above, except request a specific bucket to upload the file to diff --git a/tests/data/test_data.py b/tests/data/test_data.py index e6bccd839..f0a3a9852 100755 --- a/tests/data/test_data.py +++ b/tests/data/test_data.py @@ -1622,7 +1622,12 @@ def json(self): def test_blank_index_upload_unauthorized( - app, client, auth_client, encoded_creds_jwt, user_client + app, + client, + auth_client, + encoded_creds_jwt, + user_client, + aws_signed_url, ): class MockResponse(object): def __init__(self, data, status_code=200): From 83b8b38af3df5c1179309dc6499990c9988ea747 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 14 Aug 2024 16:25:47 -0500 Subject: [PATCH 14/38] clean up and fix function case --- fence/blueprints/data/indexd.py | 7 ++++--- fence/blueprints/data/multipart_upload.py | 2 +- poetry.lock | 22 +++++++++++----------- tests/conftest.py | 14 ++++++++------ 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 411e158d9..650cdfdfd 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1100,6 +1100,7 @@ def get_signed_url( "s3", aws_access_key_id=credential["aws_access_key_id"], aws_secret_access_key=credential["aws_secret_access_key"], + region_name=region, ) cirrus_aws = AwsService(client) @@ -1108,14 +1109,14 @@ def get_signed_url( action = ACTION_DICT["s3"][action] if action == "PUT": # get presigned url for upload - url = cirrus_aws.uploadPresignedURL(bucket_name, object_id, expires_in) + url = cirrus_aws.upload_presigned_url(bucket_name, object_id, expires_in) else: # get presigned url for download if bucket.get("requester_pays") == True: - url = cirrus_aws.requesterPaysDownloadPresignedURL( + url = cirrus_aws.requester_pays_download_presigned_url( bucket_name, object_id, expires_in ) else: - url = cirrus_aws.downloadPresignedURL( + url = cirrus_aws.download_presigned_url( bucket_name, object_id, expires_in ) diff --git a/fence/blueprints/data/multipart_upload.py b/fence/blueprints/data/multipart_upload.py index b8112f0c1..a1ab5bf94 100644 --- a/fence/blueprints/data/multipart_upload.py +++ b/fence/blueprints/data/multipart_upload.py @@ -157,7 +157,7 @@ def generate_presigned_url_for_uploading_part( ) cirrus_aws = AwsService(client) - presigned_url = cirrus_aws.multipartUploadPresignedURL( + presigned_url = cirrus_aws.multipart_upload_presigned_url( bucket, key, expires, uploadId, partNumber ) diff --git a/poetry.lock b/poetry.lock index f0630a08a..250303678 100644 --- a/poetry.lock +++ b/poetry.lock @@ -250,17 +250,17 @@ files = [ [[package]] name = "boto3" -version = "1.34.159" +version = "1.34.161" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.159-py3-none-any.whl", hash = "sha256:21120d23cc37c0e80dc4f64434bc5664d2a5645dcd9bf8a8fa97ed5c82164ca0"}, - {file = "boto3-1.34.159.tar.gz", hash = "sha256:ffe7bbb88ba81b5d54bc8fa0cfb2f3b7fe63a6cffa0f9207df2ef5c22a1c0587"}, + {file = "boto3-1.34.161-py3-none-any.whl", hash = "sha256:4ef285334a0edc3047e27a04caf00f7742e32c0f03a361101e768014ac5709dd"}, + {file = "boto3-1.34.161.tar.gz", hash = "sha256:a872d8fdb3203c1eb0b12fa9e9d879e6f7fd02983a485f02189e6d5914ccd834"}, ] [package.dependencies] -botocore = ">=1.34.159,<1.35.0" +botocore = ">=1.34.161,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -269,13 +269,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.159" +version = "1.34.161" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.159-py3-none-any.whl", hash = "sha256:7633062491457419a49f5860c014251ae85689f78266a3ce020c2c8688a76b97"}, - {file = "botocore-1.34.159.tar.gz", hash = "sha256:dc28806eb21e3c8d690c422530dff8b4b242ac033cbe98f160a9d37796c09cb1"}, + {file = "botocore-1.34.161-py3-none-any.whl", hash = "sha256:6c606d2da6f62fde06880aff1190566af208875c29938b6b68741e607817975a"}, + {file = "botocore-1.34.161.tar.gz", hash = "sha256:16381bfb786142099abf170ce734b95a402a3a7f8e4016358712ac333c5568b2"}, ] [package.dependencies] @@ -982,7 +982,7 @@ oauth2client = "*" type = "git" url = "https://github.com/uc-cdis/cirrus" reference = "feat/s3-functionality" -resolved_reference = "b648ed107386409e5c91daeda548b0f036689c53" +resolved_reference = "acda7010a323d158cc7c5034bb37095b2a5a5b20" [[package]] name = "gen3config" @@ -1043,13 +1043,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-api-python-client" -version = "2.140.0" +version = "2.141.0" description = "Google API Client Library for Python" optional = false python-versions = ">=3.7" files = [ - {file = "google_api_python_client-2.140.0-py2.py3-none-any.whl", hash = "sha256:aeb4bb99e9fdd241473da5ff35464a0658fea0db76fe89c0f8c77ecfc3813404"}, - {file = "google_api_python_client-2.140.0.tar.gz", hash = "sha256:0bb973adccbe66a3d0a70abe4e49b3f2f004d849416bfec38d22b75649d389d8"}, + {file = "google_api_python_client-2.141.0-py2.py3-none-any.whl", hash = "sha256:43c05322b91791204465291b3852718fae38d4f84b411d8be847c4f86882652a"}, + {file = "google_api_python_client-2.141.0.tar.gz", hash = "sha256:0f225b1f45d5a6f8c2a400f48729f5d6da9a81138e81e0478d61fdd8edf6563a"}, ] [package.dependencies] diff --git a/tests/conftest.py b/tests/conftest.py index c3b106936..041994eef 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1607,21 +1607,23 @@ def aws_signed_url(): """ def presigned_url_side_effect(*args, **kwargs): - print(args) return f"https://{args[0]}/{args[1]}/?X-Amz-Expires={args[2]}" manager = MagicMock(side_effect=presigned_url_side_effect) - patch( - "fence.blueprints.data.indexd.gen3cirrus.aws.services.AwsService.downloadPresignedURL", + down = patch( + "fence.blueprints.data.indexd.gen3cirrus.aws.services.AwsService.download_presigned_url", manager, ).start() - patch( - "fence.blueprints.data.indexd.gen3cirrus.aws.services.AwsService.uploadPresignedURL", + up = patch( + "fence.blueprints.data.indexd.gen3cirrus.aws.services.AwsService.upload_presigned_url", manager, ).start() - return manager + yield manager + + down.stop() + up.stop() @pytest.fixture(scope="function") From af9684051f7c903841334d16fe0ff5f5de57ca86 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 15 Aug 2024 14:19:14 -0500 Subject: [PATCH 15/38] adding config to boto clients --- fence/blueprints/data/indexd.py | 2 ++ fence/blueprints/data/multipart_upload.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 650cdfdfd..54f903423 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -2,6 +2,7 @@ import time import json import boto3 +from botocore.client import Config from urllib.parse import urlparse, ParseResult, urlunparse, quote from datetime import datetime, timedelta @@ -1101,6 +1102,7 @@ def get_signed_url( aws_access_key_id=credential["aws_access_key_id"], aws_secret_access_key=credential["aws_secret_access_key"], region_name=region, + config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"), ) cirrus_aws = AwsService(client) diff --git a/fence/blueprints/data/multipart_upload.py b/fence/blueprints/data/multipart_upload.py index a1ab5bf94..f17f95b9d 100644 --- a/fence/blueprints/data/multipart_upload.py +++ b/fence/blueprints/data/multipart_upload.py @@ -1,4 +1,5 @@ import boto3 +from botocore.client import Config from botocore.exceptions import ClientError from retry.api import retry_call @@ -154,6 +155,7 @@ def generate_presigned_url_for_uploading_part( "s3", aws_access_key_id=credentials["aws_access_key_id"], aws_secret_access_key=credentials["aws_secret_access_key"], + config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"), ) cirrus_aws = AwsService(client) From 4b565cb4751a66f922ab4cde1d05ec7e096a60cf Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 15 Aug 2024 16:15:10 -0500 Subject: [PATCH 16/38] add handler for custom parameters for s3 presigned urls --- fence/blueprints/data/indexd.py | 37 ++++++++++++++++++++++++++------- poetry.lock | 16 +++++++------- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 54f903423..419bc3bb9 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1045,6 +1045,24 @@ def get_bucket_region(self): else: return bucket_cred["region"] + def is_custom(k): + if k == "user_id" or k == "username": + return True + else: + return False + + def client_param_handler(*, params, context, **_kw): + # Store custom parameters in context for later event handlers + context["custom_params"] = {k: v for k, v in params.items() if is_custom(k)} + # Remove custom parameters from client parameters, + # because validation would fail on them + return {k: v for k, v in params.items() if not is_custom(k)} + + def request_param_injector(*, request, **_kw): + if request.context["custom_params"]: + request.url += "&" if "?" in request.url else "?" + request.url += urlencode(request.context["custom_params"]) + def get_signed_url( self, action, @@ -1105,28 +1123,31 @@ def get_signed_url( config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"), ) + client.meta.events.register( + "provide-client-params.s3.GetObject", client_param_handler + ) + client.meta.events.register("before-sign.s3.GetObject", request_param_injector) + cirrus_aws = AwsService(client) auth_info = _get_auth_info_for_id_or_from_request(user=authorized_user) action = ACTION_DICT["s3"][action] if action == "PUT": # get presigned url for upload - url = cirrus_aws.upload_presigned_url(bucket_name, object_id, expires_in) + url = cirrus_aws.upload_presigned_url( + bucket_name, object_id, expires_in, auth_info + ) else: # get presigned url for download if bucket.get("requester_pays") == True: url = cirrus_aws.requester_pays_download_presigned_url( - bucket_name, object_id, expires_in + bucket_name, object_id, expires_in, auth_info ) else: url = cirrus_aws.download_presigned_url( - bucket_name, object_id, expires_in + bucket_name, object_id, expires_in, auth_info ) - canonical_qs = "" - for key in sorted(auth_info.keys()): - canonical_qs += "&" + key + "=" + quote(auth_info[key], safe="") - - return url + canonical_qs + return url def init_multipart_upload(self, expires_in): """ diff --git a/poetry.lock b/poetry.lock index 250303678..db991a3bf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -250,17 +250,17 @@ files = [ [[package]] name = "boto3" -version = "1.34.161" +version = "1.34.162" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.161-py3-none-any.whl", hash = "sha256:4ef285334a0edc3047e27a04caf00f7742e32c0f03a361101e768014ac5709dd"}, - {file = "boto3-1.34.161.tar.gz", hash = "sha256:a872d8fdb3203c1eb0b12fa9e9d879e6f7fd02983a485f02189e6d5914ccd834"}, + {file = "boto3-1.34.162-py3-none-any.whl", hash = "sha256:d6f6096bdab35a0c0deff469563b87d184a28df7689790f7fe7be98502b7c590"}, + {file = "boto3-1.34.162.tar.gz", hash = "sha256:873f8f5d2f6f85f1018cbb0535b03cceddc7b655b61f66a0a56995238804f41f"}, ] [package.dependencies] -botocore = ">=1.34.161,<1.35.0" +botocore = ">=1.34.162,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -269,13 +269,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.161" +version = "1.34.162" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.161-py3-none-any.whl", hash = "sha256:6c606d2da6f62fde06880aff1190566af208875c29938b6b68741e607817975a"}, - {file = "botocore-1.34.161.tar.gz", hash = "sha256:16381bfb786142099abf170ce734b95a402a3a7f8e4016358712ac333c5568b2"}, + {file = "botocore-1.34.162-py3-none-any.whl", hash = "sha256:2d918b02db88d27a75b48275e6fb2506e9adaaddbec1ffa6a8a0898b34e769be"}, + {file = "botocore-1.34.162.tar.gz", hash = "sha256:adc23be4fb99ad31961236342b7cbf3c0bfc62532cd02852196032e8c0d682f3"}, ] [package.dependencies] @@ -982,7 +982,7 @@ oauth2client = "*" type = "git" url = "https://github.com/uc-cdis/cirrus" reference = "feat/s3-functionality" -resolved_reference = "acda7010a323d158cc7c5034bb37095b2a5a5b20" +resolved_reference = "16112d30342975a614622a3e9f258ad76b94a8d9" [[package]] name = "gen3config" From e8d831b7a51d647947b3b64af7e99ef4672ddeb0 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 15 Aug 2024 16:46:22 -0500 Subject: [PATCH 17/38] fix tests --- fence/blueprints/data/indexd.py | 53 ++++++++++++++++++++------------- tests/conftest.py | 5 +++- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 419bc3bb9..c94079964 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -3,7 +3,7 @@ import json import boto3 from botocore.client import Config -from urllib.parse import urlparse, ParseResult, urlunparse, quote +from urllib.parse import urlparse, ParseResult, urlunparse, quote, urlencode from datetime import datetime, timedelta from sqlalchemy.sql.functions import user @@ -1045,24 +1045,6 @@ def get_bucket_region(self): else: return bucket_cred["region"] - def is_custom(k): - if k == "user_id" or k == "username": - return True - else: - return False - - def client_param_handler(*, params, context, **_kw): - # Store custom parameters in context for later event handlers - context["custom_params"] = {k: v for k, v in params.items() if is_custom(k)} - # Remove custom parameters from client parameters, - # because validation would fail on them - return {k: v for k, v in params.items() if not is_custom(k)} - - def request_param_injector(*, request, **_kw): - if request.context["custom_params"]: - request.url += "&" if "?" in request.url else "?" - request.url += urlencode(request.context["custom_params"]) - def get_signed_url( self, action, @@ -1114,7 +1096,7 @@ def get_signed_url( region = flask.current_app.boto.get_bucket_region( self.parsed_url.netloc, credential ) - + print("making the client") client = boto3.client( "s3", aws_access_key_id=credential["aws_access_key_id"], @@ -1123,16 +1105,45 @@ def get_signed_url( config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"), ) + print("client is done") + + def is_custom(k): + if k == "user_id" or k == "username" or k == "client_id": + return True + else: + return False + + def client_param_handler(*, params, context, **_kw): + # Store custom parameters in context for later event handlers + context["custom_params"] = {k: v for k, v in params.items() if is_custom(k)} + # Remove custom parameters from client parameters, + # because validation would fail on them + return {k: v for k, v in params.items() if not is_custom(k)} + + def request_param_injector(*, request, **_kw): + if request.context["custom_params"]: + request.url += "&" if "?" in request.url else "?" + request.url += urlencode(request.context["custom_params"]) + client.meta.events.register( "provide-client-params.s3.GetObject", client_param_handler ) + + print("first event done") + client.meta.events.register("before-sign.s3.GetObject", request_param_injector) + print("second event done") + cirrus_aws = AwsService(client) auth_info = _get_auth_info_for_id_or_from_request(user=authorized_user) action = ACTION_DICT["s3"][action] + print("This is the auth info: ", auth_info) + + print("this is the part where we get the url") + if action == "PUT": # get presigned url for upload url = cirrus_aws.upload_presigned_url( bucket_name, object_id, expires_in, auth_info @@ -1147,6 +1158,8 @@ def get_signed_url( bucket_name, object_id, expires_in, auth_info ) + print(f"This is the url: {url}") + return url def init_multipart_upload(self, expires_in): diff --git a/tests/conftest.py b/tests/conftest.py index 041994eef..a4b158fa4 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1607,7 +1607,10 @@ def aws_signed_url(): """ def presigned_url_side_effect(*args, **kwargs): - return f"https://{args[0]}/{args[1]}/?X-Amz-Expires={args[2]}" + addional_qs = "" + for k in args[3]: + addional_qs += f"{k}={args[3][k]}&" + return f"https://{args[0]}/{args[1]}/?X-Amz-Expires={args[2]}&{addional_qs}" manager = MagicMock(side_effect=presigned_url_side_effect) From 7ec347687d76712177457481be3e10779b62b5d4 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 15 Aug 2024 17:55:38 -0500 Subject: [PATCH 18/38] clean up and add extra params for requester pays --- fence/blueprints/data/indexd.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index c94079964..13d27fb97 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1096,7 +1096,6 @@ def get_signed_url( region = flask.current_app.boto.get_bucket_region( self.parsed_url.netloc, credential ) - print("making the client") client = boto3.client( "s3", aws_access_key_id=credential["aws_access_key_id"], @@ -1105,10 +1104,10 @@ def get_signed_url( config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"), ) - print("client is done") + custom_params = ["user_id", "username", "client_id", "x-amz-request-payer"] def is_custom(k): - if k == "user_id" or k == "username" or k == "client_id": + if k in custom_params: return True else: return False @@ -1128,22 +1127,13 @@ def request_param_injector(*, request, **_kw): client.meta.events.register( "provide-client-params.s3.GetObject", client_param_handler ) - - print("first event done") - client.meta.events.register("before-sign.s3.GetObject", request_param_injector) - print("second event done") - cirrus_aws = AwsService(client) auth_info = _get_auth_info_for_id_or_from_request(user=authorized_user) action = ACTION_DICT["s3"][action] - print("This is the auth info: ", auth_info) - - print("this is the part where we get the url") - if action == "PUT": # get presigned url for upload url = cirrus_aws.upload_presigned_url( bucket_name, object_id, expires_in, auth_info @@ -1154,12 +1144,13 @@ def request_param_injector(*, request, **_kw): bucket_name, object_id, expires_in, auth_info ) else: + # need to add extra parameter to signing url for header + # https://github.com/boto/boto3/issues/3685 + auth_info["x-amz-request-payer"] = "requester" url = cirrus_aws.download_presigned_url( bucket_name, object_id, expires_in, auth_info ) - print(f"This is the url: {url}") - return url def init_multipart_upload(self, expires_in): From 85234737801c90de73cccd29def733bf2156a70a Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 15 Aug 2024 18:00:19 -0500 Subject: [PATCH 19/38] remove custom parameters for upload presigned url --- fence/blueprints/data/indexd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 13d27fb97..9a1585718 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1136,7 +1136,7 @@ def request_param_injector(*, request, **_kw): if action == "PUT": # get presigned url for upload url = cirrus_aws.upload_presigned_url( - bucket_name, object_id, expires_in, auth_info + bucket_name, object_id, expires_in, {} ) else: # get presigned url for download if bucket.get("requester_pays") == True: From c73f2251b2b4b8b7dc4f06a81d0869c137396539 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 15 Aug 2024 18:11:51 -0500 Subject: [PATCH 20/38] fix requester pays params logic --- fence/blueprints/data/indexd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 9a1585718..c26784da4 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1140,13 +1140,13 @@ def request_param_injector(*, request, **_kw): ) else: # get presigned url for download if bucket.get("requester_pays") == True: + # need to add extra parameter to signing url for header + # https://github.com/boto/boto3/issues/3685 + auth_info["x-amz-request-payer"] = "requester" url = cirrus_aws.requester_pays_download_presigned_url( bucket_name, object_id, expires_in, auth_info ) else: - # need to add extra parameter to signing url for header - # https://github.com/boto/boto3/issues/3685 - auth_info["x-amz-request-payer"] = "requester" url = cirrus_aws.download_presigned_url( bucket_name, object_id, expires_in, auth_info ) From 670ad260642c5158afa54fa88c048abf035eafb5 Mon Sep 17 00:00:00 2001 From: Mingfei Shao <2475897+mfshao@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:30:46 -0500 Subject: [PATCH 21/38] add session token (#1176) * add session token * fix multipart upload * rename var * fix update * fix bucket name * fix --- fence/blueprints/data/indexd.py | 32 +++++++++++++++-------- fence/blueprints/data/multipart_upload.py | 17 ++++-------- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index c26784da4..3db09f958 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -398,7 +398,7 @@ def make_signed_url(self, file_name, protocol=None, expires_in=None, bucket=None @staticmethod def init_multipart_upload(key, expires_in=None, bucket=None): """ - Initilize multipart upload given key + Initialize multipart upload given key Args: key(str): object key @@ -443,7 +443,7 @@ def generate_aws_presigned_url_for_part( Args: key(str): object key of `guid/filename` - uploadID(str): uploadId of the current upload. + uploadId(str): uploadId of the current upload. partNumber(int): the part number Returns: @@ -1096,10 +1096,11 @@ def get_signed_url( region = flask.current_app.boto.get_bucket_region( self.parsed_url.netloc, credential ) - client = boto3.client( + s3client = boto3.client( "s3", aws_access_key_id=credential["aws_access_key_id"], aws_secret_access_key=credential["aws_secret_access_key"], + aws_session_token=credential.get("aws_session_token", None), region_name=region, config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"), ) @@ -1124,12 +1125,20 @@ def request_param_injector(*, request, **_kw): request.url += "&" if "?" in request.url else "?" request.url += urlencode(request.context["custom_params"]) - client.meta.events.register( + s3client.meta.events.register( "provide-client-params.s3.GetObject", client_param_handler ) - client.meta.events.register("before-sign.s3.GetObject", request_param_injector) + s3client.meta.events.register( + "before-sign.s3.GetObject", request_param_injector + ) + s3client.meta.events.register( + "provide-client-params.s3.PutObject", client_param_handler + ) + s3client.meta.events.register( + "before-sign.s3.PutObject", request_param_injector + ) - cirrus_aws = AwsService(client) + cirrus_aws = AwsService(s3client) auth_info = _get_auth_info_for_id_or_from_request(user=authorized_user) action = ACTION_DICT["s3"][action] @@ -1139,7 +1148,7 @@ def request_param_injector(*, request, **_kw): bucket_name, object_id, expires_in, {} ) else: # get presigned url for download - if bucket.get("requester_pays") == True: + if bucket.get("requester_pays") is True: # need to add extra parameter to signing url for header # https://github.com/boto/boto3/issues/3685 auth_info["x-amz-request-payer"] = "requester" @@ -1161,7 +1170,7 @@ def init_multipart_upload(self, expires_in): expires(int): expiration time Returns: - UploadId(str) + uploadId(str) """ aws_creds = get_value( config, "AWS_CREDENTIALS", InternalError("credentials not configured") @@ -1179,18 +1188,19 @@ def generate_presigned_url_for_part_upload(self, uploadId, partNumber, expires_i Generate presigned url for uploading object part given uploadId and part number Args: - uploadId(str): uploadID of the multipart upload + uploadId(str): uploadId of the multipart upload partNumber(int): part number expires(int): expiration time Returns: presigned_url(str) """ + bucket_name = self.bucket_name() aws_creds = get_value( config, "AWS_CREDENTIALS", InternalError("credentials not configured") ) credential = S3IndexedFileLocation.get_credential_to_access_bucket( - self.bucket_name(), aws_creds, expires_in + bucket_name, aws_creds, expires_in ) region = self.get_bucket_region() @@ -1200,7 +1210,7 @@ def generate_presigned_url_for_part_upload(self, uploadId, partNumber, expires_i ) return multipart_upload.generate_presigned_url_for_uploading_part( - self.parsed_url.netloc, + bucket_name, self.parsed_url.path.strip("/"), credential, uploadId, diff --git a/fence/blueprints/data/multipart_upload.py b/fence/blueprints/data/multipart_upload.py index f17f95b9d..a3997e046 100644 --- a/fence/blueprints/data/multipart_upload.py +++ b/fence/blueprints/data/multipart_upload.py @@ -5,6 +5,7 @@ from cdispyutils.config import get_value from cdislogging import get_logger +from gen3cirrus import AwsService from fence.config import config from fence.errors import InternalError @@ -140,27 +141,19 @@ def generate_presigned_url_for_uploading_part( Returns: presigned_url(str) """ - s3_buckets = get_value( - config, "S3_BUCKETS", InternalError("S3_BUCKETS not configured") - ) - bucket = s3_buckets.get(bucket_name) - - s3_buckets = get_value( - config, "S3_BUCKETS", InternalError("S3_BUCKETS not configured") - ) - bucket = s3_buckets.get(bucket_name) - try: s3client = boto3.client( "s3", aws_access_key_id=credentials["aws_access_key_id"], aws_secret_access_key=credentials["aws_secret_access_key"], + aws_session_token=credentials.get("aws_session_token", None), + region_name=region, config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"), ) - cirrus_aws = AwsService(client) + cirrus_aws = AwsService(s3client) presigned_url = cirrus_aws.multipart_upload_presigned_url( - bucket, key, expires, uploadId, partNumber + bucket_name, key, expires, uploadId, partNumber ) return presigned_url From 0b93bad7d076c23fc0e6505fe08308419193b803 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Thu, 22 Aug 2024 14:32:36 -0500 Subject: [PATCH 22/38] refactor logic --- fence/blueprints/data/indexd.py | 35 +------------------- poetry.lock | 58 ++++++++++++++++----------------- 2 files changed, 30 insertions(+), 63 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 3db09f958..f2b9f1ddc 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -3,7 +3,7 @@ import json import boto3 from botocore.client import Config -from urllib.parse import urlparse, ParseResult, urlunparse, quote, urlencode +from urllib.parse import urlparse, ParseResult, urlunparse from datetime import datetime, timedelta from sqlalchemy.sql.functions import user @@ -1105,39 +1105,6 @@ def get_signed_url( config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"), ) - custom_params = ["user_id", "username", "client_id", "x-amz-request-payer"] - - def is_custom(k): - if k in custom_params: - return True - else: - return False - - def client_param_handler(*, params, context, **_kw): - # Store custom parameters in context for later event handlers - context["custom_params"] = {k: v for k, v in params.items() if is_custom(k)} - # Remove custom parameters from client parameters, - # because validation would fail on them - return {k: v for k, v in params.items() if not is_custom(k)} - - def request_param_injector(*, request, **_kw): - if request.context["custom_params"]: - request.url += "&" if "?" in request.url else "?" - request.url += urlencode(request.context["custom_params"]) - - s3client.meta.events.register( - "provide-client-params.s3.GetObject", client_param_handler - ) - s3client.meta.events.register( - "before-sign.s3.GetObject", request_param_injector - ) - s3client.meta.events.register( - "provide-client-params.s3.PutObject", client_param_handler - ) - s3client.meta.events.register( - "before-sign.s3.PutObject", request_param_injector - ) - cirrus_aws = AwsService(s3client) auth_info = _get_auth_info_for_id_or_from_request(user=authorized_user) diff --git a/poetry.lock b/poetry.lock index db991a3bf..ff19f807f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "addict" @@ -250,17 +250,17 @@ files = [ [[package]] name = "boto3" -version = "1.34.162" +version = "1.35.3" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.162-py3-none-any.whl", hash = "sha256:d6f6096bdab35a0c0deff469563b87d184a28df7689790f7fe7be98502b7c590"}, - {file = "boto3-1.34.162.tar.gz", hash = "sha256:873f8f5d2f6f85f1018cbb0535b03cceddc7b655b61f66a0a56995238804f41f"}, + {file = "boto3-1.35.3-py3-none-any.whl", hash = "sha256:e24f9b8a4dc85bfcf4d3e9183589dba6ed2964dfa1233ca578fc7c04db1c68b4"}, + {file = "boto3-1.35.3.tar.gz", hash = "sha256:9c2799cfeba2c68d91f840b6d018ffe76f5f27514dcbb166b20d9e3a704cbcea"}, ] [package.dependencies] -botocore = ">=1.34.162,<1.35.0" +botocore = ">=1.35.3,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -269,13 +269,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.162" +version = "1.35.3" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.162-py3-none-any.whl", hash = "sha256:2d918b02db88d27a75b48275e6fb2506e9adaaddbec1ffa6a8a0898b34e769be"}, - {file = "botocore-1.34.162.tar.gz", hash = "sha256:adc23be4fb99ad31961236342b7cbf3c0bfc62532cd02852196032e8c0d682f3"}, + {file = "botocore-1.35.3-py3-none-any.whl", hash = "sha256:3ff54075e125304a8978e5d3f27ab96485f2deedb561d458b0567a0a921dc243"}, + {file = "botocore-1.35.3.tar.gz", hash = "sha256:ff0c3189c0aa588c3aeda3f3e5e37925d64deaac6748310124307f978933e768"}, ] [package.dependencies] @@ -313,13 +313,13 @@ files = [ [[package]] name = "cachetools" -version = "5.4.0" +version = "5.5.0" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.4.0-py3-none-any.whl", hash = "sha256:3ae3b49a3d5e28a77a0be2b37dbcb89005058959cb2323858c2657c4a8cab474"}, - {file = "cachetools-5.4.0.tar.gz", hash = "sha256:b8adc2e7c07f105ced7bc56dbb6dfbe7c4a00acce20e2227b3f355be89bc6827"}, + {file = "cachetools-5.5.0-py3-none-any.whl", hash = "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292"}, + {file = "cachetools-5.5.0.tar.gz", hash = "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a"}, ] [[package]] @@ -961,7 +961,7 @@ six = ">=1.16.0,<2.0.0" [[package]] name = "gen3cirrus" -version = "3.0.1" +version = "3.1.0" description = "" optional = false python-versions = "^3.9" @@ -982,7 +982,7 @@ oauth2client = "*" type = "git" url = "https://github.com/uc-cdis/cirrus" reference = "feat/s3-functionality" -resolved_reference = "16112d30342975a614622a3e9f258ad76b94a8d9" +resolved_reference = "e6fb45b79d30d37148d68785b45b436dd2e53fb3" [[package]] name = "gen3config" @@ -1043,13 +1043,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-api-python-client" -version = "2.141.0" +version = "2.142.0" description = "Google API Client Library for Python" optional = false python-versions = ">=3.7" files = [ - {file = "google_api_python_client-2.141.0-py2.py3-none-any.whl", hash = "sha256:43c05322b91791204465291b3852718fae38d4f84b411d8be847c4f86882652a"}, - {file = "google_api_python_client-2.141.0.tar.gz", hash = "sha256:0f225b1f45d5a6f8c2a400f48729f5d6da9a81138e81e0478d61fdd8edf6563a"}, + {file = "google_api_python_client-2.142.0-py2.py3-none-any.whl", hash = "sha256:266799082bb8301f423ec204dffbffb470b502abbf29efd1f83e644d36eb5a8f"}, + {file = "google_api_python_client-2.142.0.tar.gz", hash = "sha256:a1101ac9e24356557ca22f07ff48b7f61fa5d4b4e7feeef3bda16e5dcb86350e"}, ] [package.dependencies] @@ -1061,13 +1061,13 @@ uritemplate = ">=3.0.1,<5" [[package]] name = "google-auth" -version = "2.33.0" +version = "2.34.0" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google_auth-2.33.0-py2.py3-none-any.whl", hash = "sha256:8eff47d0d4a34ab6265c50a106a3362de6a9975bb08998700e389f857e4d39df"}, - {file = "google_auth-2.33.0.tar.gz", hash = "sha256:d6a52342160d7290e334b4d47ba390767e4438ad0d45b7630774533e82655b95"}, + {file = "google_auth-2.34.0-py2.py3-none-any.whl", hash = "sha256:72fd4733b80b6d777dcde515628a9eb4a577339437012874ea286bca7261ee65"}, + {file = "google_auth-2.34.0.tar.gz", hash = "sha256:8eb87396435c19b20d32abd2f984e31c191a15284af72eb922f10e5bde9c04cc"}, ] [package.dependencies] @@ -1077,7 +1077,7 @@ rsa = ">=3.1.4,<5" [package.extras] aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] -enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +enterprise-cert = ["cryptography", "pyopenssl"] pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] reauth = ["pyu2f (>=0.1.5)"] requests = ["requests (>=2.20.0,<3.0.0.dev0)"] @@ -1407,13 +1407,13 @@ files = [ [[package]] name = "importlib-metadata" -version = "8.2.0" +version = "8.4.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_metadata-8.2.0-py3-none-any.whl", hash = "sha256:11901fa0c2f97919b288679932bb64febaeacf289d18ac84dd68cb2e74213369"}, - {file = "importlib_metadata-8.2.0.tar.gz", hash = "sha256:72e8d4399996132204f9a16dcc751af254a48f8d1b20b9ff0f98d4a8f901e73d"}, + {file = "importlib_metadata-8.4.0-py3-none-any.whl", hash = "sha256:66f342cc6ac9818fc6ff340576acd24d65ba0b3efabb2b4ac08b598965a4a2f1"}, + {file = "importlib_metadata-8.4.0.tar.gz", hash = "sha256:9a547d3bc3608b025f93d403fdd1aae741c24fbb8314df4b155675742ce303c5"}, ] [package.dependencies] @@ -1535,13 +1535,13 @@ testing = ["pytest"] [[package]] name = "markdown" -version = "3.6" +version = "3.7" description = "Python implementation of John Gruber's Markdown." optional = false python-versions = ">=3.8" files = [ - {file = "Markdown-3.6-py3-none-any.whl", hash = "sha256:48f276f4d8cfb8ce6527c8f79e2ee29708508bf4d40aa410fbc3b4ee832c850f"}, - {file = "Markdown-3.6.tar.gz", hash = "sha256:ed4f41f6daecbeeb96e576ce414c41d2d876daa9a16cb35fa8ed8c2ddfad0224"}, + {file = "Markdown-3.7-py3-none-any.whl", hash = "sha256:7eb6df5690b81a1d7942992c97fad2938e956e79df20cbc6186e9c3a77b1c803"}, + {file = "markdown-3.7.tar.gz", hash = "sha256:2ae2471477cfd02dbbf038d5d9bc226d40def84b4fe2986e49b59b6b472bbed2"}, ] [package.dependencies] @@ -2447,13 +2447,13 @@ sqlalchemy = ">=1.3.3" [[package]] name = "werkzeug" -version = "3.0.3" +version = "3.0.4" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.8" files = [ - {file = "werkzeug-3.0.3-py3-none-any.whl", hash = "sha256:fc9645dc43e03e4d630d23143a04a7f947a9a3b5727cd535fdfe155a17cc48c8"}, - {file = "werkzeug-3.0.3.tar.gz", hash = "sha256:097e5bfda9f0aba8da6b8545146def481d06aa7d3266e7448e2cccf67dd8bd18"}, + {file = "werkzeug-3.0.4-py3-none-any.whl", hash = "sha256:02c9eb92b7d6c06f31a782811505d2157837cea66aaede3e217c7c27c039476c"}, + {file = "werkzeug-3.0.4.tar.gz", hash = "sha256:34f2371506b250df4d4f84bfe7b0921e4762525762bbd936614909fe25cd7306"}, ] [package.dependencies] From 49618b260d6d8021b77ffbb6c100c571c1e7485d Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Thu, 22 Aug 2024 14:43:43 -0500 Subject: [PATCH 23/38] update lock --- .secrets.baseline | 6 +++--- poetry.lock | 2 +- tests/conftest.py | 5 +---- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index b70519cdc..82a2c6b2a 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -268,14 +268,14 @@ "filename": "tests/conftest.py", "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9", "is_verified": false, - "line_number": 1573 + "line_number": 1570 }, { "type": "Base64 High Entropy String", "filename": "tests/conftest.py", "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108", "is_verified": false, - "line_number": 1597 + "line_number": 1594 } ], "tests/credentials/google/test_credentials.py": [ @@ -422,5 +422,5 @@ } ] }, - "generated_at": "2024-08-13T16:48:45Z" + "generated_at": "2024-08-22T19:43:39Z" } diff --git a/poetry.lock b/poetry.lock index ff19f807f..0c77cc3d1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -982,7 +982,7 @@ oauth2client = "*" type = "git" url = "https://github.com/uc-cdis/cirrus" reference = "feat/s3-functionality" -resolved_reference = "e6fb45b79d30d37148d68785b45b436dd2e53fb3" +resolved_reference = "6aaf512f7ed15c1132cbd5f6be079e71778234ab" [[package]] name = "gen3config" diff --git a/tests/conftest.py b/tests/conftest.py index a4b158fa4..4bd97b518 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,10 +33,7 @@ import pytest import requests from sqlalchemy.ext.compiler import compiles -import boto3 -import botocore -from gen3cirrus import AwsService -from botocore.stub import Stubber + # Set FENCE_CONFIG_PATH *before* loading the configuration CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) From 0a6537d4c9a3e8bf5497d2cb53fac1dcc1553f9c Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 26 Aug 2024 11:27:30 -0500 Subject: [PATCH 24/38] address pr comments --- fence/blueprints/data/indexd.py | 10 +++++----- fence/blueprints/data/multipart_upload.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index f2b9f1ddc..f3eb24474 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1110,11 +1110,11 @@ def get_signed_url( action = ACTION_DICT["s3"][action] - if action == "PUT": # get presigned url for upload - url = cirrus_aws.upload_presigned_url( - bucket_name, object_id, expires_in, {} - ) - else: # get presigned url for download + # get presigned url for upload + if action == "PUT": + url = cirrus_aws.upload_presigned_url(bucket_name, object_id, expires_in) + # get presigned url for download + else: if bucket.get("requester_pays") is True: # need to add extra parameter to signing url for header # https://github.com/boto/boto3/issues/3685 diff --git a/fence/blueprints/data/multipart_upload.py b/fence/blueprints/data/multipart_upload.py index a3997e046..fd271317b 100644 --- a/fence/blueprints/data/multipart_upload.py +++ b/fence/blueprints/data/multipart_upload.py @@ -59,7 +59,7 @@ def initialize_multipart_upload(bucket_name, key, credentials): key, error ) ) - raise InternalError("Can not initilize multipart upload for {}".format(key)) + raise InternalError("Can not initialize multipart upload for {}".format(key)) return multipart_upload.get("UploadId") From 913dbc780389fa45d02700596546e3abf8019f85 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 26 Aug 2024 11:36:18 -0500 Subject: [PATCH 25/38] update --- tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4bd97b518..11c8ccc64 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1604,10 +1604,10 @@ def aws_signed_url(): """ def presigned_url_side_effect(*args, **kwargs): - addional_qs = "" + additional_qs = "" for k in args[3]: - addional_qs += f"{k}={args[3][k]}&" - return f"https://{args[0]}/{args[1]}/?X-Amz-Expires={args[2]}&{addional_qs}" + additional_qs += f"{k}={args[3][k]}&" + return f"https://{args[0]}/{args[1]}/?X-Amz-Expires={args[2]}&{additional_qs}" manager = MagicMock(side_effect=presigned_url_side_effect) From c7cd09df623084dca047a8512a94493fdad98d7b Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 26 Aug 2024 11:41:14 -0500 Subject: [PATCH 26/38] update lock --- poetry.lock | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0c77cc3d1..45cab5843 100644 --- a/poetry.lock +++ b/poetry.lock @@ -97,13 +97,13 @@ tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] [[package]] name = "authlib" -version = "1.3.1" +version = "1.3.2" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = false python-versions = ">=3.8" files = [ - {file = "Authlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:d35800b973099bbadc49b42b256ecb80041ad56b7fe1216a362c7943c088f377"}, - {file = "authlib-1.3.1.tar.gz", hash = "sha256:7ae843f03c06c5c0debd63c9db91f9fda64fa62a42a77419fa15fbb7e7a58917"}, + {file = "Authlib-1.3.2-py2.py3-none-any.whl", hash = "sha256:ede026a95e9f5cdc2d4364a52103f5405e75aa156357e831ef2bfd0bc5094dfc"}, + {file = "authlib-1.3.2.tar.gz", hash = "sha256:4b16130117f9eb82aa6eec97f6dd4673c3f960ac0283ccdae2897ee4bc030ba2"}, ] [package.dependencies] @@ -250,17 +250,17 @@ files = [ [[package]] name = "boto3" -version = "1.35.3" +version = "1.35.5" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.35.3-py3-none-any.whl", hash = "sha256:e24f9b8a4dc85bfcf4d3e9183589dba6ed2964dfa1233ca578fc7c04db1c68b4"}, - {file = "boto3-1.35.3.tar.gz", hash = "sha256:9c2799cfeba2c68d91f840b6d018ffe76f5f27514dcbb166b20d9e3a704cbcea"}, + {file = "boto3-1.35.5-py3-none-any.whl", hash = "sha256:2cef3aa476181395c260f4b6e6c5565e5a3022a874fb6b579d8e6b169f94e0b3"}, + {file = "boto3-1.35.5.tar.gz", hash = "sha256:5724ddeda8e18c7614c20a09c20159ed87ff7439755cf5e250a1a3feaf9afb7e"}, ] [package.dependencies] -botocore = ">=1.35.3,<1.36.0" +botocore = ">=1.35.5,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -269,13 +269,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.35.3" +version = "1.35.5" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.35.3-py3-none-any.whl", hash = "sha256:3ff54075e125304a8978e5d3f27ab96485f2deedb561d458b0567a0a921dc243"}, - {file = "botocore-1.35.3.tar.gz", hash = "sha256:ff0c3189c0aa588c3aeda3f3e5e37925d64deaac6748310124307f978933e768"}, + {file = "botocore-1.35.5-py3-none-any.whl", hash = "sha256:8116b72c7ae845c195146e437e2afd9d17538a37b3f3548dcf67c12c86ba0742"}, + {file = "botocore-1.35.5.tar.gz", hash = "sha256:3a0086c7124cb3b0d9f98563d00ffd14a942c3f9e731d8d1ccf0d3a1ac7ed884"}, ] [package.dependencies] @@ -1238,13 +1238,13 @@ requests = ["requests (>=2.18.0,<3.0.0dev)"] [[package]] name = "googleapis-common-protos" -version = "1.63.2" +version = "1.64.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.63.2.tar.gz", hash = "sha256:27c5abdffc4911f28101e635de1533fb4cfd2c37fbaa9174587c799fac90aa87"}, - {file = "googleapis_common_protos-1.63.2-py2.py3-none-any.whl", hash = "sha256:27a2499c7e8aff199665b22741997e485eccc8645aa9176c7c988e6fae507945"}, + {file = "googleapis_common_protos-1.64.0-py2.py3-none-any.whl", hash = "sha256:d1bfc569f70ed2e96ccf06ead265c2cf42b5abfc817cda392e3835f3b67b5c59"}, + {file = "googleapis_common_protos-1.64.0.tar.gz", hash = "sha256:7d77ca6b7c0c38eb6b1bab3b4c9973acf57ce4f2a6d3a4136acba10bcbfb3025"}, ] [package.dependencies] @@ -1396,13 +1396,13 @@ socks = ["socksio (==1.*)"] [[package]] name = "idna" -version = "3.7" +version = "3.8" description = "Internationalized Domain Names in Applications (IDNA)" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" files = [ - {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, - {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, + {file = "idna-3.8-py3-none-any.whl", hash = "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac"}, + {file = "idna-3.8.tar.gz", hash = "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"}, ] [[package]] @@ -1975,13 +1975,13 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] [[package]] name = "pyparsing" -version = "3.1.2" +version = "3.1.4" description = "pyparsing module - Classes and methods to define and execute parsing grammars" optional = false python-versions = ">=3.6.8" files = [ - {file = "pyparsing-3.1.2-py3-none-any.whl", hash = "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742"}, - {file = "pyparsing-3.1.2.tar.gz", hash = "sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad"}, + {file = "pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c"}, + {file = "pyparsing-3.1.4.tar.gz", hash = "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032"}, ] [package.extras] From 2f54f152881fb93abff4d896b2a90accf5ceb723 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 26 Aug 2024 12:00:46 -0500 Subject: [PATCH 27/38] test --- fence/blueprints/data/indexd.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index f3eb24474..81abfee98 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1112,7 +1112,9 @@ def get_signed_url( # get presigned url for upload if action == "PUT": - url = cirrus_aws.upload_presigned_url(bucket_name, object_id, expires_in) + url = cirrus_aws.upload_presigned_url( + bucket_name, object_id, expires_in, None + ) # get presigned url for download else: if bucket.get("requester_pays") is True: From 3d8b6fdf21b7863b7bae744abd6385aa211de74d Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 26 Aug 2024 12:56:26 -0500 Subject: [PATCH 28/38] fix --- fence/blueprints/data/indexd.py | 4 +--- tests/conftest.py | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 81abfee98..f3eb24474 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1112,9 +1112,7 @@ def get_signed_url( # get presigned url for upload if action == "PUT": - url = cirrus_aws.upload_presigned_url( - bucket_name, object_id, expires_in, None - ) + url = cirrus_aws.upload_presigned_url(bucket_name, object_id, expires_in) # get presigned url for download else: if bucket.get("requester_pays") is True: diff --git a/tests/conftest.py b/tests/conftest.py index 11c8ccc64..9baba01a1 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1605,8 +1605,9 @@ def aws_signed_url(): def presigned_url_side_effect(*args, **kwargs): additional_qs = "" - for k in args[3]: - additional_qs += f"{k}={args[3][k]}&" + if args[3] and isinstance(args[3], dict): + for k in args[3]: + additional_qs += f"{k}={args[3][k]}&" return f"https://{args[0]}/{args[1]}/?X-Amz-Expires={args[2]}&{additional_qs}" manager = MagicMock(side_effect=presigned_url_side_effect) From 7d3b980f64f1bc62e6c1c73ebb3ce842415f8243 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 26 Aug 2024 13:30:48 -0500 Subject: [PATCH 29/38] test --- fence/blueprints/data/indexd.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index f3eb24474..81abfee98 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1112,7 +1112,9 @@ def get_signed_url( # get presigned url for upload if action == "PUT": - url = cirrus_aws.upload_presigned_url(bucket_name, object_id, expires_in) + url = cirrus_aws.upload_presigned_url( + bucket_name, object_id, expires_in, None + ) # get presigned url for download else: if bucket.get("requester_pays") is True: From f02631aeeea71adf8f5a53c47a73da372dc6889c Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 26 Aug 2024 13:53:35 -0500 Subject: [PATCH 30/38] restore --- fence/blueprints/data/indexd.py | 2 +- tests/conftest.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 81abfee98..b8dc1faba 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1113,7 +1113,7 @@ def get_signed_url( # get presigned url for upload if action == "PUT": url = cirrus_aws.upload_presigned_url( - bucket_name, object_id, expires_in, None + bucket_name, object_id, expires_in, {} ) # get presigned url for download else: diff --git a/tests/conftest.py b/tests/conftest.py index 9baba01a1..8f9d52b2d 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1605,9 +1605,9 @@ def aws_signed_url(): def presigned_url_side_effect(*args, **kwargs): additional_qs = "" - if args[3] and isinstance(args[3], dict): - for k in args[3]: - additional_qs += f"{k}={args[3][k]}&" + # if args[3] and isinstance(args[3], dict): + for k in args[3]: + additional_qs += f"{k}={args[3][k]}&" return f"https://{args[0]}/{args[1]}/?X-Amz-Expires={args[2]}&{additional_qs}" manager = MagicMock(side_effect=presigned_url_side_effect) From edf2fa64a5aed793dba3eb171731ae1ac9641277 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 26 Aug 2024 14:03:37 -0500 Subject: [PATCH 31/38] try authlib version --- fence/blueprints/data/indexd.py | 2 +- poetry.lock | 8 ++++---- pyproject.toml | 2 +- tests/conftest.py | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index b8dc1faba..81abfee98 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1113,7 +1113,7 @@ def get_signed_url( # get presigned url for upload if action == "PUT": url = cirrus_aws.upload_presigned_url( - bucket_name, object_id, expires_in, {} + bucket_name, object_id, expires_in, None ) # get presigned url for download else: diff --git a/poetry.lock b/poetry.lock index 45cab5843..d13aebbfa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -97,13 +97,13 @@ tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] [[package]] name = "authlib" -version = "1.3.2" +version = "1.3.1" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = false python-versions = ">=3.8" files = [ - {file = "Authlib-1.3.2-py2.py3-none-any.whl", hash = "sha256:ede026a95e9f5cdc2d4364a52103f5405e75aa156357e831ef2bfd0bc5094dfc"}, - {file = "authlib-1.3.2.tar.gz", hash = "sha256:4b16130117f9eb82aa6eec97f6dd4673c3f960ac0283ccdae2897ee4bc030ba2"}, + {file = "Authlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:d35800b973099bbadc49b42b256ecb80041ad56b7fe1216a362c7943c088f377"}, + {file = "authlib-1.3.1.tar.gz", hash = "sha256:7ae843f03c06c5c0debd63c9db91f9fda64fa62a42a77419fa15fbb7e7a58917"}, ] [package.dependencies] @@ -2587,4 +2587,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0.0" -content-hash = "597d2a7ac8980713561f7d16c82f045dcb4f8997da0cfba73d2299d791c9b58e" +content-hash = "f0e269197267ddea6b56b445fc18eec5ceae0fde0c54e30ad25f5c5894596af0" diff --git a/pyproject.toml b/pyproject.toml index 2a157f090..77474d5e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ include = [ [tool.poetry.dependencies] python = ">=3.9,<4.0.0" alembic = "^1.7.7" -authlib = "*" #let authutils decide which version to use +authlib = "1.3.1" #let authutils decide which version to use authutils = "^6.2.3" bcrypt = "^3.1.4" boto3 = "*" diff --git a/tests/conftest.py b/tests/conftest.py index 8f9d52b2d..9baba01a1 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1605,9 +1605,9 @@ def aws_signed_url(): def presigned_url_side_effect(*args, **kwargs): additional_qs = "" - # if args[3] and isinstance(args[3], dict): - for k in args[3]: - additional_qs += f"{k}={args[3][k]}&" + if args[3] and isinstance(args[3], dict): + for k in args[3]: + additional_qs += f"{k}={args[3][k]}&" return f"https://{args[0]}/{args[1]}/?X-Amz-Expires={args[2]}&{additional_qs}" manager = MagicMock(side_effect=presigned_url_side_effect) From bfa47a5e8303f2072921c3a3a2caa946f3da38e6 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 26 Aug 2024 15:30:36 -0500 Subject: [PATCH 32/38] update comment --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 77474d5e6..01de9d1a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ include = [ [tool.poetry.dependencies] python = ">=3.9,<4.0.0" alembic = "^1.7.7" -authlib = "1.3.1" #let authutils decide which version to use +authlib = "1.3.1" # have to pin it to this version, 1.3.2 will cause some unit tests to fail authutils = "^6.2.3" bcrypt = "^3.1.4" boto3 = "*" From 612690b9c4bbe7fc864d9c74942c6bd2abeb8097 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 26 Aug 2024 16:02:42 -0500 Subject: [PATCH 33/38] update lock --- poetry.lock | 42 +++++++++++++++++++++--------------------- pyproject.toml | 2 +- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/poetry.lock b/poetry.lock index d13aebbfa..03532ffed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -250,17 +250,17 @@ files = [ [[package]] name = "boto3" -version = "1.35.5" +version = "1.35.6" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.35.5-py3-none-any.whl", hash = "sha256:2cef3aa476181395c260f4b6e6c5565e5a3022a874fb6b579d8e6b169f94e0b3"}, - {file = "boto3-1.35.5.tar.gz", hash = "sha256:5724ddeda8e18c7614c20a09c20159ed87ff7439755cf5e250a1a3feaf9afb7e"}, + {file = "boto3-1.35.6-py3-none-any.whl", hash = "sha256:c35c560ef0cb0f133b6104bc374d60eeb7cb69c1d5d7907e4305a285d162bef0"}, + {file = "boto3-1.35.6.tar.gz", hash = "sha256:b41deed9ca7e0a619510a22e256e3e38b5f532624b4aff8964a1e870877b37bc"}, ] [package.dependencies] -botocore = ">=1.35.5,<1.36.0" +botocore = ">=1.35.6,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -269,13 +269,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.35.5" +version = "1.35.6" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.35.5-py3-none-any.whl", hash = "sha256:8116b72c7ae845c195146e437e2afd9d17538a37b3f3548dcf67c12c86ba0742"}, - {file = "botocore-1.35.5.tar.gz", hash = "sha256:3a0086c7124cb3b0d9f98563d00ffd14a942c3f9e731d8d1ccf0d3a1ac7ed884"}, + {file = "botocore-1.35.6-py3-none-any.whl", hash = "sha256:8378c6cfef2dee15eb7b3ebbb55ba9c1de959f231292039b81eb35b72c50ad59"}, + {file = "botocore-1.35.6.tar.gz", hash = "sha256:93ef31b80b05758db4dd67e010348a05b9ff43f82839629b7ac334f2a454996e"}, ] [package.dependencies] @@ -964,9 +964,11 @@ name = "gen3cirrus" version = "3.1.0" description = "" optional = false -python-versions = "^3.9" -files = [] -develop = false +python-versions = "<4.0,>=3.9" +files = [ + {file = "gen3cirrus-3.1.0-py3-none-any.whl", hash = "sha256:42c89d1579d7d89c87c5c355815197e1dbf8045a2030310a6b2f6ca089a74fde"}, + {file = "gen3cirrus-3.1.0.tar.gz", hash = "sha256:81e5a0a4b5dc2d820ad3351bd5326151424806bc8f64e022100320f01027a310"}, +] [package.dependencies] backoff = "*" @@ -978,12 +980,6 @@ google-auth-httplib2 = "*" google-cloud-storage = "*" oauth2client = "*" -[package.source] -type = "git" -url = "https://github.com/uc-cdis/cirrus" -reference = "feat/s3-functionality" -resolved_reference = "6aaf512f7ed15c1132cbd5f6be079e71778234ab" - [[package]] name = "gen3config" version = "1.1.0" @@ -2571,20 +2567,24 @@ files = [ [[package]] name = "zipp" -version = "3.20.0" +version = "3.20.1" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.20.0-py3-none-any.whl", hash = "sha256:58da6168be89f0be59beb194da1250516fdaa062ccebd30127ac65d30045e10d"}, - {file = "zipp-3.20.0.tar.gz", hash = "sha256:0145e43d89664cfe1a2e533adc75adafed82fe2da404b4bbb6b026c0157bdb31"}, + {file = "zipp-3.20.1-py3-none-any.whl", hash = "sha256:9960cd8967c8f85a56f920d5d507274e74f9ff813a0ab8889a5b5be2daf44064"}, + {file = "zipp-3.20.1.tar.gz", hash = "sha256:c22b14cc4763c5a5b04134207736c107db42e9d3ef2d9779d465f5f1bcba572b"}, ] [package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0.0" -content-hash = "f0e269197267ddea6b56b445fc18eec5ceae0fde0c54e30ad25f5c5894596af0" +content-hash = "3742c22936a9218570cb5439de91afe6da3dffe24784c97f1a869acc73eb488c" diff --git a/pyproject.toml b/pyproject.toml index 01de9d1a5..47e22a478 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ flask-cors = ">=3.0.3" flask-restful = ">=0.3.8" email_validator = "^1.1.1" gen3authz = "^1.5.1" -gen3cirrus = {git = "https://github.com/uc-cdis/cirrus", rev = "feat/s3-functionality"} +gen3cirrus = "^3.1.0" gen3config = ">=1.1.0" gen3users = "^1.0.2" idna = "^3.7" From db11006b7fd62c736e5c966d424710b41018b28a Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Tue, 27 Aug 2024 10:05:09 -0500 Subject: [PATCH 34/38] update version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 47e22a478..277436931 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "fence" -version = "10.2.0" +version = "10.3.0" description = "Gen3 AuthN/AuthZ OIDC Service" authors = ["CTDS UChicago "] license = "Apache-2.0" From d82205c0dc1bfb933e40e1c1a24697140381e62a Mon Sep 17 00:00:00 2001 From: Mingfei Shao <2475897+mfshao@users.noreply.github.com> Date: Tue, 27 Aug 2024 10:58:27 -0500 Subject: [PATCH 35/38] Update pyproject.toml Co-authored-by: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 277436931..1aec100c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ include = [ [tool.poetry.dependencies] python = ">=3.9,<4.0.0" alembic = "^1.7.7" -authlib = "1.3.1" # have to pin it to this version, 1.3.2 will cause some unit tests to fail +authlib = "<1.3.2" # let authutils decide which version to use, but 1.3.2 will cause some unit tests to fail authutils = "^6.2.3" bcrypt = "^3.1.4" boto3 = "*" From 8bc9a67c3bd1e8f4123ede76a9d90d64bef9d3b3 Mon Sep 17 00:00:00 2001 From: Mingfei Shao <2475897+mfshao@users.noreply.github.com> Date: Tue, 27 Aug 2024 10:58:34 -0500 Subject: [PATCH 36/38] Update pyproject.toml Co-authored-by: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1aec100c4..9086fa0ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ markupsafe = "^2.0.1" paramiko = ">=2.6.0" prometheus-client = "<1" -psycopg2 = "*" +psycopg2 = "<3" PyJWT = "^2.4.0" python_dateutil = "^2.6.1" python-jose = "^2.0.2" From b454062b8ddcf7f1edfacdb08b6972dcec0c2a4f Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Tue, 27 Aug 2024 10:59:27 -0500 Subject: [PATCH 37/38] redo lock --- poetry.lock | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 03532ffed..a91239b85 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1368,13 +1368,13 @@ pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0 [[package]] name = "httpx" -version = "0.27.0" +version = "0.27.2" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpx-0.27.0-py3-none-any.whl", hash = "sha256:71d5465162c13681bff01ad59b2cc68dd838ea1f10e51574bac27103f00c91a5"}, - {file = "httpx-0.27.0.tar.gz", hash = "sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5"}, + {file = "httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0"}, + {file = "httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"}, ] [package.dependencies] @@ -1389,6 +1389,7 @@ brotli = ["brotli", "brotlicffi"] cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] [[package]] name = "idna" @@ -1721,13 +1722,13 @@ invoke = ["invoke (>=2.0)"] [[package]] name = "pbr" -version = "6.0.0" +version = "6.1.0" description = "Python Build Reasonableness" optional = false python-versions = ">=2.6" files = [ - {file = "pbr-6.0.0-py2.py3-none-any.whl", hash = "sha256:4a7317d5e3b17a3dccb6a8cfe67dab65b20551404c52c8ed41279fa4f0cb4cda"}, - {file = "pbr-6.0.0.tar.gz", hash = "sha256:d1377122a5a00e2f940ee482999518efe16d745d423a670c27773dfbc3c9a7d9"}, + {file = "pbr-6.1.0-py2.py3-none-any.whl", hash = "sha256:a776ae228892d8013649c0aeccbb3d5f99ee15e005a4cbb7e61d55a067b28a2a"}, + {file = "pbr-6.1.0.tar.gz", hash = "sha256:788183e382e3d1d7707db08978239965e8b9e4e5ed42669bf4758186734d5f24"}, ] [[package]] @@ -2587,4 +2588,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0.0" -content-hash = "3742c22936a9218570cb5439de91afe6da3dffe24784c97f1a869acc73eb488c" +content-hash = "8c9ff3ba536d69f449fb04dab58808da7996c3449472c751f2b6da24394d7305" From 240fb48d7ac1b26b3df33a4bb95c5de58165af67 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Tue, 27 Aug 2024 11:01:26 -0500 Subject: [PATCH 38/38] add to default config --- fence/config-default.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fence/config-default.yaml b/fence/config-default.yaml index 5e43e21dc..a570989c0 100755 --- a/fence/config-default.yaml +++ b/fence/config-default.yaml @@ -676,6 +676,10 @@ S3_BUCKETS: {} # cred: 'CRED1' # region: 'us-east-1' # role-arn: 'arn:aws:iam::role1' +# bucket5: +# cred: 'CRED3' +# region: 'us-east-1' +# requester_pays: true # to indicate this is a requester pay enabled S3 bucket GS_BUCKETS: {} # NOTE: Remove the {} and supply buckets if needed. Example in comments below # bucket1: