From 82c1a6576c1b5955fe04db51266752d932fd0aaf Mon Sep 17 00:00:00 2001 From: xieydd Date: Sat, 2 Nov 2024 16:52:22 +0800 Subject: [PATCH] ci: add pg slim, cnpg Dockerfile and add ci Signed-off-by: xieydd --- .github/workflows/release_enterprise.yml | 8 +- .github/workflows/release_pg_slim.yml | 36 ++ .github/workflows/trunk-install-test.yml | 46 +++ docker/enterprise.Dockerfile | 14 - docker/pg-cnpg/Dockerfile | 180 +++++++++ docker/pg-cnpg/requirements.txt | 464 +++++++++++++++++++++++ docker/pg-cnpg/trunk-install.sh | 80 ++++ docker/pg-slim/Dockerfile | 138 +++++++ docker/pg-slim/docker-ensure-initdb.sh | 71 ++++ docker/pg-slim/docker-entrypoint.sh | 356 +++++++++++++++++ 10 files changed, 1376 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/release_pg_slim.yml create mode 100644 .github/workflows/trunk-install-test.yml delete mode 100644 docker/enterprise.Dockerfile create mode 100644 docker/pg-cnpg/Dockerfile create mode 100644 docker/pg-cnpg/requirements.txt create mode 100755 docker/pg-cnpg/trunk-install.sh create mode 100644 docker/pg-slim/Dockerfile create mode 100755 docker/pg-slim/docker-ensure-initdb.sh create mode 100755 docker/pg-slim/docker-entrypoint.sh diff --git a/.github/workflows/release_enterprise.yml b/.github/workflows/release_enterprise.yml index 642087d08..ce6ae2371 100644 --- a/.github/workflows/release_enterprise.yml +++ b/.github/workflows/release_enterprise.yml @@ -146,6 +146,7 @@ jobs: matrix: version: [14, 15, 16, 17] schema: ["vectors", "extensions", "public"] + platform: ["amd64", "arm64"] steps: - name: Checkout uses: actions/checkout@v4 @@ -169,10 +170,11 @@ jobs: with: context: . push: true - platforms: "linux/amd64,linux/arm64" - file: ./docker/enterprise.Dockerfile + platforms: "linux/${{ matrix.platform }}" + file: ./docker/pg-cnpg/Dockerfile build-args: | + PG_MAJOR=${{ matrix.version }} FROM_TAG=pg${{ matrix.version }}-v${{ github.event.inputs.version }} - CNPG_TAG=${{ matrix.version }} SCHEMA=${{ matrix.schema }} + TARGETARCH=${{ matrix.platform }} tags: ${{ steps.variables.outputs.tags }} diff --git a/.github/workflows/release_pg_slim.yml b/.github/workflows/release_pg_slim.yml new file mode 100644 index 000000000..ca4d243de --- /dev/null +++ b/.github/workflows/release_pg_slim.yml @@ -0,0 +1,36 @@ +name: Release for Postgres slim + +on: + workflow_dispatch: + +jobs: + pg-slim: + strategy: + matrix: + version: [14, 15, 16, 17] + platforms: "amd64,arm64" + runs-on: ubuntu-latest + env: + PG_MAJOR: ${{ matrix.version }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERIO_MODELZ_USERNAME }} + password: ${{ secrets.DOCKERIO_MODELZ_TOKEN }} + - name: Push binary release to Docker Registry + uses: docker/build-push-action@v4 + with: + context: . + push: true + platforms: "linux/${{ matrix.platform }}" + file: ./docker/pg-slim/Dockerfile + build-args: | + PG_MAJOR=${{ matrix.version }} + tags: modelzai/pg-slim:${{ matrix.version }} \ No newline at end of file diff --git a/.github/workflows/trunk-install-test.yml b/.github/workflows/trunk-install-test.yml new file mode 100644 index 000000000..c790e2b82 --- /dev/null +++ b/.github/workflows/trunk-install-test.yml @@ -0,0 +1,46 @@ +name: Trunk install test workflow + +on: + workflow_dispatch: + inputs: + version: + type: string + description: Version + required: true + pull_request: + branches: + - main + paths: + - '.github/workflows/trunk-install-test.yml' + +jobs: + test: + name: Run tests + runs-on: + - ubuntu-latest + strategy: + matrix: + version: [14, 15, 16, 17] + schema: ["vectors", "extensions", "public"] + platform: ["amd64", "arm64"] + container: + image: modelzai/pgvecto-rs:${{ matrix.version }}-v${{ github.event.inputs.version }}-${{ matrix.schema }} + options: --user root + env: + PGHOST: "localhost" + PGPORT: "5432" + PGDATABASE: "postgres" + PGUSER: "postgres" + PGPASSWORD: "postgres" + POSTGRES_PASSWORD: "password" + + steps: + - name: Install all extensions in registry + # Entrypoint is overwritten by GitHub Action. We need to execute it manually in order to start Postgres. + # More information here https://github.com/actions/runner/issues/1964 + run: | + bash /usr/local/bin/docker-entrypoint.sh postgres & + sleep 5 + curl https://registry.pgtrunk.io/extensions/all | jq -r ".[] | .name" > /tmp/extensions.txt + trunk-install.sh | tee /tmp/output.txt + cat /tmp/output.txt diff --git a/docker/enterprise.Dockerfile b/docker/enterprise.Dockerfile deleted file mode 100644 index d0f606170..000000000 --- a/docker/enterprise.Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -ARG FROM_TAG -ARG CNPG_TAG -ARG SCHEMA -FROM scratch as nothing -ARG TARGETARCH -FROM modelzai/pgvecto-rs-binary:${FROM_TAG}-${TARGETARCH}-${SCHEMA} as binary - -FROM ghcr.io/cloudnative-pg/postgresql:$CNPG_TAG-bookworm -# use root to install the extension -USER root -COPY --from=binary /pgvecto-rs-binary-release.deb /tmp/vectors.deb -RUN apt install -y /tmp/vectors.deb && rm -f /tmp/vectors.deb - -USER postgres diff --git a/docker/pg-cnpg/Dockerfile b/docker/pg-cnpg/Dockerfile new file mode 100644 index 000000000..670d10c7f --- /dev/null +++ b/docker/pg-cnpg/Dockerfile @@ -0,0 +1,180 @@ +ARG PG_MAJOR +ARG FROM_TAG +ARG SCHEMA +ARG TARGETARCH + +FROM modelzai/pgvecto-rs-binary:${FROM_TAG}-${TARGETARCH}-${SCHEMA} as binary + +FROM rust:1.78-bookworm as builder +ARG TRUNK_VER=0.12.25 +ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL sparse +RUN cargo install --version $TRUNK_VER pg-trunk + +# FROM modelzai/pg-slim:${PG_MAJOR} +FROM xieydd/pg-slim:${PG_MAJOR} +ARG PG_MAJOR + +USER root + +COPY --from=binary /pgvecto-rs-binary-release.deb /tmp/vectors.deb +RUN apt install -y /tmp/vectors.deb && rm -f /tmp/vectors.deb + +# PGDATA is set in pg-slim and used by dependents on this image. +RUN if [ -z "${PGDATA}" ]; then echo "PGDATA is not set"; exit 1; fi + +# Install trunk +COPY --from=builder /usr/local/cargo/bin/trunk /usr/bin/trunk +COPY ./requirements.txt . + +# Install barman-cloud +RUN set -xe; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + python3-pip \ + python3-psycopg2 \ + python3-setuptools \ + ; \ + pip3 install --upgrade pip; \ +# TODO: Remove --no-deps once https://github.com/pypa/pip/issues/9644 is solved + pip3 install --no-deps -r requirements.txt; \ + apt-get autoremove -y; \ + apt-get clean; \ + rm -rf /var/lib/apt/lists/*; + +RUN chown -R postgres:postgres /usr/lib/postgresql/${PG_MAJOR} && \ + chmod -R 0700 /usr/lib/postgresql/${PG_MAJOR} +RUN chown postgres /usr/share/postgresql/${PG_MAJOR}/extension + +RUN apt-get update && apt-get install -y \ + jq \ + curl \ + wget \ + && rm -rf /var/lib/apt/lists/* + +# Install extension dependencies +RUN apt-get update && apt-get install -y \ + libmysqlclient-dev \ + libtcl8.6 \ + libgeos-dev \ + libproj-dev \ + libjson-c-dev \ + libprotobuf-c-dev \ + libxml2-dev \ + libboost-serialization1.74-dev \ + libhiredis-dev \ + libsybdb5 \ + libpython3.10-dev \ + r-base-core \ + openssl \ + liblz4-1 \ + libpcre2-8-0 \ + libuuid1 \ + libgroonga0 \ + libopenblas0-pthread \ + libcurl4 \ + libjson-c5 \ + libsodium23 \ + libgcc-s1 \ + libselinux1 \ + librdkafka1 \ + libgdal30 \ + libcrypt1 \ + liburiparser1 \ + libfreetype6 \ + libzstd1 \ + zlib1g \ + libperl5.34 \ + libgomp1 \ + libssl3 \ + libsfcgal1 \ + openjdk-11-jdk \ + libaio1 \ + libbson-dev \ + libgsl-dev \ + && rm -rf /var/lib/apt/lists/* +RUN ln -s /usr/lib/jvm/java-11-openjdk-amd64/lib/server/libjvm.so /usr/lib/x86_64-linux-gnu/libjvm.so +RUN wget https://download.oracle.com/otn_software/linux/instantclient/1920000/instantclient-basiclite-linux.x64-19.20.0.0.0dbru.zip && \ + unzip instantclient-basiclite-linux.x64-19.20.0.0.0dbru.zip && \ + cp instantclient_19_20/libclntsh.so.19.1 /usr/lib/x86_64-linux-gnu/ && \ + cp instantclient_19_20/libnnz19.so /usr/lib/x86_64-linux-gnu/ && \ + cp instantclient_19_20/libclntshcore.so.19.1 /usr/lib/x86_64-linux-gnu/ && \ + rm -rf instantclient_19_20 && \ + rm instantclient-basiclite-linux.x64-19.20.0.0.0dbru.zip + +# Install zhparser dependency +RUN wget http://www.xunsearch.com/scws/down/scws-1.2.3.tar.bz2 && \ + tar xvf scws-1.2.3.tar.bz2 && \ + cd scws-1.2.3 && \ + ./configure && \ + make install && \ + cd .. && \ + rm -rf scws-1.2.3.tar.bz2 scws-1.2.3 && \ + ln -s /usr/local/lib/libscws.so /usr/lib/x86_64-linux-gnu/libscws.so + +# Install duckdb libs +RUN wget https://github.com/duckdb/duckdb/releases/download/v0.8.1/libduckdb-linux-amd64.zip && \ + unzip libduckdb-linux-amd64.zip && \ + cp libduckdb.so /usr/lib/x86_64-linux-gnu/ && \ + rm -rf libduckdb-linux-amd64.zip libduckdb.so + +# Test trunk +COPY trunk-install.sh /usr/local/bin/ + +# Install pg_stat_statements +RUN trunk install pg_stat_statements + +# Install auto_explain +RUN trunk install auto_explain + +# cache pg_stat_statements and auto_explain and pg_stat_kcache to temp directory +RUN set -eux; \ + mkdir /tmp/pg_pkglibdir; \ + mkdir /tmp/pg_sharedir; \ + cp -r $(pg_config --pkglibdir)/* /tmp/pg_pkglibdir; \ + cp -r $(pg_config --sharedir)/* /tmp/pg_sharedir + +# Clone and build AWS SDK for C++ +RUN git clone https://github.com/aws/aws-sdk-cpp.git && \ + cd aws-sdk-cpp && \ + git checkout 1.9.263 && \ + git submodule update --init --recursive && \ + mkdir build && cd build && \ + cmake -DBUILD_ONLY="s3;core;config;sts;cognito-identity;transfer;identity-management" -DAUTORUN_UNIT_TESTS=OFF -DCMAKE_CXX_FLAGS=-Wno-error=deprecated-declarations .. && \ + make -j$(nproc) && \ + make install && \ + cd ../../../ && rm -rf aws-sdk-cpp + +# Clone and build Apache Arrow +RUN git clone https://github.com/apache/arrow.git && \ + cd arrow && \ + git checkout apache-arrow-7.0.1 && \ + cd cpp && \ + mkdir build && cd build && \ + cmake -DARROW_PARQUET=ON -DARROW_S3=ON -DARROW_WITH_SNAPPY=ON .. && \ + make -j$(nproc) && \ + make install && \ + cd ../../../ && rm -rf arrow + +# Clone and build pgaudit +RUN git clone https://github.com/pgaudit/pgaudit.git && \ + cd pgaudit && \ + git checkout REL_${PG_MAJOR}_STABLE && \ + make install USE_PGXS=1 PG_CONFIG=/usr/lib/postgresql/${PG_MAJOR}/bin/pg_config && \ + cd ../ && rm -rf pgaudit + +# Clone and build pg_failover_slots +RUN git clone https://github.com/EnterpriseDB/pg_failover_slots.git && \ + cd pg_failover_slots && \ + make install PG_CONFIG=/usr/lib/postgresql/${PG_MAJOR}/bin/pg_config && \ + cd ../ && rm -rf pg_failover_slots + +# cache all extensions +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + +COPY ./trunk-install.sh /usr/local/bin/ +# Change the uid of postgres to 26 +RUN usermod -u 26 postgres +RUN chown -R postgres:postgres /usr/lib/postgresql/${PG_MAJOR} +RUN cp /usr/share/postgresql/${PG_MAJOR}/extension/* /usr/lib/postgresql/${PG_MAJOR}/share/extension/ +USER 26 +ENV PATH $PATH:/usr/lib/postgresql/${PG_MAJOR}/bin \ No newline at end of file diff --git a/docker/pg-cnpg/requirements.txt b/docker/pg-cnpg/requirements.txt new file mode 100644 index 000000000..29cb2d28e --- /dev/null +++ b/docker/pg-cnpg/requirements.txt @@ -0,0 +1,464 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --generate-hashes +# +argcomplete==3.0.8 \ + --hash=sha256:b9ca96448e14fa459d7450a4ab5a22bbf9cee4ba7adddf03e65c398b5daeea28 \ + --hash=sha256:e36fd646839933cbec7941c662ecb65338248667358dd3d968405a4506a60d9b +azure-core==1.26.4 \ + --hash=sha256:075fe06b74c3007950dd93d49440c2f3430fd9b4a5a2756ec8c79454afc989c6 \ + --hash=sha256:d9664b4bc2675d72fba461a285ac43ae33abb2967014a955bf136d9703a2ab3c + # via + # azure-identity + # azure-storage-blob +azure-identity==1.13.0 \ + --hash=sha256:bd700cebb80cd9862098587c29d8677e819beca33c62568ced6d5a8e5e332b82 \ + --hash=sha256:c931c27301ffa86b07b4dcf574e29da73e3deba9ab5d1fe4f445bb6a3117e260 +azure-storage-blob==12.16.0 \ + --hash=sha256:43b45f19a518a5c6895632f263b3825ebc23574f25cc84b66e1630a6160e466f \ + --hash=sha256:91bb192b2a97939c4259c72373bac0f41e30810bbc853d5184f0f45904eacafd +barman[azure,cloud,google,snappy]==3.5.0 \ + --hash=sha256:078643961d421a5f54825d3da4bbd04faa94b96a6a0f6cefe23babdc53aff83a \ + --hash=sha256:bea6885c1efe2e140b640d4b2daec8aff03563a5cee199a73f874ae39fede051 + # via -r requirements.in +boto3==1.26.139 \ + --hash=sha256:5b61a82f0c1cd006bd109ddf27c93d9b010c4c188fc583ee257ff6f3bb89970d \ + --hash=sha256:fe19d287bc8ede385e1b9136f135ee8f93eab81404ad1445b1a70cabfe3f7087 +botocore==1.29.139 \ + --hash=sha256:acc62710bdf11e47f4f26fb290a9082ff00377d7e93a16e1f080f9c789898114 \ + --hash=sha256:b164af929eb2f1507833718de9eb8811e3adc6943b464c1869e95ac87f3bab88 + # via + # boto3 + # s3transfer +cachetools==5.3.0 \ + --hash=sha256:13dfddc7b8df938c21a940dfa6557ce6e94a2f1cdfa58eb90c805721d58f2c14 \ + --hash=sha256:429e1a1e845c008ea6c85aa35d4b98b65d6a9763eeef3e37e92728a12d1de9d4 + # via google-auth +certifi==2023.5.7 \ + --hash=sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7 \ + --hash=sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716 + # via requests +cffi==1.15.1 \ + --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ + --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \ + --hash=sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104 \ + --hash=sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426 \ + --hash=sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405 \ + --hash=sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375 \ + --hash=sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a \ + --hash=sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e \ + --hash=sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc \ + --hash=sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf \ + --hash=sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185 \ + --hash=sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497 \ + --hash=sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3 \ + --hash=sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35 \ + --hash=sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c \ + --hash=sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83 \ + --hash=sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21 \ + --hash=sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca \ + --hash=sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984 \ + --hash=sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac \ + --hash=sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd \ + --hash=sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee \ + --hash=sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a \ + --hash=sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2 \ + --hash=sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192 \ + --hash=sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7 \ + --hash=sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585 \ + --hash=sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f \ + --hash=sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e \ + --hash=sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27 \ + --hash=sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b \ + --hash=sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e \ + --hash=sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e \ + --hash=sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d \ + --hash=sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c \ + --hash=sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415 \ + --hash=sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82 \ + --hash=sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02 \ + --hash=sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314 \ + --hash=sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325 \ + --hash=sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c \ + --hash=sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3 \ + --hash=sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914 \ + --hash=sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045 \ + --hash=sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d \ + --hash=sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9 \ + --hash=sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5 \ + --hash=sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2 \ + --hash=sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c \ + --hash=sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3 \ + --hash=sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2 \ + --hash=sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8 \ + --hash=sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d \ + --hash=sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d \ + --hash=sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9 \ + --hash=sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162 \ + --hash=sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76 \ + --hash=sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4 \ + --hash=sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e \ + --hash=sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9 \ + --hash=sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6 \ + --hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \ + --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \ + --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0 + # via cryptography +charset-normalizer==3.1.0 \ + --hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \ + --hash=sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1 \ + --hash=sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e \ + --hash=sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373 \ + --hash=sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62 \ + --hash=sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230 \ + --hash=sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be \ + --hash=sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c \ + --hash=sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0 \ + --hash=sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448 \ + --hash=sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f \ + --hash=sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649 \ + --hash=sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d \ + --hash=sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0 \ + --hash=sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706 \ + --hash=sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a \ + --hash=sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59 \ + --hash=sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23 \ + --hash=sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5 \ + --hash=sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb \ + --hash=sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e \ + --hash=sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e \ + --hash=sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c \ + --hash=sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28 \ + --hash=sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d \ + --hash=sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41 \ + --hash=sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974 \ + --hash=sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce \ + --hash=sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f \ + --hash=sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1 \ + --hash=sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d \ + --hash=sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8 \ + --hash=sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017 \ + --hash=sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31 \ + --hash=sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7 \ + --hash=sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8 \ + --hash=sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e \ + --hash=sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14 \ + --hash=sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd \ + --hash=sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d \ + --hash=sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795 \ + --hash=sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b \ + --hash=sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b \ + --hash=sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b \ + --hash=sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203 \ + --hash=sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f \ + --hash=sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19 \ + --hash=sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1 \ + --hash=sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a \ + --hash=sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac \ + --hash=sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9 \ + --hash=sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0 \ + --hash=sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137 \ + --hash=sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f \ + --hash=sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6 \ + --hash=sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5 \ + --hash=sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909 \ + --hash=sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f \ + --hash=sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0 \ + --hash=sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324 \ + --hash=sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755 \ + --hash=sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb \ + --hash=sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854 \ + --hash=sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c \ + --hash=sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60 \ + --hash=sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84 \ + --hash=sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0 \ + --hash=sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b \ + --hash=sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1 \ + --hash=sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531 \ + --hash=sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1 \ + --hash=sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11 \ + --hash=sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326 \ + --hash=sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df \ + --hash=sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab + # via requests +cryptography==40.0.2 \ + --hash=sha256:05dc219433b14046c476f6f09d7636b92a1c3e5808b9a6536adf4932b3b2c440 \ + --hash=sha256:0dcca15d3a19a66e63662dc8d30f8036b07be851a8680eda92d079868f106288 \ + --hash=sha256:142bae539ef28a1c76794cca7f49729e7c54423f615cfd9b0b1fa90ebe53244b \ + --hash=sha256:3daf9b114213f8ba460b829a02896789751626a2a4e7a43a28ee77c04b5e4958 \ + --hash=sha256:48f388d0d153350f378c7f7b41497a54ff1513c816bcbbcafe5b829e59b9ce5b \ + --hash=sha256:4df2af28d7bedc84fe45bd49bc35d710aede676e2a4cb7fc6d103a2adc8afe4d \ + --hash=sha256:4f01c9863da784558165f5d4d916093737a75203a5c5286fde60e503e4276c7a \ + --hash=sha256:7a38250f433cd41df7fcb763caa3ee9362777fdb4dc642b9a349721d2bf47404 \ + --hash=sha256:8f79b5ff5ad9d3218afb1e7e20ea74da5f76943ee5edb7f76e56ec5161ec782b \ + --hash=sha256:956ba8701b4ffe91ba59665ed170a2ebbdc6fc0e40de5f6059195d9f2b33ca0e \ + --hash=sha256:a04386fb7bc85fab9cd51b6308633a3c271e3d0d3eae917eebab2fac6219b6d2 \ + --hash=sha256:a95f4802d49faa6a674242e25bfeea6fc2acd915b5e5e29ac90a32b1139cae1c \ + --hash=sha256:adc0d980fd2760c9e5de537c28935cc32b9353baaf28e0814df417619c6c8c3b \ + --hash=sha256:aecbb1592b0188e030cb01f82d12556cf72e218280f621deed7d806afd2113f9 \ + --hash=sha256:b12794f01d4cacfbd3177b9042198f3af1c856eedd0a98f10f141385c809a14b \ + --hash=sha256:c0764e72b36a3dc065c155e5b22f93df465da9c39af65516fe04ed3c68c92636 \ + --hash=sha256:c33c0d32b8594fa647d2e01dbccc303478e16fdd7cf98652d5b3ed11aa5e5c99 \ + --hash=sha256:cbaba590180cba88cb99a5f76f90808a624f18b169b90a4abb40c1fd8c19420e \ + --hash=sha256:d5a1bd0e9e2031465761dfa920c16b0065ad77321d8a8c1f5ee331021fda65e9 + # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt +google-api-core==2.11.0 \ + --hash=sha256:4b9bb5d5a380a0befa0573b302651b8a9a89262c1730e37bf423cec511804c22 \ + --hash=sha256:ce222e27b0de0d7bc63eb043b956996d6dccab14cc3b690aaea91c9cc99dc16e + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.18.1 \ + --hash=sha256:55a395cdfd3f3dd3f649131d41f97c17b4ed8a2aac1be3502090c716314e8a37 \ + --hash=sha256:d7a3249027e7f464fbbfd7ee8319a08ad09d2eea51578575c4bd360ffa049ccb + # via + # google-api-core + # google-cloud-core + # google-cloud-storage +google-cloud-core==2.3.2 \ + --hash=sha256:8417acf6466be2fa85123441696c4badda48db314c607cf1e5d543fa8bdc22fe \ + --hash=sha256:b9529ee7047fd8d4bf4a2182de619154240df17fbe60ead399078c1ae152af9a + # via google-cloud-storage +google-cloud-storage==2.9.0 \ + --hash=sha256:83a90447f23d5edd045e0037982c270302e3aeb45fc1288d2c2ca713d27bad94 \ + --hash=sha256:9b6ae7b509fc294bdacb84d0f3ea8e20e2c54a8b4bbe39c5707635fec214eff3 +google-crc32c==1.5.0 \ + --hash=sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a \ + --hash=sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876 \ + --hash=sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c \ + --hash=sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289 \ + --hash=sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298 \ + --hash=sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02 \ + --hash=sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f \ + --hash=sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2 \ + --hash=sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a \ + --hash=sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb \ + --hash=sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210 \ + --hash=sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5 \ + --hash=sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee \ + --hash=sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c \ + --hash=sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a \ + --hash=sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314 \ + --hash=sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd \ + --hash=sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65 \ + --hash=sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37 \ + --hash=sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4 \ + --hash=sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13 \ + --hash=sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894 \ + --hash=sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31 \ + --hash=sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e \ + --hash=sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709 \ + --hash=sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740 \ + --hash=sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc \ + --hash=sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d \ + --hash=sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c \ + --hash=sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c \ + --hash=sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d \ + --hash=sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906 \ + --hash=sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61 \ + --hash=sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57 \ + --hash=sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c \ + --hash=sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a \ + --hash=sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438 \ + --hash=sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946 \ + --hash=sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7 \ + --hash=sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96 \ + --hash=sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091 \ + --hash=sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae \ + --hash=sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d \ + --hash=sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88 \ + --hash=sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2 \ + --hash=sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd \ + --hash=sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541 \ + --hash=sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728 \ + --hash=sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178 \ + --hash=sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968 \ + --hash=sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346 \ + --hash=sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8 \ + --hash=sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93 \ + --hash=sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7 \ + --hash=sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273 \ + --hash=sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462 \ + --hash=sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94 \ + --hash=sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd \ + --hash=sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e \ + --hash=sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57 \ + --hash=sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b \ + --hash=sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9 \ + --hash=sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a \ + --hash=sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100 \ + --hash=sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325 \ + --hash=sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183 \ + --hash=sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556 \ + --hash=sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4 + # via google-resumable-media +google-resumable-media==2.5.0 \ + --hash=sha256:218931e8e2b2a73a58eb354a288e03a0fd5fb1c4583261ac6e4c078666468c93 \ + --hash=sha256:da1bd943e2e114a56d85d6848497ebf9be6a14d3db23e9fc57581e7c3e8170ec + # via google-cloud-storage +googleapis-common-protos==1.59.0 \ + --hash=sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44 \ + --hash=sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f + # via google-api-core +idna==3.4 \ + --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ + --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 + # via requests +isodate==0.6.1 \ + --hash=sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96 \ + --hash=sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9 + # via azure-storage-blob +jmespath==1.0.1 \ + --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 \ + --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe + # via + # boto3 + # botocore +msal==1.22.0 \ + --hash=sha256:8a82f5375642c1625c89058018430294c109440dce42ea667d466c2cab520acd \ + --hash=sha256:9120b7eafdf061c92f7b3d744e5f325fca35873445fa8ffebb40b1086a13dd58 + # via + # azure-identity + # msal-extensions +msal-extensions==1.0.0 \ + --hash=sha256:91e3db9620b822d0ed2b4d1850056a0f133cba04455e62f11612e40f5502f2ee \ + --hash=sha256:c676aba56b0cce3783de1b5c5ecfe828db998167875126ca4b47dc6436451354 + # via azure-identity +portalocker==2.7.0 \ + --hash=sha256:032e81d534a88ec1736d03f780ba073f047a06c478b06e2937486f334e955c51 \ + --hash=sha256:a07c5b4f3985c3cf4798369631fb7011adb498e2a46d8440efc75a8f29a0f983 + # via msal-extensions +protobuf==4.23.1 \ + --hash=sha256:2036a3a1e7fc27f973fa0a7888dce712393af644f4695385f117886abc792e39 \ + --hash=sha256:32e78beda26d7a101fecf15d7a4a792278a0d26a31bc327ff05564a9d68ab8ee \ + --hash=sha256:346990f634272caac1f09efbcfbbacb23098b1f606d172534c6fa2d9758bb436 \ + --hash=sha256:3b8905eafe4439076e1f58e9d1fa327025fd2777cf90f14083092ae47f77b0aa \ + --hash=sha256:3ce113b3f3362493bddc9069c2163a38f240a9ed685ff83e7bcb756b05e1deb0 \ + --hash=sha256:410bcc0a5b279f634d3e16082ce221dfef7c3392fac723500e2e64d1806dd2be \ + --hash=sha256:5b9cd6097e6acae48a68cb29b56bc79339be84eca65b486910bb1e7a30e2b7c1 \ + --hash=sha256:65f0ac96ef67d7dd09b19a46aad81a851b6f85f89725577f16de38f2d68ad477 \ + --hash=sha256:91fac0753c3c4951fbb98a93271c43cc7cf3b93cf67747b3e600bb1e5cc14d61 \ + --hash=sha256:95789b569418a3e32a53f43d7763be3d490a831e9c08042539462b6d972c2d7e \ + --hash=sha256:ac50be82491369a9ec3710565777e4da87c6d2e20404e0abb1f3a8f10ffd20f0 \ + --hash=sha256:decf119d54e820f298ee6d89c72d6b289ea240c32c521f00433f9dc420595f38 \ + --hash=sha256:f9510cac91e764e86acd74e2b7f7bc5e6127a7f3fb646d7c8033cfb84fd1176a + # via + # google-api-core + # googleapis-common-protos +pyasn1==0.5.0 \ + --hash=sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57 \ + --hash=sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 \ + --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ + --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d + # via google-auth +pycparser==2.21 \ + --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ + --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 + # via cffi +pyjwt[crypto]==2.7.0 \ + --hash=sha256:ba2b425b15ad5ef12f200dc67dd56af4e26de2331f965c5439994dad075876e1 \ + --hash=sha256:bd6ca4a3c4285c1a2d4349e5a035fdf8fb94e04ccd0fcbe6ba289dae9cc3e074 + # via msal +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 + # via + # barman + # botocore +python-snappy==0.6.1 \ + --hash=sha256:03bb511380fca2a13325b6f16fe8234c8e12da9660f0258cd45d9a02ffc916af \ + --hash=sha256:0bdb6942180660bda7f7d01f4c0def3cfc72b1c6d99aad964801775a3e379aba \ + --hash=sha256:0d489b50f49433494160c45048fe806de6b3aeab0586e497ebd22a0bab56e427 \ + --hash=sha256:1a993dc8aadd901915a510fe6af5f20ae4256f527040066c22a154db8946751f \ + --hash=sha256:1d029f7051ec1bbeaa3e03030b6d8ed47ceb69cae9016f493c802a08af54e026 \ + --hash=sha256:277757d5dad4e239dc1417438a0871b65b1b155beb108888e7438c27ffc6a8cc \ + --hash=sha256:2a7e528ab6e09c0d67dcb61a1730a292683e5ff9bb088950638d3170cf2a0a54 \ + --hash=sha256:2aaaf618c68d8c9daebc23a20436bd01b09ee70d7fbf7072b7f38b06d2fab539 \ + --hash=sha256:2be4f4550acd484912441f5f1209ba611ac399aac9355fee73611b9a0d4f949c \ + --hash=sha256:39692bedbe0b717001a99915ac0eb2d9d0bad546440d392a2042b96d813eede1 \ + --hash=sha256:3fb9a88a4dd6336488f3de67ce75816d0d796dce53c2c6e4d70e0b565633c7fd \ + --hash=sha256:4038019b1bcaadde726a57430718394076c5a21545ebc5badad2c045a09546cf \ + --hash=sha256:463fd340a499d47b26ca42d2f36a639188738f6e2098c6dbf80aef0e60f461e1 \ + --hash=sha256:4d3cafdf454354a621c8ab7408e45aa4e9d5c0b943b61ff4815f71ca6bdf0130 \ + --hash=sha256:4ec533a8c1f8df797bded662ec3e494d225b37855bb63eb0d75464a07947477c \ + --hash=sha256:530bfb9efebcc1aab8bb4ebcbd92b54477eed11f6cf499355e882970a6d3aa7d \ + --hash=sha256:546c1a7470ecbf6239101e9aff0f709b68ca0f0268b34d9023019a55baa1f7c6 \ + --hash=sha256:5843feb914796b1f0405ccf31ea0fb51034ceb65a7588edfd5a8250cb369e3b2 \ + --hash=sha256:586724a0276d7a6083a17259d0b51622e492289a9998848a1b01b6441ca12b2f \ + --hash=sha256:59e975be4206cc54d0a112ef72fa3970a57c2b1bcc2c97ed41d6df0ebe518228 \ + --hash=sha256:5a453c45178d7864c1bdd6bfe0ee3ed2883f63b9ba2c9bb967c6b586bf763f96 \ + --hash=sha256:5bb05c28298803a74add08ba496879242ef159c75bc86a5406fac0ffc7dd021b \ + --hash=sha256:5e973e637112391f05581f427659c05b30b6843bc522a65be35ac7b18ce3dedd \ + --hash=sha256:66c80e9b366012dbee262bb1869e4fc5ba8786cda85928481528bc4a72ec2ee8 \ + --hash=sha256:6a7620404da966f637b9ce8d4d3d543d363223f7a12452a575189c5355fc2d25 \ + --hash=sha256:6f8bf4708a11b47517baf962f9a02196478bbb10fdb9582add4aa1459fa82380 \ + --hash=sha256:735cd4528c55dbe4516d6d2b403331a99fc304f8feded8ae887cf97b67d589bb \ + --hash=sha256:7778c224efc38a40d274da4eb82a04cac27aae20012372a7db3c4bbd8926c4d4 \ + --hash=sha256:8277d1f6282463c40761f802b742f833f9f2449fcdbb20a96579aa05c8feb614 \ + --hash=sha256:88b6ea78b83d2796f330b0af1b70cdd3965dbdab02d8ac293260ec2c8fe340ee \ + --hash=sha256:8c07220408d3268e8268c9351c5c08041bc6f8c6172e59d398b71020df108541 \ + --hash=sha256:8d0c019ee7dcf2c60e240877107cddbd95a5b1081787579bf179938392d66480 \ + --hash=sha256:90b0186516b7a101c14764b0c25931b741fb0102f21253eff67847b4742dfc72 \ + --hash=sha256:9837ac1650cc68d22a3cf5f15fb62c6964747d16cecc8b22431f113d6e39555d \ + --hash=sha256:9eac51307c6a1a38d5f86ebabc26a889fddf20cbba7a116ccb54ba1446601d5b \ + --hash=sha256:9f0c0d88b84259f93c3aa46398680646f2c23e43394779758d9f739c34e15295 \ + --hash=sha256:a0ad38bc98d0b0497a0b0dbc29409bcabfcecff4511ed7063403c86de16927bc \ + --hash=sha256:b265cde49774752aec9ca7f5d272e3f98718164afc85521622a8a5394158a2b5 \ + --hash=sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a \ + --hash=sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b \ + --hash=sha256:c20498bd712b6e31a4402e1d027a1cd64f6a4a0066a3fe3c7344475886d07fdf \ + --hash=sha256:cb18d9cd7b3f35a2f5af47bb8ed6a5bdbf4f3ddee37f3daade4ab7864c292f5b \ + --hash=sha256:cf5bb9254e1c38aacf253d510d3d9be631bba21f3d068b17672b38b5cbf2fff5 \ + --hash=sha256:d017775851a778ec9cc32651c4464079d06d927303c2dde9ae9830ccf6fe94e1 \ + --hash=sha256:dc96668d9c7cc656609764275c5f8da58ef56d89bdd6810f6923d36497468ff7 \ + --hash=sha256:e066a0586833d610c4bbddba0be5ba0e3e4f8e0bc5bb6d82103d8f8fc47bb59a \ + --hash=sha256:e3a013895c64352b49d0d8e107a84f99631b16dbab156ded33ebf0becf56c8b2 \ + --hash=sha256:eaf905a580f2747c4a474040a5063cd5e0cc3d1d2d6edb65f28196186493ad4a +requests==2.31.0 \ + --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ + --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 + # via + # azure-core + # google-api-core + # google-cloud-storage + # msal +rsa==4.9 \ + --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ + --hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21 + # via google-auth +s3transfer==0.6.1 \ + --hash=sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346 \ + --hash=sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9 + # via boto3 +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 + # via + # azure-core + # azure-identity + # google-auth + # isodate + # python-dateutil +typing-extensions==4.6.0 \ + --hash=sha256:6ad00b63f849b7dcc313b70b6b304ed67b2b2963b3098a33efe18056b1a9a223 \ + --hash=sha256:ff6b238610c747e44c268aa4bb23c8c735d665a63726df3f9431ce707f2aa768 + # via + # azure-core + # azure-storage-blob +urllib3==1.26.16 \ + --hash=sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f \ + --hash=sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14 + # via + # botocore + # google-auth + # requests diff --git a/docker/pg-cnpg/trunk-install.sh b/docker/pg-cnpg/trunk-install.sh new file mode 100755 index 000000000..6acd896e4 --- /dev/null +++ b/docker/pg-cnpg/trunk-install.sh @@ -0,0 +1,80 @@ +#!/bin/bash +trunk_install_failed_extensions=() +need_load_shared_preload_libraries_extensions=() +version_not_found_extensions=() +file='/tmp/extensions.txt' +extension_count=$(<$file wc -l) +lines=$(cat $file) +for line in $lines +do + output=$(trunk install $line 2>&1) + + if [ $? -ne 0 ]; then + if [[ $output == *"Failed to find an archive for"* || $output == *"Failed to fetch Trunk archive from"* ]]; then + version_not_found_extensions+=("$line") + else + echo "trunk install command failed" + trunk_install_failed_extensions+=("$line") + fi + fi + echo $output + printf "\n\n" +done +IFS=$'\n' extensions=(`psql postgres://postgres:postgres@localhost:5432 -tA postgres -c 'select name from pg_available_extensions;'`) +for ext in "${extensions[@]}" +do + # drop schema columnar if ext name is columnar + if [ "$ext" == "columnar" ]; then + psql postgres://postgres:postgres@localhost:5432 -c "drop extension if exists citus_columnar cascade;" + fi + # drop type semver if ext name is semver + if [ "$ext" == "semver" ]; then + psql postgres://postgres:postgres@localhost:5432 -c "drop extension if exists pg_text_semver cascade;" + fi + # if extension name is meta_triggers, create extension meta first and create extension meta_triggers + if [ "$ext" == "meta_triggers" ]; then + psql postgres://postgres:postgres@localhost:5432 -c "create extension if not exists hstore cascade;" + psql postgres://postgres:postgres@localhost:5432 -c "create extension if not exists meta cascade;" + psql postgres://postgres:postgres@localhost:5432 -c "create extension if not exists meta_triggers cascade;" + fi + output=$(psql postgres://postgres:postgres@localhost:5432 -c "create extension if not exists \"$ext\" cascade;" 2>&1) + if [ $? -ne 0 ]; then + if [[ $output == *"shared_preload_libraries"* ]]; then + need_load_shared_preload_libraries_extensions+=("$ext") + elif [[ $output == *"already exists"* ]]; then + echo "extension \"$ext\" already exists" + else + echo "CREATE EXTENSION command failed" + failed_extensions+=("$ext") + fi + fi + echo $output + printf "\n\n" +done +available_extensions_count=${#extensions[@]} +failure_count=${#failed_extensions[@]} +need_load_shared_preload_libraries_count=${#need_load_shared_preload_libraries_extensions[@]} +not_found_count=${#version_not_found_extensions[@]} +success=$(($available_extensions_count-$failure_count)) +success_percent=$(awk "BEGIN { pc=100*${success}/${extension_count}; i=int(pc); print (pc-i<0.5)?i:i+1 }") +failure_percent=$(awk "BEGIN { pc=100*${failure_count}/${extension_count}; i=int(pc); print (pc-i<0.5)?i:i+1 }") + +printf "\n\n***TRUNK INSTALL EXTENSIONS THAT VERSION NOT FOUND RATE***\n" +echo "$not_found_count / $extension_count" +printf "***CREATE EXTENSIONS SUCCESS RATE***\n" +echo "$success / $extension_count ($success_percent%)" +printf "\n\n***CREATE EXTENSION FAILURE RATE***\n" +echo "$failure_count / $extension_count ($failure_percent%)" +printf "\n\n***CREATE EXTENSIONS THAT NEED TO BE LOADED IN shared_preload_libraries***\n" +echo "$need_load_shared_preload_libraries_count / $extension_count" +printf "\n\n***NEED TO LOAD shared_preload_libraries EXTENSIONS***\n" +for need in "${need_load_shared_preload_libraries_extensions[@]}" +do + echo $need +done + +printf "\n\n***FAILED EXTENSIONS***\n" +for failed in "${failed_extensions[@]}" +do + echo $failed +done diff --git a/docker/pg-slim/Dockerfile b/docker/pg-slim/Dockerfile new file mode 100644 index 000000000..68eea8220 --- /dev/null +++ b/docker/pg-slim/Dockerfile @@ -0,0 +1,138 @@ +FROM ubuntu:22.04 + +# 15, 16, 17 +ARG PG_MAJOR +# 15.8-1.pgdg120+1 16.4-1.pgdg120+2 17.0-1.pgdg120+1 +# ARG PG_VERSION +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC +ENV PGDATA /var/lib/postgresql/data +ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin + +# Get latest package updates +RUN set -eux; \ + apt-get update; \ + apt-get upgrade -y + +# explicitly set user/group IDs +RUN set -eux; \ + groupadd -r postgres --gid=999; \ +# https://salsa.debian.org/postgresql/postgresql-common/blob/997d842ee744687d99a2b2d95c1083a2615c79e8/debian/postgresql-common.postinst#L32-35 + useradd -r -g postgres --uid=999 --home-dir=/var/lib/postgresql --shell=/bin/bash postgres; \ +# also create the postgres user's home directory with appropriate permissions +# see https://github.com/docker-library/postgres/issues/274 + install --verbose --directory --owner postgres --group postgres --mode 1777 /var/lib/postgresql + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + locales curl ca-certificates gnupg lsb-release lbzip2 git cmake \ + ; \ + rm -rf /var/lib/apt/lists/*; \ + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 +ENV LANG en_US.utf8 + +RUN mkdir /docker-entrypoint-initdb.d + +RUN git clone https://github.com/postgres/postgres.git && \ + cd postgres && \ + PG_RELEASE=$(git tag | grep '^REL_'${PG_MAJOR}'_' | grep -vE 'RC|BETA' | sort -V | tail -1 | sed 's/REL_//') && \ + echo $PG_RELEASE && \ + git checkout REL_$PG_RELEASE + +RUN set -eux; \ + apt-get update && apt-get install -y \ + libreadline-dev \ + zlib1g-dev \ + libpq-dev \ + build-essential \ + python3-dev \ + tcl-dev \ + libxslt1-dev \ + libperl-dev \ + libpam0g-dev \ + libreadline-dev \ + libssl-dev \ + xz-utils \ + libnss-wrapper \ + llvm \ + clang \ + icu-devtools \ + pkg-config \ + libgss-dev \ + libkrb5-dev \ + uuid-dev \ + gettext \ + liblz4-dev \ + libsystemd-dev \ + libselinux1-dev \ + libzstd-dev \ + vim \ + flex \ + bison; \ + apt-get autoremove -y; \ + apt-get clean -y; \ + rm -rf /var/lib/apt/lists/* + +WORKDIR postgres + +ENV CFLAGS "-g -O2 -fstack-protector-strong -Wformat -Werror=format-security -fno-omit-frame-pointer" +ENV LDFLAGS "-Wl,-z,relro -Wl,-z,now" +RUN ./configure --prefix=/usr/lib/postgresql/${PG_MAJOR} \ + --libdir=/usr/lib/postgresql/$PG_MAJOR/lib \ + --with-perl \ + --with-python \ + --with-tcl \ + --with-pam \ + --with-libxml \ + --with-libxslt \ + --with-openssl \ + --enable-nls \ + --enable-thread-safety \ + --enable-debug \ + --disable-rpath \ + --with-uuid=e2fs \ + --with-gnu-ld \ + --with-gssapi \ + --with-pgport=5432 \ + --with-system-tzdata=/usr/share/zoneinfo \ + --with-icu \ + --with-llvm \ + --with-lz4 \ + --with-zstd \ + --with-systemd \ + --with-selinux + +RUN make -j$(nproc) +RUN make install + +WORKDIR / +RUN rm -rf /postgres + +RUN mkdir -p /var/run/postgresql && chmod 775 /var/run/postgresql +RUN mkdir -p /usr/share/postgresql/${PG_MAJOR}/extension && chmod 775 /usr/share/postgresql/${PG_MAJOR}/extension + +COPY --from=tianon/gosu /gosu /usr/local/bin/ + +# make the sample config easier to munge (and "correct by default") +RUN set -eux; \ + echo "unix_socket_directories = '/var/run/postgresql'" >> /usr/lib/postgresql/${PG_MAJOR}/share/postgresql.conf.sample; \ + sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/lib/postgresql/${PG_MAJOR}/share/postgresql.conf.sample; \ + grep -F "listen_addresses = '*'" /usr/lib/postgresql/16/share/postgresql.conf.sample + +RUN install --verbose --directory --owner postgres --group postgres --mode 3777 /var/run/postgresql + +# this 1777 will be replaced by 0700 at runtime (allows semi-arbitrary "--user" values) +RUN install --verbose --directory --owner postgres --group postgres --mode 1777 ${PGDATA} + +ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin:/usr/local/bin +COPY ./docker-entrypoint.sh ./docker-ensure-initdb.sh /usr/local/bin/ +RUN ln -sT docker-ensure-initdb.sh /usr/local/bin/docker-enforce-initdb.sh +ENTRYPOINT ["docker-entrypoint.sh"] + +# Remove pre-installed pg_config +RUN rm /usr/bin/pg_config + +STOPSIGNAL SIGINT +EXPOSE 5432 +CMD ["postgres"] \ No newline at end of file diff --git a/docker/pg-slim/docker-ensure-initdb.sh b/docker/pg-slim/docker-ensure-initdb.sh new file mode 100755 index 000000000..ae1f6b6b9 --- /dev/null +++ b/docker/pg-slim/docker-ensure-initdb.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# +# This script is intended for three main use cases: +# +# 1. (most importantly) as an example of how to use "docker-entrypoint.sh" to extend/reuse the initialization behavior +# +# 2. ("docker-ensure-initdb.sh") as a Kubernetes "init container" to ensure the provided database directory is initialized; see also "startup probes" for an alternative solution +# (no-op if database is already initialized) +# +# 3. ("docker-enforce-initdb.sh") as part of CI to ensure the database is fully initialized before use +# (error if database is already initialized) +# + +source /usr/local/bin/docker-entrypoint.sh + +# arguments to this script are assumed to be arguments to the "postgres" server (same as "docker-entrypoint.sh"), and most "docker-entrypoint.sh" functions assume "postgres" is the first argument (see "_main" over there) +if [ "$#" -eq 0 ] || [ "$1" != 'postgres' ]; then + set -- postgres "$@" +fi + +# see also "_main" in "docker-entrypoint.sh" + +docker_setup_env +# setup data directories and permissions (when run as root) +docker_create_db_directories +if [ "$(id -u)" = '0' ]; then + # then restart script as postgres user + exec gosu postgres "$BASH_SOURCE" "$@" +fi + +# only run initialization on an empty data directory +if [ -z "$DATABASE_ALREADY_EXISTS" ]; then + docker_verify_minimum_env + + # check dir permissions to reduce likelihood of half-initialized database + ls /docker-entrypoint-initdb.d/ > /dev/null + + docker_init_database_dir + pg_setup_hba_conf "$@" + + # PGPASSWORD is required for psql when authentication is required for 'local' connections via pg_hba.conf and is otherwise harmless + # e.g. when '--auth=md5' or '--auth-local=md5' is used in POSTGRES_INITDB_ARGS + export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" + docker_temp_server_start "$@" + + docker_setup_db + docker_process_init_files /docker-entrypoint-initdb.d/* + + docker_temp_server_stop + unset PGPASSWORD +else + self="$(basename "$0")" + case "$self" in + docker-ensure-initdb.sh) + echo >&2 "$self: note: database already initialized in '$PGDATA'!" + exit 0 + ;; + + docker-enforce-initdb.sh) + echo >&2 "$self: error: (unexpected) database found in '$PGDATA'!" + exit 1 + ;; + + *) + echo >&2 "$self: error: unknown file name: $self" + exit 99 + ;; + esac +fi diff --git a/docker/pg-slim/docker-entrypoint.sh b/docker/pg-slim/docker-entrypoint.sh new file mode 100755 index 000000000..6f59993e0 --- /dev/null +++ b/docker/pg-slim/docker-entrypoint.sh @@ -0,0 +1,356 @@ +#!/usr/bin/env bash +set -Eeo pipefail +# TODO swap to -Eeuo pipefail above (after handling all potentially-unset variables) + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + printf >&2 'error: both %s and %s are set (but are exclusive)\n' "$var" "$fileVar" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# check to see if this file is being run or sourced from another script +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} + +# used to create initial postgres directories and if run as root, ensure ownership to the "postgres" user +docker_create_db_directories() { + local user; user="$(id -u)" + + mkdir -p "$PGDATA" + # ignore failure since there are cases where we can't chmod (and PostgreSQL might fail later anyhow - it's picky about permissions of this directory) + chmod 00700 "$PGDATA" || : + + # ignore failure since it will be fine when using the image provided directory; see also https://github.com/docker-library/postgres/pull/289 + mkdir -p /var/run/postgresql || : + chmod 03775 /var/run/postgresql || : + + # Create the transaction log directory before initdb is run so the directory is owned by the correct user + if [ -n "${POSTGRES_INITDB_WALDIR:-}" ]; then + mkdir -p "$POSTGRES_INITDB_WALDIR" + if [ "$user" = '0' ]; then + find "$POSTGRES_INITDB_WALDIR" \! -user postgres -exec chown postgres '{}' + + fi + chmod 700 "$POSTGRES_INITDB_WALDIR" + fi + + # allow the container to be started with `--user` + if [ "$user" = '0' ]; then + find "$PGDATA" \! -user postgres -exec chown postgres '{}' + + find /var/run/postgresql \! -user postgres -exec chown postgres '{}' + + fi +} + +# initialize empty PGDATA directory with new database via 'initdb' +# arguments to `initdb` can be passed via POSTGRES_INITDB_ARGS or as arguments to this function +# `initdb` automatically creates the "postgres", "template0", and "template1" dbnames +# this is also where the database user is created, specified by `POSTGRES_USER` env +docker_init_database_dir() { + # "initdb" is particular about the current user existing in "/etc/passwd", so we use "nss_wrapper" to fake that if necessary + # see https://github.com/docker-library/postgres/pull/253, https://github.com/docker-library/postgres/issues/359, https://cwrap.org/nss_wrapper.html + local uid; uid="$(id -u)" + if ! getent passwd "$uid" &> /dev/null; then + # see if we can find a suitable "libnss_wrapper.so" (https://salsa.debian.org/sssd-team/nss-wrapper/-/commit/b9925a653a54e24d09d9b498a2d913729f7abb15) + local wrapper + for wrapper in {/usr,}/lib{/*,}/libnss_wrapper.so; do + if [ -s "$wrapper" ]; then + NSS_WRAPPER_PASSWD="$(mktemp)" + NSS_WRAPPER_GROUP="$(mktemp)" + export LD_PRELOAD="$wrapper" NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP + local gid; gid="$(id -g)" + printf 'postgres:x:%s:%s:PostgreSQL:%s:/bin/false\n' "$uid" "$gid" "$PGDATA" > "$NSS_WRAPPER_PASSWD" + printf 'postgres:x:%s:\n' "$gid" > "$NSS_WRAPPER_GROUP" + break + fi + done + fi + + if [ -n "${POSTGRES_INITDB_WALDIR:-}" ]; then + set -- --waldir "$POSTGRES_INITDB_WALDIR" "$@" + fi + + # --pwfile refuses to handle a properly-empty file (hence the "\n"): https://github.com/docker-library/postgres/issues/1025 + eval 'initdb --username="$POSTGRES_USER" --pwfile=<(printf "%s\n" "$POSTGRES_PASSWORD") '"$POSTGRES_INITDB_ARGS"' "$@"' + + # unset/cleanup "nss_wrapper" bits + if [[ "${LD_PRELOAD:-}" == */libnss_wrapper.so ]]; then + rm -f "$NSS_WRAPPER_PASSWD" "$NSS_WRAPPER_GROUP" + unset LD_PRELOAD NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP + fi +} + +# print large warning if POSTGRES_PASSWORD is long +# error if both POSTGRES_PASSWORD is empty and POSTGRES_HOST_AUTH_METHOD is not 'trust' +# print large warning if POSTGRES_HOST_AUTH_METHOD is set to 'trust' +# assumes database is not set up, ie: [ -z "$DATABASE_ALREADY_EXISTS" ] +docker_verify_minimum_env() { + case "${PG_MAJOR:-}" in + 12 | 13) # https://github.com/postgres/postgres/commit/67a472d71c98c3d2fa322a1b4013080b20720b98 + # check password first so we can output the warning before postgres + # messes it up + if [ "${#POSTGRES_PASSWORD}" -ge 100 ]; then + cat >&2 <<-'EOWARN' + + WARNING: The supplied POSTGRES_PASSWORD is 100+ characters. + + This will not work if used via PGPASSWORD with "psql". + + https://www.postgresql.org/message-id/flat/E1Rqxp2-0004Qt-PL%40wrigleys.postgresql.org (BUG #6412) + https://github.com/docker-library/postgres/issues/507 + + EOWARN + fi + ;; + esac + if [ -z "$POSTGRES_PASSWORD" ] && [ 'trust' != "$POSTGRES_HOST_AUTH_METHOD" ]; then + # The - option suppresses leading tabs but *not* spaces. :) + cat >&2 <<-'EOE' + Error: Database is uninitialized and superuser password is not specified. + You must specify POSTGRES_PASSWORD to a non-empty value for the + superuser. For example, "-e POSTGRES_PASSWORD=password" on "docker run". + + You may also use "POSTGRES_HOST_AUTH_METHOD=trust" to allow all + connections without a password. This is *not* recommended. + + See PostgreSQL documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + EOE + exit 1 + fi + if [ 'trust' = "$POSTGRES_HOST_AUTH_METHOD" ]; then + cat >&2 <<-'EOWARN' + ******************************************************************************** + WARNING: POSTGRES_HOST_AUTH_METHOD has been set to "trust". This will allow + anyone with access to the Postgres port to access your database without + a password, even if POSTGRES_PASSWORD is set. See PostgreSQL + documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + In Docker's default configuration, this is effectively any other + container on the same system. + + It is not recommended to use POSTGRES_HOST_AUTH_METHOD=trust. Replace + it with "-e POSTGRES_PASSWORD=password" instead to set a password in + "docker run". + ******************************************************************************** + EOWARN + fi +} + +# usage: docker_process_init_files [file [file [...]]] +# ie: docker_process_init_files /always-initdb.d/* +# process initializer files, based on file extensions and permissions +docker_process_init_files() { + # psql here for backwards compatibility "${psql[@]}" + psql=( docker_process_sql ) + + printf '\n' + local f + for f; do + case "$f" in + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then + printf '%s: running %s\n' "$0" "$f" + "$f" + else + printf '%s: sourcing %s\n' "$0" "$f" + . "$f" + fi + ;; + *.sql) printf '%s: running %s\n' "$0" "$f"; docker_process_sql -f "$f"; printf '\n' ;; + *.sql.gz) printf '%s: running %s\n' "$0" "$f"; gunzip -c "$f" | docker_process_sql; printf '\n' ;; + *.sql.xz) printf '%s: running %s\n' "$0" "$f"; xzcat "$f" | docker_process_sql; printf '\n' ;; + *.sql.zst) printf '%s: running %s\n' "$0" "$f"; zstd -dc "$f" | docker_process_sql; printf '\n' ;; + *) printf '%s: ignoring %s\n' "$0" "$f" ;; + esac + printf '\n' + done +} + +# Execute sql script, passed via stdin (or -f flag of pqsl) +# usage: docker_process_sql [psql-cli-args] +# ie: docker_process_sql --dbname=mydb <<<'INSERT ...' +# ie: docker_process_sql -f my-file.sql +# ie: docker_process_sql > "$PGDATA/pg_hba.conf" +} + +# start socket-only postgresql server for setting up or running scripts +# all arguments will be passed along as arguments to `postgres` (via pg_ctl) +docker_temp_server_start() { + if [ "$1" = 'postgres' ]; then + shift + fi + + # internal start of server in order to allow setup using psql client + # does not listen on external TCP/IP and waits until start finishes + set -- "$@" -c listen_addresses='' -p "${PGPORT:-5432}" + + PGUSER="${PGUSER:-$POSTGRES_USER}" \ + pg_ctl -D "$PGDATA" \ + -o "$(printf '%q ' "$@")" \ + -w start +} + +# stop postgresql server after done setting up user and running scripts +docker_temp_server_stop() { + PGUSER="${PGUSER:-postgres}" \ + pg_ctl -D "$PGDATA" -m fast -w stop +} + +# check arguments for an option that would cause postgres to stop +# return true if there is one +_pg_want_help() { + local arg + for arg; do + case "$arg" in + # postgres --help | grep 'then exit' + # leaving out -C on purpose since it always fails and is unhelpful: + # postgres: could not access the server configuration file "/var/lib/postgresql/data/postgresql.conf": No such file or directory + -'?'|--help|--describe-config|-V|--version) + return 0 + ;; + esac + done + return 1 +} + +_main() { + # if first arg looks like a flag, assume we want to run postgres server + if [ "${1:0:1}" = '-' ]; then + set -- postgres "$@" + fi + + if [ "$1" = 'postgres' ] && ! _pg_want_help "$@"; then + docker_setup_env + # setup data directories and permissions (when run as root) + docker_create_db_directories + if [ "$(id -u)" = '0' ]; then + # then restart script as postgres user + exec gosu postgres "$BASH_SOURCE" "$@" + fi + + # only run initialization on an empty data directory + if [ -z "$DATABASE_ALREADY_EXISTS" ]; then + docker_verify_minimum_env + + # check dir permissions to reduce likelihood of half-initialized database + ls /docker-entrypoint-initdb.d/ > /dev/null + + docker_init_database_dir + pg_setup_hba_conf "$@" + + # PGPASSWORD is required for psql when authentication is required for 'local' connections via pg_hba.conf and is otherwise harmless + # e.g. when '--auth=md5' or '--auth-local=md5' is used in POSTGRES_INITDB_ARGS + export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" + docker_temp_server_start "$@" + + docker_setup_db + docker_process_init_files /docker-entrypoint-initdb.d/* + + docker_temp_server_stop + unset PGPASSWORD + + cat <<-'EOM' + + PostgreSQL init process complete; ready for start up. + + EOM + else + cat <<-'EOM' + + PostgreSQL Database directory appears to contain a database; Skipping initialization + + EOM + fi + fi + + exec "$@" +} + +if ! _is_sourced; then + _main "$@" +fi