diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml new file mode 100644 index 0000000000..cd8bc8740c --- /dev/null +++ b/.github/.OwlBot.lock.yaml @@ -0,0 +1,9 @@ +# Copyright 2021 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +docker: + image: gcr.io/repo-automation-bots/owlbot-python:latest + digest: sha256:0ffe3bdd6c7159692df5f7744da74e5ef19966288a6bf76023e8e04e0c424d7d diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml new file mode 100644 index 0000000000..1bfc664928 --- /dev/null +++ b/.github/.OwlBot.yaml @@ -0,0 +1,8 @@ +# Copyright 2021 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +docker: + image: gcr.io/repo-automation-bots/owlbot-python:latest diff --git a/.github/workflows/django_tests_against_emulator0.yml b/.github/workflows/django_tests_against_emulator0.yml index 0f7f7c99db..2b31049e72 100644 --- a/.github/workflows/django_tests_against_emulator0.yml +++ b/.github/workflows/django_tests_against_emulator0.yml @@ -1,7 +1,7 @@ on: push: branches: - - master + - main pull_request: name: django-tests0 jobs: diff --git a/.github/workflows/django_tests_against_emulator1.yml b/.github/workflows/django_tests_against_emulator1.yml index 921a9a5848..722a593af1 100644 --- a/.github/workflows/django_tests_against_emulator1.yml +++ b/.github/workflows/django_tests_against_emulator1.yml @@ -1,7 +1,7 @@ on: push: branches: - - master + - main pull_request: name: django-tests1 jobs: diff --git a/.github/workflows/django_tests_against_emulator2.yml b/.github/workflows/django_tests_against_emulator2.yml index 682de38ed6..364803ab3a 100644 --- a/.github/workflows/django_tests_against_emulator2.yml +++ b/.github/workflows/django_tests_against_emulator2.yml @@ -1,7 +1,7 @@ on: push: branches: - - master + - main pull_request: name: django-tests2 jobs: diff --git a/.github/workflows/django_tests_against_emulator3.yml b/.github/workflows/django_tests_against_emulator3.yml index ea83185a82..9e2df8ae94 100644 --- a/.github/workflows/django_tests_against_emulator3.yml +++ b/.github/workflows/django_tests_against_emulator3.yml @@ -1,7 +1,7 @@ on: push: branches: - - master + - main pull_request: name: django-tests3 jobs: diff --git a/.github/workflows/django_tests_against_emulator4.yml b/.github/workflows/django_tests_against_emulator4.yml index 6a59ff7725..361dccfbc7 100644 --- a/.github/workflows/django_tests_against_emulator4.yml +++ b/.github/workflows/django_tests_against_emulator4.yml @@ -1,7 +1,7 @@ on: push: branches: - - master + - main pull_request: name: django-tests4 jobs: diff --git a/.github/workflows/django_tests_against_emulator5.yml b/.github/workflows/django_tests_against_emulator5.yml index f07609fb2f..817b0c4295 100644 --- a/.github/workflows/django_tests_against_emulator5.yml +++ b/.github/workflows/django_tests_against_emulator5.yml @@ -1,7 +1,7 @@ on: push: branches: - - master + - main pull_request: name: django-tests5 jobs: diff --git a/.github/workflows/django_tests_against_emulator6.yml b/.github/workflows/django_tests_against_emulator6.yml index 3984852de2..ab9d817c98 100644 --- a/.github/workflows/django_tests_against_emulator6.yml +++ b/.github/workflows/django_tests_against_emulator6.yml @@ -1,7 +1,7 @@ on: push: branches: - - master + - main pull_request: name: django-tests6 jobs: diff --git a/.github/workflows/django_tests_against_emulator7.yml b/.github/workflows/django_tests_against_emulator7.yml index 0932197f86..4ee19f9373 100644 --- a/.github/workflows/django_tests_against_emulator7.yml +++ b/.github/workflows/django_tests_against_emulator7.yml @@ -1,7 +1,7 @@ on: push: branches: - - master + - main pull_request: name: django-tests7 jobs: diff --git a/.github/workflows/django_tests_against_emulator8.yml b/.github/workflows/django_tests_against_emulator8.yml index 98d4b5f1d8..5c148ddd23 100644 --- a/.github/workflows/django_tests_against_emulator8.yml +++ b/.github/workflows/django_tests_against_emulator8.yml @@ -1,7 +1,7 @@ on: push: branches: - - master + - main pull_request: name: django-tests8 jobs: diff --git a/.github/workflows/django_tests_against_emulator9.yml b/.github/workflows/django_tests_against_emulator9.yml index 0ba4baefce..4388c3cfa6 100644 --- a/.github/workflows/django_tests_against_emulator9.yml +++ b/.github/workflows/django_tests_against_emulator9.yml @@ -1,7 +1,7 @@ on: push: branches: - - master + - main pull_request: name: django-tests9 jobs: diff --git a/.github/workflows/integration-tests-against-emulator.yml b/.github/workflows/integration-tests-against-emulator.yml index efd2c4e5bb..5767e966bb 100644 --- a/.github/workflows/integration-tests-against-emulator.yml +++ b/.github/workflows/integration-tests-against-emulator.yml @@ -1,7 +1,7 @@ on: push: branches: - - master + - main pull_request: name: Run Django Spanner integration tests against emulator jobs: diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 4ac9dd8b76..eea1d84933 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -1,17 +1,9 @@ #!/bin/bash # Copyright 2020 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd set -eo pipefail diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile index 412b0b56a9..92ade8f99f 100644 --- a/.kokoro/docker/docs/Dockerfile +++ b/.kokoro/docker/docs/Dockerfile @@ -1,16 +1,8 @@ # Copyright 2020 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd from ubuntu:20.04 @@ -40,6 +32,7 @@ RUN apt-get update \ libssl-dev \ libsqlite3-dev \ portaudio19-dev \ + python3-distutils \ redis-server \ software-properties-common \ ssh \ @@ -59,40 +52,8 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && rm -f /var/cache/apt/archives/*.deb - -COPY fetch_gpg_keys.sh /tmp -# Install the desired versions of Python. -RUN set -ex \ - && export GNUPGHOME="$(mktemp -d)" \ - && echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \ - && /tmp/fetch_gpg_keys.sh \ - && for PYTHON_VERSION in 3.7.8 3.8.5; do \ - wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ - && wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \ - && gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \ - && rm -r python-${PYTHON_VERSION}.tar.xz.asc \ - && mkdir -p /usr/src/python-${PYTHON_VERSION} \ - && tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \ - && rm python-${PYTHON_VERSION}.tar.xz \ - && cd /usr/src/python-${PYTHON_VERSION} \ - && ./configure \ - --enable-shared \ - # This works only on Python 2.7 and throws a warning on every other - # version, but seems otherwise harmless. - --enable-unicode=ucs4 \ - --with-system-ffi \ - --without-ensurepip \ - && make -j$(nproc) \ - && make install \ - && ldconfig \ - ; done \ - && rm -rf "${GNUPGHOME}" \ - && rm -rf /usr/src/python* \ - && rm -rf ~/.cache/ - RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ - && python3.7 /tmp/get-pip.py \ && python3.8 /tmp/get-pip.py \ && rm /tmp/get-pip.py -CMD ["python3.7"] +CMD ["python3.8"] diff --git a/.kokoro/docker/docs/fetch_gpg_keys.sh b/.kokoro/docker/docs/fetch_gpg_keys.sh index d653dd868e..c26695ddf0 100755 --- a/.kokoro/docker/docs/fetch_gpg_keys.sh +++ b/.kokoro/docker/docs/fetch_gpg_keys.sh @@ -1,21 +1,13 @@ #!/bin/bash # Copyright 2020 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd # A script to fetch gpg keys with retry. # Avoid jinja parsing the file. -# +# function retry { if [[ "${#}" -le 1 ]]; then @@ -42,4 +34,4 @@ retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \ retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \ E3FF2839C048B25C084DEBE9B26995E310250568 -# +# diff --git a/.kokoro/docs/common.cfg b/.kokoro/docs/common.cfg index 2003193838..cdf8e2a4ef 100644 --- a/.kokoro/docs/common.cfg +++ b/.kokoro/docs/common.cfg @@ -62,4 +62,4 @@ before_action { keyname: "docuploader_service_account" } } -} +} \ No newline at end of file diff --git a/.kokoro/populate-secrets.sh b/.kokoro/populate-secrets.sh new file mode 100755 index 0000000000..7b511c3257 --- /dev/null +++ b/.kokoro/populate-secrets.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +set -eo pipefail + +function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;} +function msg { println "$*" >&2 ;} +function println { printf '%s\n' "$(now) $*" ;} + + +# Populates requested secrets set in SECRET_MANAGER_KEYS from service account: +# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com +SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" +msg "Creating folder on disk for secrets: ${SECRET_LOCATION}" +mkdir -p ${SECRET_LOCATION} +for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g") +do + msg "Retrieving secret ${key}" + docker run --entrypoint=gcloud \ + --volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \ + gcr.io/google.com/cloudsdktool/cloud-sdk \ + secrets versions access latest \ + --project cloud-devrel-kokoro-resources \ + --secret ${key} > \ + "${SECRET_LOCATION}/${key}" + if [[ $? == 0 ]]; then + msg "Secret written to ${SECRET_LOCATION}/${key}" + else + msg "Error retrieving secret ${key}" + fi +done diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh index 8acb14e802..a80ea7c0a0 100755 --- a/.kokoro/publish-docs.sh +++ b/.kokoro/publish-docs.sh @@ -1,17 +1,9 @@ #!/bin/bash # Copyright 2020 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd set -eo pipefail diff --git a/.kokoro/release.sh b/.kokoro/release.sh index 9f60d5f251..57e2f28ce1 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -1,19 +1,9 @@ #!/bin/bash # Copyright 2020 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#!/bin/bash +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd set -eo pipefail @@ -28,7 +18,7 @@ python3 -m pip install --upgrade twine wheel setuptools export PYTHONUNBUFFERED=1 # Move into the package, build the distribution and upload. -TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google_cloud_pypi_password") +TWINE_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secret_manager/google-cloud-pypi-token") cd github/python-spanner-django python3 setup.py sdist bdist_wheel -twine upload --username gcloudpypi --password "${TWINE_PASSWORD}" dist/* +twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg index 31533b977a..2cf743579d 100644 --- a/.kokoro/release/common.cfg +++ b/.kokoro/release/common.cfg @@ -23,42 +23,8 @@ env_vars: { value: "github/python-spanner-django/.kokoro/release.sh" } -# Fetch the token needed for reporting release status to GitHub -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "yoshi-automation-github-key" - } - } -} - -# Fetch PyPI password -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "google_cloud_pypi_password" - } - } -} - -# Fetch magictoken to use with Magic Github Proxy -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "releasetool-magictoken" - } - } -} - -# Fetch api key to use with Magic Github Proxy -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "magic-github-proxy-api-key" - } - } +# Tokens needed to report release status back to GitHub +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,google-cloud-pypi-token" } diff --git a/.kokoro/release/release.cfg b/.kokoro/release/release.cfg index 18a4c35325..8f43917d92 100644 --- a/.kokoro/release/release.cfg +++ b/.kokoro/release/release.cfg @@ -1 +1 @@ -# Format: //devtools/kokoro/config/proto/build.proto +# Format: //devtools/kokoro/config/proto/build.proto \ No newline at end of file diff --git a/.kokoro/samples/lint/common.cfg b/.kokoro/samples/lint/common.cfg new file mode 100644 index 0000000000..b42e774271 --- /dev/null +++ b/.kokoro/samples/lint/common.cfg @@ -0,0 +1,34 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "lint" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-spanner-django/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-spanner-django/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/.kokoro/samples/lint/continuous.cfg b/.kokoro/samples/lint/continuous.cfg new file mode 100644 index 0000000000..a1c8d9759c --- /dev/null +++ b/.kokoro/samples/lint/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/lint/periodic.cfg b/.kokoro/samples/lint/periodic.cfg new file mode 100644 index 0000000000..50fec96497 --- /dev/null +++ b/.kokoro/samples/lint/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/.kokoro/samples/lint/presubmit.cfg b/.kokoro/samples/lint/presubmit.cfg new file mode 100644 index 0000000000..a1c8d9759c --- /dev/null +++ b/.kokoro/samples/lint/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.6/common.cfg b/.kokoro/samples/python3.6/common.cfg new file mode 100644 index 0000000000..6e32d40d20 --- /dev/null +++ b/.kokoro/samples/python3.6/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.6" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-py36" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-spanner-django/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-spanner-django/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.6/continuous.cfg b/.kokoro/samples/python3.6/continuous.cfg new file mode 100644 index 0000000000..7218af1499 --- /dev/null +++ b/.kokoro/samples/python3.6/continuous.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + diff --git a/.kokoro/samples/python3.6/periodic-head.cfg b/.kokoro/samples/python3.6/periodic-head.cfg new file mode 100644 index 0000000000..f9cfcd33e0 --- /dev/null +++ b/.kokoro/samples/python3.6/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.6/periodic.cfg b/.kokoro/samples/python3.6/periodic.cfg new file mode 100644 index 0000000000..50fec96497 --- /dev/null +++ b/.kokoro/samples/python3.6/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.6/presubmit.cfg b/.kokoro/samples/python3.6/presubmit.cfg new file mode 100644 index 0000000000..a1c8d9759c --- /dev/null +++ b/.kokoro/samples/python3.6/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.7/common.cfg b/.kokoro/samples/python3.7/common.cfg new file mode 100644 index 0000000000..407c854ff0 --- /dev/null +++ b/.kokoro/samples/python3.7/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.7" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-py37" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-spanner-django/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-spanner-django/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.7/continuous.cfg b/.kokoro/samples/python3.7/continuous.cfg new file mode 100644 index 0000000000..a1c8d9759c --- /dev/null +++ b/.kokoro/samples/python3.7/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.7/periodic-head.cfg b/.kokoro/samples/python3.7/periodic-head.cfg new file mode 100644 index 0000000000..f9cfcd33e0 --- /dev/null +++ b/.kokoro/samples/python3.7/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.7/periodic.cfg b/.kokoro/samples/python3.7/periodic.cfg new file mode 100644 index 0000000000..50fec96497 --- /dev/null +++ b/.kokoro/samples/python3.7/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.7/presubmit.cfg b/.kokoro/samples/python3.7/presubmit.cfg new file mode 100644 index 0000000000..a1c8d9759c --- /dev/null +++ b/.kokoro/samples/python3.7/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.8/common.cfg b/.kokoro/samples/python3.8/common.cfg new file mode 100644 index 0000000000..07b0e8fddb --- /dev/null +++ b/.kokoro/samples/python3.8/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.8" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-py38" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-spanner-django/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-spanner-django/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.8/continuous.cfg b/.kokoro/samples/python3.8/continuous.cfg new file mode 100644 index 0000000000..a1c8d9759c --- /dev/null +++ b/.kokoro/samples/python3.8/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.8/periodic-head.cfg b/.kokoro/samples/python3.8/periodic-head.cfg new file mode 100644 index 0000000000..f9cfcd33e0 --- /dev/null +++ b/.kokoro/samples/python3.8/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.8/periodic.cfg b/.kokoro/samples/python3.8/periodic.cfg new file mode 100644 index 0000000000..50fec96497 --- /dev/null +++ b/.kokoro/samples/python3.8/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.8/presubmit.cfg b/.kokoro/samples/python3.8/presubmit.cfg new file mode 100644 index 0000000000..a1c8d9759c --- /dev/null +++ b/.kokoro/samples/python3.8/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.9/common.cfg b/.kokoro/samples/python3.9/common.cfg new file mode 100644 index 0000000000..0c8833cfa5 --- /dev/null +++ b/.kokoro/samples/python3.9/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.9" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-py39" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-spanner-django/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-spanner-django/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.9/continuous.cfg b/.kokoro/samples/python3.9/continuous.cfg new file mode 100644 index 0000000000..a1c8d9759c --- /dev/null +++ b/.kokoro/samples/python3.9/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.9/periodic-head.cfg b/.kokoro/samples/python3.9/periodic-head.cfg new file mode 100644 index 0000000000..f9cfcd33e0 --- /dev/null +++ b/.kokoro/samples/python3.9/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.9/periodic.cfg b/.kokoro/samples/python3.9/periodic.cfg new file mode 100644 index 0000000000..50fec96497 --- /dev/null +++ b/.kokoro/samples/python3.9/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.9/presubmit.cfg b/.kokoro/samples/python3.9/presubmit.cfg new file mode 100644 index 0000000000..a1c8d9759c --- /dev/null +++ b/.kokoro/samples/python3.9/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/test-samples-against-head.sh b/.kokoro/test-samples-against-head.sh new file mode 100755 index 0000000000..104d0e40bb --- /dev/null +++ b/.kokoro/test-samples-against-head.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + + +# A customized test runner for samples. +# +# For periodic builds, you can specify this file for testing against head. + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +cd github/python-spanner-django + +exec .kokoro/test-samples-impl.sh diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh new file mode 100755 index 0000000000..595dca8b0d --- /dev/null +++ b/.kokoro/test-samples-impl.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +# Exit early if samples don't exist +if ! find samples -name 'requirements.txt' | grep -q .; then + echo "No tests run. './samples/**/requirements.txt' not found" + exit 0 +fi + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Debug: show build environment +env | grep KOKORO + +# Install nox +python3.6 -m pip install --upgrade --quiet nox + +# Use secrets acessor service account to get secrets +if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then + gcloud auth activate-service-account \ + --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \ + --project="cloud-devrel-kokoro-resources" +fi + +# This script will create 3 files: +# - testing/test-env.sh +# - testing/service-account.json +# - testing/client-secrets.json +./scripts/decrypt-secrets.sh + +source ./testing/test-env.sh +export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json + +# For cloud-run session, we activate the service account for gcloud sdk. +gcloud auth activate-service-account \ + --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" + +export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json + +echo -e "\n******************** TESTING PROJECTS ********************" + +# Switch to 'fail at end' to allow all tests to complete before exiting. +set +e +# Use RTN to return a non-zero value if the test fails. +RTN=0 +ROOT=$(pwd) +# Find all requirements.txt in the samples directory (may break on whitespace). +for file in samples/**/requirements.txt; do + cd "$ROOT" + # Navigate to the project folder. + file=$(dirname "$file") + cd "$file" + + echo "------------------------------------------------------------" + echo "- testing $file" + echo "------------------------------------------------------------" + + # Use nox to execute the tests for the project. + python3.6 -m nox -s "$RUN_TESTS_SESSION" + EXIT=$? + + # If this is a periodic build, send the test log to the FlakyBot. + # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. + if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + fi + + if [[ $EXIT -ne 0 ]]; then + RTN=1 + echo -e "\n Testing failed: Nox returned a non-zero exit code. \n" + else + echo -e "\n Testing completed.\n" + fi + +done +cd "$ROOT" + +# Workaround for Kokoro permissions issue: delete secrets +rm testing/{test-env.sh,client-secrets.json,service-account.json} + +exit "$RTN" diff --git a/.kokoro/test-samples.sh b/.kokoro/test-samples.sh new file mode 100755 index 0000000000..64bd21dd92 --- /dev/null +++ b/.kokoro/test-samples.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + + +# The default test runner for samples. +# +# For periodic builds, we rewinds the repo to the latest release, and +# run test-samples-impl.sh. + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +cd github/python-spanner-django + +# Run periodic samples tests at latest release +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + # preserving the test runner implementation. + cp .kokoro/test-samples-impl.sh "${TMPDIR}/test-samples-impl.sh" + echo "--- IMPORTANT IMPORTANT IMPORTANT ---" + echo "Now we rewind the repo back to the latest release..." + LATEST_RELEASE=$(git describe --abbrev=0 --tags) + git checkout $LATEST_RELEASE + echo "The current head is: " + echo $(git rev-parse --verify HEAD) + echo "--- IMPORTANT IMPORTANT IMPORTANT ---" + # move back the test runner implementation if there's no file. + if [ ! -f .kokoro/test-samples-impl.sh ]; then + cp "${TMPDIR}/test-samples-impl.sh" .kokoro/test-samples-impl.sh + fi +fi + +exec .kokoro/test-samples-impl.sh diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh index 8f0237f322..d42a40acbb 100755 --- a/.kokoro/trampoline.sh +++ b/.kokoro/trampoline.sh @@ -1,23 +1,20 @@ #!/bin/bash -# Copyright 2020 Google LLC. +# Copyright 2017 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd set -eo pipefail -python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" || ret_code=$? - -chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh -${KOKORO_GFILE_DIR}/trampoline_cleanup.sh || true +# Always run the cleanup script, regardless of the success of bouncing into +# the container. +function cleanup() { + chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + echo "cleanup"; +} +trap cleanup EXIT -exit ${ret_code} +$(dirname $0)/populate-secrets.sh # Secret Manager secrets. +python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" \ No newline at end of file diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh index 4af6cdc26d..591faf816e 100755 --- a/.kokoro/trampoline_v2.sh +++ b/.kokoro/trampoline_v2.sh @@ -1,17 +1,9 @@ #!/usr/bin/env bash # Copyright 2020 Google LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd # trampoline_v2.sh # diff --git a/.trampolinerc b/.trampolinerc index 28037bbb86..383b6ec89f 100644 --- a/.trampolinerc +++ b/.trampolinerc @@ -1,4 +1,4 @@ -# Copyright 2021 Google LLC +# Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d1d5d4875..a370dd14c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,35 @@ # Changelog +## 2.2.1b3 (2021-07-30) + +### Miscellaneous Chores +- release 2.2.1b3 ([de23f65](https://www.github.com/googleapis/python-spanner-django/commit/de23f65)) +- Update repo to say beta release instead of alpha ([#691](https://www.github.com/googleapis/python-spanner-django/issues/691)) ([2144d09](https://www.github.com/googleapis/python-spanner-django/commit/2144d09)) + + +## 2.2.1b2 (2021-07-27) + + +### Features +- Added support for check constraint ([#679](https://www.github.com/googleapis/python-spanner-django/issues/679)) ([42352c0](https://www.github.com/googleapis/python-spanner-django/commit/42352c0)) +- Add open telemetry trace in schema and related unit tests ([#648](https://www.github.com/googleapis/python-spanner-django/issues/648)) ([fc51086](https://www.github.com/googleapis/python-spanner-django/commit/fc51086)) + + +### Bug Fixes +- updated assets to have text background so it works with dark mode ([#674](https://www.github.com/googleapis/python-spanner-django/issues/674)) ([306eeba](https://www.github.com/googleapis/python-spanner-django/commit/306eeba)) +- updated assets to have text background so it works with dark mode ([#671](https://www.github.com/googleapis/python-spanner-django/issues/671)) ([0f99938](https://www.github.com/googleapis/python-spanner-django/commit/0f99938)) +- bump version number after 2.2.1b1 release ([#652](https://www.github.com/googleapis/python-spanner-django/issues/652)) ([287b893](https://www.github.com/googleapis/python-spanner-django/commit/287b893)) + + +### Documentation +- update docs to show decimal field support and check constraints but no support for unsigned data type ([#683](https://www.github.com/googleapis/python-spanner-django/issues/683)) ([74f2269](https://www.github.com/googleapis/python-spanner-django/commit/74f2269)) +- Adding documentation for GA ([#665](https://www.github.com/googleapis/python-spanner-django/issues/665)) ([216c2e0](https://www.github.com/googleapis/python-spanner-django/commit/216c2e0)) + + +### Miscellaneous Chores +- release 2.2.1b2 ([#685](https://www.github.com/googleapis/python-spanner-django/issues/685)) ([96a809d](https://www.github.com/googleapis/python-spanner-django/commit/96a809d)) +- fix release build ([#659](https://www.github.com/googleapis/python-spanner-django/issues/659)) ([11bc9c2](https://www.github.com/googleapis/python-spanner-django/commit/11bc9c2)) + ## 2.2.1b1 (2021-06-17) diff --git a/README.rst b/README.rst index e79e23cd91..62739a3182 100644 --- a/README.rst +++ b/README.rst @@ -220,13 +220,13 @@ How it works Overall design ~~~~~~~~~~~~~~ -.. figure:: https://raw.githubusercontent.com/googleapis/python-spanner-django/master/assets/overview.png +.. figure:: https://raw.githubusercontent.com/googleapis/python-spanner-django/main/assets/overview.png :alt: "Overall Design" Internals ~~~~~~~~~ -.. figure:: https://raw.githubusercontent.com/googleapis/python-spanner-django/master/assets/internals.png +.. figure:: https://raw.githubusercontent.com/googleapis/python-spanner-django/main/assets/internals.png :alt: "Internals" @@ -247,11 +247,11 @@ HOW TO CONTRIBUTE Contributions to this library are always welcome and highly encouraged. -See `CONTRIBUTING `_ for more information on how to get started. +See `CONTRIBUTING `_ for more information on how to get started. Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See the `Code -of Conduct `_ for more information. +of Conduct `_ for more information. Current limitations ------------------- @@ -283,23 +283,16 @@ were created. Spanner does not support ``ON DELETE CASCADE`` when creating foreign-key constraints, so this is not supported in ``django-google-spanner``. -Check constraints aren't supported -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``Unsigned`` datatypes are not supported +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Spanner does not support ``CHECK`` constraints so one isn't created for -`PositiveIntegerField +Spanner does not support ``Unsigned`` datatypes so `PositiveIntegerField `__ -and `CheckConstraint -`__ -can't be used. - -No native support for ``DecimalField`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Spanner's support for `Decimal `__ -types is limited to -`NUMERIC `__ -precision. Higher-precision values can be stored as strings instead. +and `PositiveSmallIntegerField +`__ +are both stored as `Integer type +`__ +. ``Meta.order_with_respect_to`` model option isn't supported ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/assets/overview.png b/assets/overview.png index a556727a18..1fc2933e51 100644 Binary files a/assets/overview.png and b/assets/overview.png differ diff --git a/django_spanner/__init__.py b/django_spanner/__init__.py index 861e3abb94..a26703d5a5 100644 --- a/django_spanner/__init__.py +++ b/django_spanner/__init__.py @@ -5,6 +5,7 @@ # https://developers.google.com/open-source/licenses/bsd import datetime +import os # Monkey-patch AutoField to generate a random value since Cloud Spanner can't # do that. @@ -24,6 +25,8 @@ __version__ = pkg_resources.get_distribution("django-google-spanner").version +USE_EMULATOR = os.getenv("SPANNER_EMULATOR_HOST") is not None + check_django_compatability() register_expressions() register_functions() diff --git a/django_spanner/features.py b/django_spanner/features.py index af7e4c1131..a0ae6299c3 100644 --- a/django_spanner/features.py +++ b/django_spanner/features.py @@ -38,6 +38,8 @@ class DatabaseFeatures(BaseDatabaseFeatures): skip_tests = ( # No foreign key constraints in Spanner. "backends.tests.FkConstraintsTests.test_check_constraints", + # Spanner does not support empty list of DML statement. + "backends.tests.BackendTestCase.test_cursor_executemany_with_empty_params_list", "fixtures_regress.tests.TestFixtures.test_loaddata_raises_error_when_fixture_has_invalid_foreign_key", # No Django transaction management in Spanner. "basic.tests.SelectOnSaveTests.test_select_on_save_lying_update", @@ -184,7 +186,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): "db_functions.comparison.test_cast.CastTests.test_cast_to_decimal_field", "model_fields.test_decimalfield.DecimalFieldTests.test_fetch_from_db_without_float_rounding", "model_fields.test_decimalfield.DecimalFieldTests.test_roundtrip_with_trailing_zeros", - # No CHECK constraints in Spanner. + # Spanner does not support unsigned integer field. "model_fields.test_integerfield.PositiveIntegerFieldTests.test_negative_values", # Spanner doesn't support the variance the standard deviation database # functions: diff --git a/django_spanner/schema.py b/django_spanner/schema.py index d28dcc4f6e..247358857a 100644 --- a/django_spanner/schema.py +++ b/django_spanner/schema.py @@ -7,6 +7,7 @@ from django.db import NotSupportedError from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django_spanner._opentelemetry_tracing import trace_call +from django_spanner import USE_EMULATOR class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): @@ -472,8 +473,13 @@ def _alter_column_type_sql(self, model, old_field, new_field, new_type): ) def _check_sql(self, name, check): - # Spanner doesn't support CHECK constraints. - return None + # Emulator does not support check constraints yet. + if USE_EMULATOR: + return None + return self.sql_constraint % { + "name": self.quote_name(name), + "constraint": self.sql_check_constraint % {"check": check}, + } def _unique_sql(self, model, fields, name, condition=None): # Inline constraints aren't supported, so create the index separately. diff --git a/docs/changelog.md b/docs/changelog.md deleted file mode 100644 index 825c32f0d0..0000000000 --- a/docs/changelog.md +++ /dev/null @@ -1 +0,0 @@ -# Changelog diff --git a/docs/changelog.md b/docs/changelog.md new file mode 120000 index 0000000000..04c99a55ca --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1 @@ +../CHANGELOG.md \ No newline at end of file diff --git a/docs/example_from_scratch.md b/docs/example_from_scratch.md new file mode 120000 index 0000000000..1e40292b5e --- /dev/null +++ b/docs/example_from_scratch.md @@ -0,0 +1 @@ +../examples/from-scratch/README.md \ No newline at end of file diff --git a/docs/example_healthchecks.md b/docs/example_healthchecks.md new file mode 120000 index 0000000000..08983d890b --- /dev/null +++ b/docs/example_healthchecks.md @@ -0,0 +1 @@ +../examples/healthchecks/README.md \ No newline at end of file diff --git a/docs/samples.rst b/docs/samples.rst index 4d9ef417a2..09d8c37590 100644 --- a/docs/samples.rst +++ b/docs/samples.rst @@ -1,24 +1,12 @@ -Sample Code -#################################### +Sample Examples +############### -Create and register your first model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To define your database layout create a models file in your app folder and add the relevant -classes to it. Spanner works exactly like any other database you may have used with Django. -Here is a simple example you can run with Spanner. In our poll application below we create -the following two models: +django-spanner for Django tutorial +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. code:: python +This `Example `_ shows how to use django-spanner for Cloud Spanner as a backend database for `healthchecks.io `_ - from django.db import models - - class Question(models.Model): - question_text = models.CharField(max_length=200) - pub_date = models.DateTimeField('date published') - def __str__(self): - return str(self.rating) - - class Choice(models.Model): - question = models.ForeignKey(Question, on_delete=models.CASCADE) - choice_text = models.CharField(max_length=200) - votes = models.IntegerField(default=0) +django-spanner on healthchecks.io +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This `Example `_ shows how to use django-spanner for Cloud Spanner as a backend database for `Django's tutorials `_ diff --git a/noxfile.py b/noxfile.py index a5c05e7a02..3b51d73841 100644 --- a/noxfile.py +++ b/noxfile.py @@ -43,7 +43,7 @@ def lint(session): session.run("flake8", "django_spanner", "tests") -@nox.session(python="3.6") +@nox.session(python=DEFAULT_PYTHON_VERSION) def blacken(session): """Run black. diff --git a/owlbot.py b/owlbot.py new file mode 100644 index 0000000000..d0087b4c0b --- /dev/null +++ b/owlbot.py @@ -0,0 +1,42 @@ +# Copyright 2021 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""This script is used to synthesize generated parts of this library.""" +import re + +import synthtool as s +import synthtool.gcp as gcp + +# ---------------------------------------------------------------------------- +# Add templated files +# ---------------------------------------------------------------------------- +templated_files = gcp.CommonTemplates().py_library(microgenerator=True) + +# Just move templates for building docs and releases +# Presubmit and continuous are configured differently +s.move(templated_files / ".trampolinerc") +s.move(templated_files / ".kokoro" / "docker") +s.move(templated_files / ".kokoro" / "docs") +s.move(templated_files / ".kokoro" / "release.sh") +s.move(templated_files / ".kokoro" / "trampoline_v2.sh") +s.move(templated_files / ".kokoro" / "trampoline.sh") +s.move(templated_files / ".kokoro" / "populate-secrets.sh") +s.move(templated_files / ".kokoro" / "release") + +# Replace the Apache Licenses in the `.kokoro` directory +# with the BSD license expected in this repository +s.replace( + ".kokoro/**/*", + "# Copyright.*(\d{4}).*# limitations under the License\.", + """# Copyright \g<1> Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd""", + flags=re.DOTALL +) + +s.shell.run(["nox", "-s", "blacken"], hide_output=False) diff --git a/setup.py b/setup.py index c310fda167..a3143566bb 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ # 'Development Status :: 3 - Alpha' # 'Development Status :: 4 - Beta' # 'Development Status :: 5 - Production/Stable' -release_status = "Development Status :: 3 - Alpha" +release_status = "Development Status :: 4 - Beta" dependencies = ["sqlparse >= 0.3.0", "google-cloud-spanner >= 3.0.0"] extras = { "tracing": [ diff --git a/tests/performance/__init__.py b/tests/performance/__init__.py new file mode 100644 index 0000000000..529352b757 --- /dev/null +++ b/tests/performance/__init__.py @@ -0,0 +1,5 @@ +# Copyright 2021 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd diff --git a/tests/performance/django_spanner/__init__.py b/tests/performance/django_spanner/__init__.py new file mode 100644 index 0000000000..529352b757 --- /dev/null +++ b/tests/performance/django_spanner/__init__.py @@ -0,0 +1,5 @@ +# Copyright 2021 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd diff --git a/tests/performance/django_spanner/models.py b/tests/performance/django_spanner/models.py new file mode 100644 index 0000000000..d387a6ef86 --- /dev/null +++ b/tests/performance/django_spanner/models.py @@ -0,0 +1,14 @@ +# Copyright 2021 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +from django.db import models + + +class Author(models.Model): + id = models.IntegerField(primary_key=True) + first_name = models.CharField(max_length=20) + last_name = models.CharField(max_length=20) + rating = models.CharField(max_length=50) diff --git a/tests/performance/django_spanner/test_benchmark.py b/tests/performance/django_spanner/test_benchmark.py new file mode 100644 index 0000000000..31dd8f2987 --- /dev/null +++ b/tests/performance/django_spanner/test_benchmark.py @@ -0,0 +1,296 @@ +# Copyright 2021 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +import random +import time +import unittest + +import pandas as pd +import pytest +from django.db import connection +from google.api_core.exceptions import Aborted +from google.cloud import spanner_dbapi +from google.cloud.spanner_v1 import Client, KeySet + +from tests.performance.django_spanner.models import Author +from tests.settings import DATABASE_NAME, INSTANCE_ID +from tests.system.django_spanner.utils import setup_database, setup_instance + + +def measure_execution_time(function): + """Decorator to measure a wrapped method execution time.""" + + def wrapper(self, measures): + """Execute the wrapped method and measure its execution time. + Args: + measures (dict): Test cases and their execution time. + """ + t_start = time.time() + try: + function(self) + measures[function.__name__] = round(time.time() - t_start, 4) + except Aborted: + measures[function.__name__] = 0 + + return wrapper + + +def insert_one_row(transaction, one_row): + """A transaction-function for the original Spanner client. + Inserts a single row into a database and then fetches it back. + """ + transaction.execute_update( + "INSERT Author (id, first_name, last_name, rating) " + " VALUES {}".format(str(one_row)) + ) + last_name = transaction.execute_sql( + "SELECT last_name FROM Author WHERE id=1" + ).one()[0] + if last_name != "Allison": + raise ValueError("Received invalid last name: " + last_name) + + +def insert_many_rows(transaction, many_rows): + """A transaction-function for the original Spanner client. + Insert 100 rows into a database. + """ + statements = [] + for row in many_rows: + statements.append( + "INSERT Author (id, first_name, last_name, rating) " + " VALUES {}".format(str(row)) + ) + _, count = transaction.batch_update(statements) + if sum(count) != 99: + raise ValueError("Wrong number of inserts: " + str(sum(count))) + + +class DjangoBenchmarkTest: + """The Django performace testing class.""" + + def __init__(self): + with connection.schema_editor() as editor: + # Create the tables + editor.create_model(Author) + + self._many_rows = [] + self._many_rows2 = [] + for i in range(99): + num = round(random.randint(0, 100000000)) + self._many_rows.append(Author(num, "Pete", "Allison", "2.1")) + num2 = round(random.randint(0, 100000000)) + self._many_rows2.append(Author(num2, "Pete", "Allison", "2.1")) + + @measure_execution_time + def insert_one_row_with_fetch_after(self): + author_kent = Author( + id=2, first_name="Pete", last_name="Allison", rating="2.1", + ) + author_kent.save() + last_name = Author.objects.get(pk=author_kent.id).last_name + if last_name != "Allison": + raise ValueError("Received invalid last name: " + last_name) + + @measure_execution_time + def insert_many_rows(self): + for row in self._many_rows: + row.save() + + @measure_execution_time + def insert_many_rows_with_mutations(self): + Author.objects.bulk_create(self._many_rows2) + + @measure_execution_time + def read_one_row(self): + row = Author.objects.all().first() + if row is None: + raise ValueError("No rows read") + + @measure_execution_time + def select_many_rows(self): + rows = Author.objects.all() + if len(rows) != 100: + raise ValueError("Wrong number of rows read") + + def _cleanup(self): + """Drop the test table.""" + with connection.schema_editor() as editor: + # delete the table + editor.delete_model(Author) + + def run(self): + """Execute every test case.""" + measures = {} + for method in ( + self.insert_one_row_with_fetch_after, + self.read_one_row, + self.insert_many_rows, + self.select_many_rows, + self.insert_many_rows_with_mutations, + ): + method(measures) + self._cleanup() + return measures + + +class SpannerBenchmarkTest: + """The Spanner performace testing class.""" + + def __init__(self): + self._create_table() + self._one_row = ( + 1, + "Pete", + "Allison", + "2.1", + ) + self._client = Client() + self._instance = self._client.instance(INSTANCE_ID) + self._database = self._instance.database(DATABASE_NAME) + + self._many_rows = [] + self._many_rows2 = [] + for i in range(99): + num = round(random.randint(0, 100000000)) + self._many_rows.append((num, "Pete", "Allison", "2.1")) + num2 = round(random.randint(0, 100000000)) + self._many_rows2.append((num2, "Pete", "Allison", "2.1")) + + # initiate a session + with self._database.snapshot(): + pass + + def _create_table(self): + """Create a table for performace testing.""" + conn = spanner_dbapi.connect(INSTANCE_ID, DATABASE_NAME) + conn.database.update_ddl( + [ + """ +CREATE TABLE Author ( + id INT64, + first_name STRING(20), + last_name STRING(20), + rating STRING(50), +) PRIMARY KEY (id) + """ + ] + ).result(120) + + conn.close() + + @measure_execution_time + def insert_one_row_with_fetch_after(self): + self._database.run_in_transaction(insert_one_row, self._one_row) + + @measure_execution_time + def insert_many_rows(self): + self._database.run_in_transaction(insert_many_rows, self._many_rows) + + @measure_execution_time + def insert_many_rows_with_mutations(self): + with self._database.batch() as batch: + batch.insert( + table="Author", + columns=("id", "first_name", "last_name", "rating"), + values=self._many_rows2, + ) + + @measure_execution_time + def read_one_row(self): + with self._database.snapshot() as snapshot: + keyset = KeySet(all_=True) + snapshot.read( + table="Author", + columns=("id", "first_name", "last_name", "rating"), + keyset=keyset, + ).one() + + @measure_execution_time + def select_many_rows(self): + with self._database.snapshot() as snapshot: + rows = list( + snapshot.execute_sql("SELECT * FROM Author ORDER BY last_name") + ) + if len(rows) != 100: + raise ValueError("Wrong number of rows read") + + def _cleanup(self): + """Drop the test table.""" + conn = spanner_dbapi.connect(INSTANCE_ID, DATABASE_NAME) + conn.database.update_ddl(["DROP TABLE Author"]) + conn.close() + + def run(self): + """Execute every test case.""" + measures = {} + for method in ( + self.insert_one_row_with_fetch_after, + self.read_one_row, + self.insert_many_rows, + self.select_many_rows, + self.insert_many_rows_with_mutations, + ): + method(measures) + self._cleanup() + return measures + + +@pytest.mark.django_db() +class BenchmarkTest(unittest.TestCase): + def setUp(self): + setup_instance() + setup_database() + + def test_run(self): + django_obj = pd.DataFrame( + columns=[ + "insert_one_row_with_fetch_after", + "read_one_row", + "insert_many_rows", + "select_many_rows", + "insert_many_rows_with_mutations", + ] + ) + spanner_obj = pd.DataFrame( + columns=[ + "insert_one_row_with_fetch_after", + "read_one_row", + "insert_many_rows", + "select_many_rows", + "insert_many_rows_with_mutations", + ] + ) + + for _ in range(50): + django_obj = django_obj.append( + DjangoBenchmarkTest().run(), ignore_index=True + ) + spanner_obj = spanner_obj.append( + SpannerBenchmarkTest().run(), ignore_index=True + ) + + avg = pd.concat( + [django_obj.mean(axis=0), spanner_obj.mean(axis=0)], axis=1 + ) + avg.columns = ["Django", "Spanner"] + std = pd.concat( + [django_obj.std(axis=0), spanner_obj.std(axis=0)], axis=1 + ) + std.columns = ["Django", "Spanner"] + err = pd.concat( + [django_obj.sem(axis=0), spanner_obj.sem(axis=0)], axis=1 + ) + err.columns = ["Django", "Spanner"] + + print( + "Average: ", + avg, + "Standard Deviation: ", + std, + "Error:", + err, + sep="\n", + ) diff --git a/tests/system/django_spanner/models.py b/tests/system/django_spanner/models.py index 5524ad8ec9..f7153ba994 100644 --- a/tests/system/django_spanner/models.py +++ b/tests/system/django_spanner/models.py @@ -21,3 +21,16 @@ class Number(models.Model): def __str__(self): return str(self.num) + + +class Event(models.Model): + start_date = models.DateTimeField() + end_date = models.DateTimeField() + + class Meta: + constraints = [ + models.CheckConstraint( + check=models.Q(end_date__gt=models.F("start_date")), + name="check_start_date", + ), + ] diff --git a/tests/system/django_spanner/test_check_constraint.py b/tests/system/django_spanner/test_check_constraint.py new file mode 100644 index 0000000000..9177166ce9 --- /dev/null +++ b/tests/system/django_spanner/test_check_constraint.py @@ -0,0 +1,64 @@ +# Copyright 2021 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +from .models import Event +from django.test import TransactionTestCase +import datetime +import unittest +from django.utils import timezone +from google.api_core.exceptions import OutOfRange +from django.db import connection +from django_spanner import USE_EMULATOR +from tests.system.django_spanner.utils import ( + setup_instance, + teardown_instance, + setup_database, + teardown_database, +) + + +@unittest.skipIf( + USE_EMULATOR, "Check Constraint is not implemented in emulator." +) +class TestCheckConstraint(TransactionTestCase): + @classmethod + def setUpClass(cls): + setup_instance() + setup_database() + with connection.schema_editor() as editor: + # Create the table + editor.create_model(Event) + + @classmethod + def tearDownClass(cls): + with connection.schema_editor() as editor: + # delete the table + editor.delete_model(Event) + teardown_database() + teardown_instance() + + def test_insert_valid_value(self): + """ + Tests model object creation with Event model. + """ + now = timezone.now() + now_plus_10 = now + datetime.timedelta(minutes=10) + event_valid = Event(start_date=now, end_date=now_plus_10) + event_valid.save() + qs1 = Event.objects.filter().values("start_date") + self.assertEqual(qs1[0]["start_date"], now) + # Delete data from Event table. + Event.objects.all().delete() + + def test_insert_invalid_value(self): + """ + Tests model object creation with invalid data in Event model. + """ + now = timezone.now() + now_minus_1_day = now - timezone.timedelta(days=1) + event_invalid = Event(start_date=now, end_date=now_minus_1_day) + with self.assertRaises(OutOfRange): + event_invalid.save() diff --git a/tests/system/django_spanner/test_decimal.py b/tests/system/django_spanner/test_decimal.py index 73df7e796b..4155599af1 100644 --- a/tests/system/django_spanner/test_decimal.py +++ b/tests/system/django_spanner/test_decimal.py @@ -6,14 +6,13 @@ from .models import Author, Number from django.test import TransactionTestCase -from django.db import connection, ProgrammingError +from django.db import connection from decimal import Decimal from tests.system.django_spanner.utils import ( setup_instance, teardown_instance, setup_database, teardown_database, - USE_EMULATOR, ) @@ -87,12 +86,8 @@ def test_decimal_precision_limit(self): Tests decimal object precission limit. """ num_val = Number(num=Decimal(1) / Decimal(3)) - if USE_EMULATOR: - with self.assertRaises(ValueError): - num_val.save() - else: - with self.assertRaises(ProgrammingError): - num_val.save() + with self.assertRaises(ValueError): + num_val.save() def test_decimal_update(self): """ diff --git a/tests/system/django_spanner/utils.py b/tests/system/django_spanner/utils.py index 7fac5166e0..3dca9db9b8 100644 --- a/tests/system/django_spanner/utils.py +++ b/tests/system/django_spanner/utils.py @@ -15,11 +15,12 @@ from test_utils.retry import RetryErrors from django_spanner.creation import DatabaseCreation +from django_spanner import USE_EMULATOR CREATE_INSTANCE = ( os.getenv("GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE") is not None ) -USE_EMULATOR = os.getenv("SPANNER_EMULATOR_HOST") is not None + SPANNER_OPERATION_TIMEOUT_IN_SECONDS = int( os.getenv("SPANNER_OPERATION_TIMEOUT_IN_SECONDS", 60) ) diff --git a/version.py b/version.py index df949c65d1..32ec82411a 100644 --- a/version.py +++ b/version.py @@ -4,4 +4,4 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd -__version__ = "2.2.1b2" +__version__ = "2.2.1b4"