From ff73fca4765f08baea18cb8dd4a957c8513535a9 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 4 May 2020 13:59:02 +1000 Subject: [PATCH 01/91] Cloud Run with Django tutorial --- run/django/.gcloudignore | 17 +++++ run/django/Dockerfile | 23 ++++++ run/django/README.md | 25 +++++++ run/django/cloudmigrate.yaml | 16 +++++ run/django/manage.py | 21 ++++++ run/django/mysite/__init__.py | 0 run/django/mysite/asgi.py | 16 +++++ run/django/mysite/settings.py | 131 ++++++++++++++++++++++++++++++++++ run/django/mysite/urls.py | 7 ++ run/django/mysite/wsgi.py | 16 +++++ run/django/polls/__init__.py | 0 run/django/polls/admin.py | 5 ++ run/django/polls/apps.py | 5 ++ run/django/polls/models.py | 12 ++++ run/django/polls/tests.py | 3 + run/django/polls/urls.py | 7 ++ run/django/polls/views.py | 5 ++ run/django/requirements.txt | 7 ++ 18 files changed, 316 insertions(+) create mode 100644 run/django/.gcloudignore create mode 100644 run/django/Dockerfile create mode 100644 run/django/README.md create mode 100644 run/django/cloudmigrate.yaml create mode 100755 run/django/manage.py create mode 100644 run/django/mysite/__init__.py create mode 100644 run/django/mysite/asgi.py create mode 100644 run/django/mysite/settings.py create mode 100644 run/django/mysite/urls.py create mode 100644 run/django/mysite/wsgi.py create mode 100644 run/django/polls/__init__.py create mode 100644 run/django/polls/admin.py create mode 100644 run/django/polls/apps.py create mode 100644 run/django/polls/models.py create mode 100644 run/django/polls/tests.py create mode 100644 run/django/polls/urls.py create mode 100644 run/django/polls/views.py create mode 100644 run/django/requirements.txt diff --git a/run/django/.gcloudignore b/run/django/.gcloudignore new file mode 100644 index 000000000000..c6be208a110f --- /dev/null +++ b/run/django/.gcloudignore @@ -0,0 +1,17 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore + +# Python pycache: +__pycache__/ diff --git a/run/django/Dockerfile b/run/django/Dockerfile new file mode 100644 index 000000000000..95e7953f86d9 --- /dev/null +++ b/run/django/Dockerfile @@ -0,0 +1,23 @@ +# Use an official lightweight Python image. +# https://hub.docker.com/_/python +FROM python:3.8-slim + +ENV APP_HOME /app +WORKDIR $APP_HOME + +# Install dependencies. +COPY requirements.txt . +RUN pip install -r requirements.txt + +# Copy local code to the container image. +COPY . . + +# Service must listen to $PORT environment variable. +# This default value facilitates local development. +ENV PORT 8080 + +# Run the web service on container startup. Here we use the gunicorn +# webserver, with one worker process and 8 threads. +# For environments with multiple CPU cores, increase the number of workers +# to be equal to the cores available. +CMD exec gunicorn --bind 0.0.0.0:$PORT --workers 1 --threads 8 --timeout 0 mysite.wsgi:application diff --git a/run/django/README.md b/run/django/README.md new file mode 100644 index 000000000000..c7e277de6676 --- /dev/null +++ b/run/django/README.md @@ -0,0 +1,25 @@ +# Getting started with Django on Google Cloud Platform on Cloud Run (fully managed) + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=run/django/README.md + +This repository is an example of how to run a [Django](https://www.djangoproject.com/) +app on Google Cloud Run (fully managed). It uses the +[Writing your first Django app](https://docs.djangoproject.com/en/3.0/#first-steps) ([Part 1](https://docs.djangoproject.com/en/3.0/intro/tutorial01/) and [Part 2](https://docs.djangoproject.com/en/3.0/intro/tutorial02/)) as the +example app to deploy. + + +# Tutorial +See our [Running Django on Cloud Run (fully managed)](https://cloud.google.com/python/django/run) tutorial for instructions for setting up and deploying this sample application. + + +## Contributing changes + +* See [CONTRIBUTING.md](CONTRIBUTING.md) + + +## Licensing + +* See [LICENSE](LICENSE) diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml new file mode 100644 index 000000000000..01a2173fb4a6 --- /dev/null +++ b/run/django/cloudmigrate.yaml @@ -0,0 +1,16 @@ +steps: +- name: "gcr.io/cloud-builders/docker" + args: ["build", "-t", "gcr.io/${PROJECT_ID}/polls", "."] + +- name: "gcr.io/cloud-builders/docker" + args: ["push", "gcr.io/${PROJECT_ID}/polls"] + +- name: "gcr.io/google-appengine/exec-wrapper" + args: ["-i", "gcr.io/$PROJECT_ID/polls", + "-s", "${PROJECT_ID}:us-central1:django-instance", + "--", "python", "manage.py", "migrate"] + +- name: "gcr.io/google-appengine/exec-wrapper" + args: ["-i", "gcr.io/$PROJECT_ID/polls", + "-s", "${PROJECT_ID}:us-central1:django-instance", + "--", "python", "manage.py", "collectstatic", "--no-input"] diff --git a/run/django/manage.py b/run/django/manage.py new file mode 100755 index 000000000000..341863cf62fa --- /dev/null +++ b/run/django/manage.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/run/django/mysite/__init__.py b/run/django/mysite/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/run/django/mysite/asgi.py b/run/django/mysite/asgi.py new file mode 100644 index 000000000000..35d925e4c5a8 --- /dev/null +++ b/run/django/mysite/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for mysite project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + +application = get_asgi_application() diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py new file mode 100644 index 000000000000..224f1671a2e5 --- /dev/null +++ b/run/django/mysite/settings.py @@ -0,0 +1,131 @@ +""" +Django settings for mysite project. + +Generated by 'django-admin startproject' using Django 3.0.5. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.0/ref/settings/ +""" +import os + +# [START secretconfig] +import environ +import google.auth +from google.cloud import secretmanager_v1beta1 as sm + +# Import settings with django-environ +env = environ.Env() + +# Import settings from Secret Manager +secrets = {} +_, project = google.auth.default() +client = sm.SecretManagerServiceClient() +parent = client.project_path(project) + +for secret in ["DATABASE_URL", "GS_BUCKET_NAME", "SECRET_KEY"]: + path = client.secret_version_path(project, secret, "latest") + payload = client.access_secret_version(path).payload.data.decode("UTF-8") + secrets[secret] = payload + +os.environ.update(secrets) +# [END secretconfig] + + +SECRET_KEY = env("SECRET_KEY") + +ALLOWED_HOSTS = ["*"] + + +# Application definition + +INSTALLED_APPS = [ + 'polls.apps.PollsConfig', + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'mysite', + 'storages', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'mysite.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'mysite.wsgi.application' + + +# [START dbconfig] +# Use django-environ to define the connection string +DATABASES = {"default": env.db()} +# [END dbconfig] + +# Password validation +# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +# [START staticconfig] +# Define static storage via django-storages[google] +GS_BUCKET_NAME = env("GS_BUCKET_NAME") + +DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" +STATICFILES_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" +GS_DEFAULT_ACL = "publicRead" +# [END staticconfig] diff --git a/run/django/mysite/urls.py b/run/django/mysite/urls.py new file mode 100644 index 000000000000..ca908e72e3c2 --- /dev/null +++ b/run/django/mysite/urls.py @@ -0,0 +1,7 @@ +from django.contrib import admin +from django.urls import include, path + +urlpatterns = [ + path('', include('polls.urls')), + path('admin/', admin.site.urls), +] diff --git a/run/django/mysite/wsgi.py b/run/django/mysite/wsgi.py new file mode 100644 index 000000000000..dbe7bb5fcc88 --- /dev/null +++ b/run/django/mysite/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for mysite project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + +application = get_wsgi_application() diff --git a/run/django/polls/__init__.py b/run/django/polls/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/run/django/polls/admin.py b/run/django/polls/admin.py new file mode 100644 index 000000000000..6af8ff674594 --- /dev/null +++ b/run/django/polls/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin + +from .models import Question + +admin.site.register(Question) diff --git a/run/django/polls/apps.py b/run/django/polls/apps.py new file mode 100644 index 000000000000..d0f109e60e45 --- /dev/null +++ b/run/django/polls/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class PollsConfig(AppConfig): + name = 'polls' diff --git a/run/django/polls/models.py b/run/django/polls/models.py new file mode 100644 index 000000000000..48780e636428 --- /dev/null +++ b/run/django/polls/models.py @@ -0,0 +1,12 @@ +from django.db import models + + +class Question(models.Model): + question_text = models.CharField(max_length=200) + pub_date = models.DateTimeField('date published') + + +class Choice(models.Model): + question = models.ForeignKey(Question, on_delete=models.CASCADE) + choice_text = models.CharField(max_length=200) + votes = models.IntegerField(default=0) diff --git a/run/django/polls/tests.py b/run/django/polls/tests.py new file mode 100644 index 000000000000..7ce503c2dd97 --- /dev/null +++ b/run/django/polls/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/run/django/polls/urls.py b/run/django/polls/urls.py new file mode 100644 index 000000000000..88a9caca8ceb --- /dev/null +++ b/run/django/polls/urls.py @@ -0,0 +1,7 @@ +from django.urls import path + +from . import views + +urlpatterns = [ + path('', views.index, name='index'), +] diff --git a/run/django/polls/views.py b/run/django/polls/views.py new file mode 100644 index 000000000000..963b6f7088b1 --- /dev/null +++ b/run/django/polls/views.py @@ -0,0 +1,5 @@ +from django.http import HttpResponse + + +def index(request): + return HttpResponse("Hello, world. You're at the polls index.") diff --git a/run/django/requirements.txt b/run/django/requirements.txt new file mode 100644 index 000000000000..cee43bab78c5 --- /dev/null +++ b/run/django/requirements.txt @@ -0,0 +1,7 @@ +django==3.0.5 +django-storages[google]==1.9.1 +django-environ==0.4.5 +# mysqlclient==1.4.1 # Uncomment this line if using MySQL +psycopg2-binary==2.8.4 +gunicorn +google-cloud-secret-manager==0.2.0 From b5ccc3cab877b3540dbb6d6cf992321f17feeeb0 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 4 May 2020 13:59:02 +1000 Subject: [PATCH 02/91] Cloud Run with Django tutorial --- run/django/.gcloudignore | 17 +++++ run/django/Dockerfile | 23 ++++++ run/django/README.md | 25 +++++++ run/django/cloudmigrate.yaml | 16 +++++ run/django/manage.py | 21 ++++++ run/django/mysite/__init__.py | 0 run/django/mysite/asgi.py | 16 +++++ run/django/mysite/settings.py | 131 ++++++++++++++++++++++++++++++++++ run/django/mysite/urls.py | 7 ++ run/django/mysite/wsgi.py | 16 +++++ run/django/polls/__init__.py | 0 run/django/polls/admin.py | 5 ++ run/django/polls/apps.py | 5 ++ run/django/polls/models.py | 12 ++++ run/django/polls/tests.py | 3 + run/django/polls/urls.py | 7 ++ run/django/polls/views.py | 5 ++ run/django/requirements.txt | 7 ++ 18 files changed, 316 insertions(+) create mode 100644 run/django/.gcloudignore create mode 100644 run/django/Dockerfile create mode 100644 run/django/README.md create mode 100644 run/django/cloudmigrate.yaml create mode 100755 run/django/manage.py create mode 100644 run/django/mysite/__init__.py create mode 100644 run/django/mysite/asgi.py create mode 100644 run/django/mysite/settings.py create mode 100644 run/django/mysite/urls.py create mode 100644 run/django/mysite/wsgi.py create mode 100644 run/django/polls/__init__.py create mode 100644 run/django/polls/admin.py create mode 100644 run/django/polls/apps.py create mode 100644 run/django/polls/models.py create mode 100644 run/django/polls/tests.py create mode 100644 run/django/polls/urls.py create mode 100644 run/django/polls/views.py create mode 100644 run/django/requirements.txt diff --git a/run/django/.gcloudignore b/run/django/.gcloudignore new file mode 100644 index 000000000000..c6be208a110f --- /dev/null +++ b/run/django/.gcloudignore @@ -0,0 +1,17 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore + +# Python pycache: +__pycache__/ diff --git a/run/django/Dockerfile b/run/django/Dockerfile new file mode 100644 index 000000000000..95e7953f86d9 --- /dev/null +++ b/run/django/Dockerfile @@ -0,0 +1,23 @@ +# Use an official lightweight Python image. +# https://hub.docker.com/_/python +FROM python:3.8-slim + +ENV APP_HOME /app +WORKDIR $APP_HOME + +# Install dependencies. +COPY requirements.txt . +RUN pip install -r requirements.txt + +# Copy local code to the container image. +COPY . . + +# Service must listen to $PORT environment variable. +# This default value facilitates local development. +ENV PORT 8080 + +# Run the web service on container startup. Here we use the gunicorn +# webserver, with one worker process and 8 threads. +# For environments with multiple CPU cores, increase the number of workers +# to be equal to the cores available. +CMD exec gunicorn --bind 0.0.0.0:$PORT --workers 1 --threads 8 --timeout 0 mysite.wsgi:application diff --git a/run/django/README.md b/run/django/README.md new file mode 100644 index 000000000000..c7e277de6676 --- /dev/null +++ b/run/django/README.md @@ -0,0 +1,25 @@ +# Getting started with Django on Google Cloud Platform on Cloud Run (fully managed) + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=run/django/README.md + +This repository is an example of how to run a [Django](https://www.djangoproject.com/) +app on Google Cloud Run (fully managed). It uses the +[Writing your first Django app](https://docs.djangoproject.com/en/3.0/#first-steps) ([Part 1](https://docs.djangoproject.com/en/3.0/intro/tutorial01/) and [Part 2](https://docs.djangoproject.com/en/3.0/intro/tutorial02/)) as the +example app to deploy. + + +# Tutorial +See our [Running Django on Cloud Run (fully managed)](https://cloud.google.com/python/django/run) tutorial for instructions for setting up and deploying this sample application. + + +## Contributing changes + +* See [CONTRIBUTING.md](CONTRIBUTING.md) + + +## Licensing + +* See [LICENSE](LICENSE) diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml new file mode 100644 index 000000000000..01a2173fb4a6 --- /dev/null +++ b/run/django/cloudmigrate.yaml @@ -0,0 +1,16 @@ +steps: +- name: "gcr.io/cloud-builders/docker" + args: ["build", "-t", "gcr.io/${PROJECT_ID}/polls", "."] + +- name: "gcr.io/cloud-builders/docker" + args: ["push", "gcr.io/${PROJECT_ID}/polls"] + +- name: "gcr.io/google-appengine/exec-wrapper" + args: ["-i", "gcr.io/$PROJECT_ID/polls", + "-s", "${PROJECT_ID}:us-central1:django-instance", + "--", "python", "manage.py", "migrate"] + +- name: "gcr.io/google-appengine/exec-wrapper" + args: ["-i", "gcr.io/$PROJECT_ID/polls", + "-s", "${PROJECT_ID}:us-central1:django-instance", + "--", "python", "manage.py", "collectstatic", "--no-input"] diff --git a/run/django/manage.py b/run/django/manage.py new file mode 100755 index 000000000000..341863cf62fa --- /dev/null +++ b/run/django/manage.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/run/django/mysite/__init__.py b/run/django/mysite/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/run/django/mysite/asgi.py b/run/django/mysite/asgi.py new file mode 100644 index 000000000000..35d925e4c5a8 --- /dev/null +++ b/run/django/mysite/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for mysite project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + +application = get_asgi_application() diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py new file mode 100644 index 000000000000..224f1671a2e5 --- /dev/null +++ b/run/django/mysite/settings.py @@ -0,0 +1,131 @@ +""" +Django settings for mysite project. + +Generated by 'django-admin startproject' using Django 3.0.5. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.0/ref/settings/ +""" +import os + +# [START secretconfig] +import environ +import google.auth +from google.cloud import secretmanager_v1beta1 as sm + +# Import settings with django-environ +env = environ.Env() + +# Import settings from Secret Manager +secrets = {} +_, project = google.auth.default() +client = sm.SecretManagerServiceClient() +parent = client.project_path(project) + +for secret in ["DATABASE_URL", "GS_BUCKET_NAME", "SECRET_KEY"]: + path = client.secret_version_path(project, secret, "latest") + payload = client.access_secret_version(path).payload.data.decode("UTF-8") + secrets[secret] = payload + +os.environ.update(secrets) +# [END secretconfig] + + +SECRET_KEY = env("SECRET_KEY") + +ALLOWED_HOSTS = ["*"] + + +# Application definition + +INSTALLED_APPS = [ + 'polls.apps.PollsConfig', + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'mysite', + 'storages', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'mysite.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'mysite.wsgi.application' + + +# [START dbconfig] +# Use django-environ to define the connection string +DATABASES = {"default": env.db()} +# [END dbconfig] + +# Password validation +# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +# [START staticconfig] +# Define static storage via django-storages[google] +GS_BUCKET_NAME = env("GS_BUCKET_NAME") + +DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" +STATICFILES_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" +GS_DEFAULT_ACL = "publicRead" +# [END staticconfig] diff --git a/run/django/mysite/urls.py b/run/django/mysite/urls.py new file mode 100644 index 000000000000..ca908e72e3c2 --- /dev/null +++ b/run/django/mysite/urls.py @@ -0,0 +1,7 @@ +from django.contrib import admin +from django.urls import include, path + +urlpatterns = [ + path('', include('polls.urls')), + path('admin/', admin.site.urls), +] diff --git a/run/django/mysite/wsgi.py b/run/django/mysite/wsgi.py new file mode 100644 index 000000000000..dbe7bb5fcc88 --- /dev/null +++ b/run/django/mysite/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for mysite project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + +application = get_wsgi_application() diff --git a/run/django/polls/__init__.py b/run/django/polls/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/run/django/polls/admin.py b/run/django/polls/admin.py new file mode 100644 index 000000000000..6af8ff674594 --- /dev/null +++ b/run/django/polls/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin + +from .models import Question + +admin.site.register(Question) diff --git a/run/django/polls/apps.py b/run/django/polls/apps.py new file mode 100644 index 000000000000..d0f109e60e45 --- /dev/null +++ b/run/django/polls/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class PollsConfig(AppConfig): + name = 'polls' diff --git a/run/django/polls/models.py b/run/django/polls/models.py new file mode 100644 index 000000000000..48780e636428 --- /dev/null +++ b/run/django/polls/models.py @@ -0,0 +1,12 @@ +from django.db import models + + +class Question(models.Model): + question_text = models.CharField(max_length=200) + pub_date = models.DateTimeField('date published') + + +class Choice(models.Model): + question = models.ForeignKey(Question, on_delete=models.CASCADE) + choice_text = models.CharField(max_length=200) + votes = models.IntegerField(default=0) diff --git a/run/django/polls/tests.py b/run/django/polls/tests.py new file mode 100644 index 000000000000..7ce503c2dd97 --- /dev/null +++ b/run/django/polls/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/run/django/polls/urls.py b/run/django/polls/urls.py new file mode 100644 index 000000000000..88a9caca8ceb --- /dev/null +++ b/run/django/polls/urls.py @@ -0,0 +1,7 @@ +from django.urls import path + +from . import views + +urlpatterns = [ + path('', views.index, name='index'), +] diff --git a/run/django/polls/views.py b/run/django/polls/views.py new file mode 100644 index 000000000000..963b6f7088b1 --- /dev/null +++ b/run/django/polls/views.py @@ -0,0 +1,5 @@ +from django.http import HttpResponse + + +def index(request): + return HttpResponse("Hello, world. You're at the polls index.") diff --git a/run/django/requirements.txt b/run/django/requirements.txt new file mode 100644 index 000000000000..cee43bab78c5 --- /dev/null +++ b/run/django/requirements.txt @@ -0,0 +1,7 @@ +django==3.0.5 +django-storages[google]==1.9.1 +django-environ==0.4.5 +# mysqlclient==1.4.1 # Uncomment this line if using MySQL +psycopg2-binary==2.8.4 +gunicorn +google-cloud-secret-manager==0.2.0 From 1ba9285e502936e506319bc9ea8443fcf1c31fd0 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 11 May 2020 13:53:56 +1000 Subject: [PATCH 03/91] black, .env file update --- run/django/mysite/settings.py | 105 +++++++++++++++++----------------- 1 file changed, 51 insertions(+), 54 deletions(-) diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 224f1671a2e5..df3205e23246 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -13,26 +13,29 @@ # [START secretconfig] import environ -import google.auth -from google.cloud import secretmanager_v1beta1 as sm -# Import settings with django-environ -env = environ.Env() +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +env_file = os.path.join(BASE_DIR, ".env") -# Import settings from Secret Manager -secrets = {} -_, project = google.auth.default() -client = sm.SecretManagerServiceClient() -parent = client.project_path(project) +if not os.path.isfile(".env"): + import google.auth + from google.cloud import secretmanager_v1beta1 as sm -for secret in ["DATABASE_URL", "GS_BUCKET_NAME", "SECRET_KEY"]: - path = client.secret_version_path(project, secret, "latest") - payload = client.access_secret_version(path).payload.data.decode("UTF-8") - secrets[secret] = payload + _, project = google.auth.default() -os.environ.update(secrets) -# [END secretconfig] + if project: + client = sm.SecretManagerServiceClient() + + SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") + path = client.secret_version_path(project, SETTINGS_NAME, "latest") + payload = client.access_secret_version(path).payload.data.decode("UTF-8") + with open(env_file, "w") as f: + f.write(payload) + +env = environ.Env() +env.read_env(env_file) +# [END secretconfig] SECRET_KEY = env("SECRET_KEY") @@ -42,46 +45,46 @@ # Application definition INSTALLED_APPS = [ - 'polls.apps.PollsConfig', - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'mysite', - 'storages', + "polls.apps.PollsConfig", + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "mysite", + "storages", ] MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", ] -ROOT_URLCONF = 'mysite.urls' +ROOT_URLCONF = "mysite.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", ], }, }, ] -WSGI_APPLICATION = 'mysite.wsgi.application' +WSGI_APPLICATION = "mysite.wsgi.application" # [START dbconfig] @@ -94,26 +97,20 @@ AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, + {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",}, + {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",}, + {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",}, ] # Internationalization # https://docs.djangoproject.com/en/3.0/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" USE_I18N = True From e6c93620e216df793d754c3577be8f656c30511f Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 15 Jun 2020 12:21:02 +1000 Subject: [PATCH 04/91] Document Dockerfile, add PYTHONUNBUFFERED --- run/django/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/run/django/Dockerfile b/run/django/Dockerfile index 95e7953f86d9..17f54919cde7 100644 --- a/run/django/Dockerfile +++ b/run/django/Dockerfile @@ -5,7 +5,10 @@ FROM python:3.8-slim ENV APP_HOME /app WORKDIR $APP_HOME -# Install dependencies. +# Removes output stream buffering, allowing for more efficient logging +ENV PYTHONUNBUFFERED 1 + +# Install dependencies COPY requirements.txt . RUN pip install -r requirements.txt From 7cd94bb4469b3b11cd6ca29f58f93e67a71b318e Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 15 Jun 2020 12:21:16 +1000 Subject: [PATCH 05/91] Document cloudmigrate.yaml --- run/django/cloudmigrate.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml index 01a2173fb4a6..d49f42bc7ecf 100644 --- a/run/django/cloudmigrate.yaml +++ b/run/django/cloudmigrate.yaml @@ -1,16 +1,24 @@ steps: + +# Build the image - name: "gcr.io/cloud-builders/docker" args: ["build", "-t", "gcr.io/${PROJECT_ID}/polls", "."] +# Push the image to the Container Registry - name: "gcr.io/cloud-builders/docker" args: ["push", "gcr.io/${PROJECT_ID}/polls"] +# Apply the Django database migration against the Cloud SQL database - name: "gcr.io/google-appengine/exec-wrapper" args: ["-i", "gcr.io/$PROJECT_ID/polls", "-s", "${PROJECT_ID}:us-central1:django-instance", "--", "python", "manage.py", "migrate"] +# Apply the static migrations against the Cloud Storage bucket - name: "gcr.io/google-appengine/exec-wrapper" args: ["-i", "gcr.io/$PROJECT_ID/polls", "-s", "${PROJECT_ID}:us-central1:django-instance", "--", "python", "manage.py", "collectstatic", "--no-input"] + +# This Cloud Build configuration could be extended to include the gcloud run deploy +# step, see https://cloud.google.com/run/docs/continuous-deployment-with-cloud-build From 01fbedf5b714dd6f9503fa85501a08855fa77591 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 15 Jun 2020 12:21:27 +1000 Subject: [PATCH 06/91] Formatiting --- run/django/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run/django/README.md b/run/django/README.md index c7e277de6676..f27567ae40f5 100644 --- a/run/django/README.md +++ b/run/django/README.md @@ -6,9 +6,9 @@ [shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=run/django/README.md This repository is an example of how to run a [Django](https://www.djangoproject.com/) -app on Google Cloud Run (fully managed). It uses the -[Writing your first Django app](https://docs.djangoproject.com/en/3.0/#first-steps) ([Part 1](https://docs.djangoproject.com/en/3.0/intro/tutorial01/) and [Part 2](https://docs.djangoproject.com/en/3.0/intro/tutorial02/)) as the -example app to deploy. +app on Google Cloud Run (fully managed). + +The Django application used in this tutorial is the [Writing your first Django app](https://docs.djangoproject.com/en/3.0/#first-steps), after completing [Part 1](https://docs.djangoproject.com/en/3.0/intro/tutorial01/) and [Part 2](https://docs.djangoproject.com/en/3.0/intro/tutorial02/)). # Tutorial From 711a50ff2ed1868d1101db82d73cfca783efef40 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 15 Jun 2020 12:21:37 +1000 Subject: [PATCH 07/91] Bump django, remove mysql --- run/django/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/run/django/requirements.txt b/run/django/requirements.txt index cee43bab78c5..4758bfb5e0c7 100644 --- a/run/django/requirements.txt +++ b/run/django/requirements.txt @@ -1,7 +1,6 @@ -django==3.0.5 +django==3.0.7 django-storages[google]==1.9.1 django-environ==0.4.5 -# mysqlclient==1.4.1 # Uncomment this line if using MySQL psycopg2-binary==2.8.4 gunicorn google-cloud-secret-manager==0.2.0 From 97088bdb97146916efb77d54dfaaa6cd4ceb008e Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 2 Oct 2020 12:54:01 +1000 Subject: [PATCH 08/91] Update secretmanager --- .../mysite/migrations/0001_createsuperuser.py | 28 +++++++++++++++++++ run/django/mysite/settings.py | 4 +-- 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 run/django/mysite/migrations/0001_createsuperuser.py diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py new file mode 100644 index 000000000000..ceaa6a59bc8a --- /dev/null +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -0,0 +1,28 @@ +from django.db import migrations + +import google.auth +from google.cloud import secretmanager_v1 as sm + + +def createsuperuser(apps, schema_editor): + # Retrieve secret from Secret Manager + _, project = google.auth.default() + client = sm.SecretManagerServiceClient() + path = f"projects/{project}/secrets/superuser_password/versions/latest" + admin_password = client.access_secret_version(path).payload.data.decode("UTF-8") + + # Create a new user using acquired password + from django.contrib.auth.models import User + User.objects.create_superuser("admin", password=admin_password) + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.RunPython(createsuperuser) + ] diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index df3205e23246..4c4a7b497e07 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -19,7 +19,7 @@ if not os.path.isfile(".env"): import google.auth - from google.cloud import secretmanager_v1beta1 as sm + from google.cloud import secretmanager_v1 as sm _, project = google.auth.default() @@ -27,7 +27,7 @@ client = sm.SecretManagerServiceClient() SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") - path = client.secret_version_path(project, SETTINGS_NAME, "latest") + path = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" payload = client.access_secret_version(path).payload.data.decode("UTF-8") with open(env_file, "w") as f: From a3960e8516a6db4a08ee15e994e99d2db1eb305a Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 2 Oct 2020 12:54:14 +1000 Subject: [PATCH 09/91] Update pinned dependencies --- run/django/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/run/django/requirements.txt b/run/django/requirements.txt index 4758bfb5e0c7..b6960e3a5144 100644 --- a/run/django/requirements.txt +++ b/run/django/requirements.txt @@ -1,6 +1,6 @@ -django==3.0.7 -django-storages[google]==1.9.1 +Django==3.1.2 +django-storages[google]==1.10.1 django-environ==0.4.5 psycopg2-binary==2.8.4 -gunicorn -google-cloud-secret-manager==0.2.0 +gunicorn==20.0.4 +google-cloud-secret-manager==2.0.0 From c0a47537616f0c93c237c61f5773c067559dcc57 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 2 Oct 2020 14:00:57 +1000 Subject: [PATCH 10/91] ensure path is set correctly with api update --- run/django/mysite/migrations/0001_createsuperuser.py | 4 ++-- run/django/mysite/settings.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index ceaa6a59bc8a..25233a816e49 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -8,8 +8,8 @@ def createsuperuser(apps, schema_editor): # Retrieve secret from Secret Manager _, project = google.auth.default() client = sm.SecretManagerServiceClient() - path = f"projects/{project}/secrets/superuser_password/versions/latest" - admin_password = client.access_secret_version(path).payload.data.decode("UTF-8") + name = f"projects/{project}/secrets/superuser_password/versions/latest" + admin_password = client.access_secret_version(name=name).payload.data.decode("UTF-8") # Create a new user using acquired password from django.contrib.auth.models import User diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 4c4a7b497e07..a80b50b23fff 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -27,8 +27,8 @@ client = sm.SecretManagerServiceClient() SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") - path = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" - payload = client.access_secret_version(path).payload.data.decode("UTF-8") + name = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" + payload = client.access_secret_version(name=name).payload.data.decode("UTF-8") with open(env_file, "w") as f: f.write(payload) From e75ee592a9f5e557fb7c95cb122910b4b1f51dc5 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 2 Oct 2020 14:01:23 +1000 Subject: [PATCH 11/91] format --- run/django/mysite/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index a80b50b23fff..9498fbfafe3e 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -41,7 +41,6 @@ ALLOWED_HOSTS = ["*"] - # Application definition INSTALLED_APPS = [ From 55db6c4e795e3bd7549dc56aa7895863e5301040 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 12 Oct 2020 14:11:53 +1100 Subject: [PATCH 12/91] Generalise cloudmigrate file --- run/django/cloudmigrate.yaml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml index d49f42bc7ecf..eeff439cbe83 100644 --- a/run/django/cloudmigrate.yaml +++ b/run/django/cloudmigrate.yaml @@ -2,23 +2,28 @@ steps: # Build the image - name: "gcr.io/cloud-builders/docker" - args: ["build", "-t", "gcr.io/${PROJECT_ID}/polls", "."] + args: ["build", "-t", "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}", "."] # Push the image to the Container Registry - name: "gcr.io/cloud-builders/docker" - args: ["push", "gcr.io/${PROJECT_ID}/polls"] + args: ["push", "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}"] # Apply the Django database migration against the Cloud SQL database - name: "gcr.io/google-appengine/exec-wrapper" - args: ["-i", "gcr.io/$PROJECT_ID/polls", - "-s", "${PROJECT_ID}:us-central1:django-instance", + args: ["-i", "gcr.io/$PROJECT_ID/${_SERVICE_NAME}", + "-s", "${PROJECT_ID}:${_REGION}:${_INSTANCE_NAME}", "--", "python", "manage.py", "migrate"] # Apply the static migrations against the Cloud Storage bucket - name: "gcr.io/google-appengine/exec-wrapper" - args: ["-i", "gcr.io/$PROJECT_ID/polls", - "-s", "${PROJECT_ID}:us-central1:django-instance", + args: ["-i", "gcr.io/$PROJECT_ID/${_SERVICE_NAME}", + "-s", "${PROJECT_ID}:${_REGION}:${_INSTANCE_NAME}", "--", "python", "manage.py", "collectstatic", "--no-input"] -# This Cloud Build configuration could be extended to include the gcloud run deploy -# step, see https://cloud.google.com/run/docs/continuous-deployment-with-cloud-build +substitutions: + _INSTANCE_NAME: django-instance + _REGION: us-central1 + _SERVICE_NAME: polls + +images: + - "gcr.io/${PROJECT_ID}/${_SERVICE_NAME} From b0d3e6174d3e7b10c03bd5a8614b1d72130950a0 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 12 Oct 2020 14:13:49 +1100 Subject: [PATCH 13/91] update service default --- run/django/cloudmigrate.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml index eeff439cbe83..93d3e7d34b60 100644 --- a/run/django/cloudmigrate.yaml +++ b/run/django/cloudmigrate.yaml @@ -23,7 +23,7 @@ steps: substitutions: _INSTANCE_NAME: django-instance _REGION: us-central1 - _SERVICE_NAME: polls + _SERVICE_NAME: polls-service images: - "gcr.io/${PROJECT_ID}/${_SERVICE_NAME} From b39521bbe4a91a97013841d87b29ab64bff64b9e Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 19 Oct 2020 16:47:15 +1100 Subject: [PATCH 14/91] add tests --- run/django/.gcloudignore | 3 + run/django/cloudmigrate.yaml | 2 +- run/django/e2e_test.py | 409 +++++++++++++++++++++++++++++++ run/django/noxfile_config.py | 39 +++ run/django/polls/test_polls.py | 22 ++ run/django/polls/tests.py | 3 - run/django/requirements-test.txt | 3 + 7 files changed, 477 insertions(+), 4 deletions(-) create mode 100644 run/django/e2e_test.py create mode 100644 run/django/noxfile_config.py create mode 100644 run/django/polls/test_polls.py delete mode 100644 run/django/polls/tests.py create mode 100644 run/django/requirements-test.txt diff --git a/run/django/.gcloudignore b/run/django/.gcloudignore index c6be208a110f..c0abe06014ba 100644 --- a/run/django/.gcloudignore +++ b/run/django/.gcloudignore @@ -15,3 +15,6 @@ # Python pycache: __pycache__/ + +venv/ +.env diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml index 93d3e7d34b60..cad8eae78491 100644 --- a/run/django/cloudmigrate.yaml +++ b/run/django/cloudmigrate.yaml @@ -26,4 +26,4 @@ substitutions: _SERVICE_NAME: polls-service images: - - "gcr.io/${PROJECT_ID}/${_SERVICE_NAME} + - "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}" \ No newline at end of file diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py new file mode 100644 index 000000000000..896381605f49 --- /dev/null +++ b/run/django/e2e_test.py @@ -0,0 +1,409 @@ +# Copyright 2020 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This test creates a Cloud SQL instance, a Cloud Storage bucket, associated +# secrets, and deploys a Django service + +import os +import subprocess +import uuid + +import pytest +import requests +from google.cloud import secretmanager_v1 as sm + +# Unique suffix to create distinct service names +SUFFIX = uuid.uuid4().hex[:10] + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +DATABASE_VERISON = "POSTGRES_12" +REGION = "us-central1" +DATABASE_TIER = "db-f1-micro" + +CLOUDSQL_INSTANCE = f"instance-{SUFFIX}" +STORAGE_BUCKET = f"{PROJECT}-media-{SUFFIX}" +DATABASE_NAME = f"polls-{SUFFIX}" +DATABASE_USERNAME = f"django-{SUFFIX}" +DATABASE_PASSWORD = uuid.uuid4().hex[:26] + +ADMIN_NAME = "admin" +ADMIN_PASSWORD = uuid.uuid4().hex[:26] + +# These are hardcoded elsewhere in the code +SECRET_SETTINGS_NAME = "django_settings" +SECRET_ADMINPASS_NAME = "superuser_password" + + +@pytest.fixture +def project_number(): + projectnum = ( + subprocess.run( + [ + "gcloud", + "projects", + "list", + "--filter", + PROJECT, + "--format", + "value(projectNumber)", + ], + stdout=subprocess.PIPE, + check=True, + ) + .stdout.strip() + .decode() + ) + + yield projectnum + + +@pytest.fixture +def postgres_host(project_number): + # Create database instance + subprocess.run( + [ + "gcloud", + "sql", + "instances", + "create", + CLOUDSQL_INSTANCE, + "--database-version", + DATABASE_VERISON, + "--tier", + DATABASE_TIER, + "--region", + REGION, + "--project", + PROJECT, + ], + check=True, + ) + + # Create database + subprocess.run( + [ + "gcloud", + "sql", + "databases", + "create", + DATABASE_NAME, + "--instance", + CLOUDSQL_INSTANCE, + "--project", + PROJECT, + ], + check=True, + ) + # Create User + # NOTE Creating a user via the tutorial method is not automatable. + subprocess.run( + [ + "gcloud", + "sql", + "users", + "create", + DATABASE_USERNAME, + "--password", + DATABASE_PASSWORD, + "--instance", + CLOUDSQL_INSTANCE, + "--project", + PROJECT, + ], + check=True, + ) + + # Allow access to database from Cloud Build + subprocess.run( + [ + "gcloud", + "projects", + "add-iam-policy-binding", + PROJECT, + "--member", + f"serviceAccount:{project_number}@cloudbuild.gserviceaccount.com", + "--role", + "roles/cloudsql.client", + ], + check=True, + ) + + yield CLOUDSQL_INSTANCE + + subprocess.run( + [ + "gcloud", + "sql", + "databases", + "delete", + DATABASE_NAME, + "--instance", + CLOUDSQL_INSTANCE, + "--quiet", + ], + check=True, + ) + + subprocess.run( + [ + "gcloud", + "sql", + "users", + "delete", + DATABASE_USERNAME, + "--instance", + CLOUDSQL_INSTANCE, + "--quiet", + ], + check=True, + ) + + subprocess.run( + ["gcloud", "sql", "instances" "delete", CLOUDSQL_INSTANCE, "--quiet"], + check=True, + ) + + +@pytest.fixture +def media_bucket(): + # Create storage bucket + subprocess.run(["gsutil", "mb", "-l", REGION, f"gs://{STORAGE_BUCKET}"], check=True) + + yield STORAGE_BUCKET + + # Delete storage bucket contents, delete bucket + subprocess.run(["gsutil", "-m", "rm", "-r", f"gs://{STORAGE_BUCKET}"], check=True) + subprocess.run(["gsutil", "rb", f"gs://{STORAGE_BUCKET}"], check=True) + + +@pytest.fixture +def secrets(project_number): + # Create a number of secrets and allow Google Cloud services access to them + + def create_secret(name, value): + secret = client.create_secret( + request={ + "parent": f"projects/{PROJECT}", + "secret": {"replication": {"automatic": {}}}, + "secret_id": name, + } + ) + + client.add_secret_version( + request={"parent": secret.name, "payload": {"data": value.encode("UTF-8")}} + ) + print(f"DEBUG: {name}\n{value}") + + def allow_access(name, member): + subprocess.run( + [ + "gcloud", + "secrets", + "add-iam-policy-binding", + name, + "--member", + member, + "--role", + "roles/secretmanager.secretAccessor", + ], + check=True, + ) + + client = sm.SecretManagerServiceClient() + secret_key = uuid.uuid4().hex[:56] + settings = f""" +DATABASE_URL=postgres://{DATABASE_USERNAME}:{DATABASE_PASSWORD}@//cloudsql/{PROJECT}:{REGION}:{CLOUDSQL_INSTANCE}/{DATABASE_NAME} +GS_BUCKET_NAME={STORAGE_BUCKET} +SECRET_KEY={secret_key} + """ + + create_secret(SECRET_SETTINGS_NAME, settings) + allow_access( + SECRET_SETTINGS_NAME, + f"serviceAccount:{project_number}-compute@developer.gserviceaccount.com", + ) + allow_access( + SECRET_SETTINGS_NAME, + f"serviceAccount:{project_number}@cloudbuild.gserviceaccount.com", + ) + + create_secret(SECRET_ADMINPASS_NAME, ADMIN_PASSWORD) + allow_access( + SECRET_ADMINPASS_NAME, + f"serviceAccount:{project_number}@cloudbuild.gserviceaccount.com", + ) + + yield SECRET_SETTINGS_NAME + + # delete secrets + subprocess.run( + ["gcloud", "secrets", "delete", SECRET_ADMINPASS_NAME, "--quiet"], check=True + ) + subprocess.run( + ["gcloud", "secrets", "delete", SECRET_SETTINGS_NAME, "--quiet"], check=True + ) + + +@pytest.fixture +def container_image(postgres_host, media_bucket, secrets): + # Build container image for Cloud Run deployment + image_name = f"gcr.io/{PROJECT}/polls-{SUFFIX}" + service_name = f"polls-{SUFFIX}" + cloudbuild_config = "cloudmigrate.yaml" + subprocess.run( + [ + "gcloud", + "builds", + "submit", + "--config", + cloudbuild_config, + "--substitutions", + f"_INSTANCE_NAME={postgres_host},_REGION={REGION},_SERVICE_NAME={service_name}", + "--project", + PROJECT, + ], + check=True, + ) + yield image_name + + # Delete container image + subprocess.run( + [ + "gcloud", + "container", + "images", + "delete", + image_name, + "--quiet", + "--project", + PROJECT, + ], + check=True, + ) + + +@pytest.fixture +def deployed_service(container_image): + # Deploy image to Cloud Run + service_name = f"polls-{SUFFIX}" + subprocess.run( + [ + "gcloud", + "run", + "deploy", + service_name, + "--image", + container_image, + "--platform=managed", + "--no-allow-unauthenticated", + "--region", + REGION, + "--add-cloudsql-instances", + f"{PROJECT}:{REGION}:{CLOUDSQL_INSTANCE}", + "--project", + PROJECT, + ], + check=True, + ) + yield service_name + + # Delete Cloud Run service + subprocess.run( + [ + "gcloud", + "run", + "services", + "delete", + service_name, + "--platform=managed", + "--region=us-central1", + "--quiet", + "--project", + PROJECT, + ], + check=True, + ) + + +@pytest.fixture +def service_url_auth_token(deployed_service): + # Get Cloud Run service URL and auth token + service_url = ( + subprocess.run( + [ + "gcloud", + "run", + "services", + "describe", + deployed_service, + "--platform", + "managed", + "--region", + REGION, + "--format", + "value(status.url)", + "--project", + PROJECT, + ], + stdout=subprocess.PIPE, + check=True, + ) + .stdout.strip() + .decode() + ) + auth_token = ( + subprocess.run( + ["gcloud", "auth", "print-identity-token"], + stdout=subprocess.PIPE, + check=True, + ) + .stdout.strip() + .decode() + ) + + yield service_url, auth_token + + # no deletion needed + + +def test_end_to_end(service_url_auth_token): + service_url, auth_token = service_url_auth_token + headers = {"Authorization": f"Bearer {auth_token}"} + login_slug = "/admin/login/?next=/admin/" + client = requests.session() + + # Check homepage + response = client.get(service_url, headers=headers) + body = response.text + + assert response.status_code == 200 + assert "Hello, world" in body + + # Load login page, collecting csrf token + client.get(service_url + login_slug, headers=headers) + csrftoken = client.cookies["csrftoken"] + + # Log into Django admin + payload = { + "username": ADMIN_NAME, + "password": ADMIN_PASSWORD, + "csrfmiddlewaretoken": csrftoken, + } + response = client.post(service_url + login_slug, data=payload, headers=headers) + body = response.text + + # Check Django admin landing page + assert response.status_code == 200 + assert "Site administration" in body + assert "Polls" in body diff --git a/run/django/noxfile_config.py b/run/django/noxfile_config.py new file mode 100644 index 000000000000..af47c4048cd6 --- /dev/null +++ b/run/django/noxfile_config.py @@ -0,0 +1,39 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# You can copy this file into your directory, then it will be imported from +# the noxfile.py. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + + # We only run the cloud run tests in py38 session. + 'ignored_versions': ["2.7", "3.6", "3.7"], + + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + 'envs': {}, +} diff --git a/run/django/polls/test_polls.py b/run/django/polls/test_polls.py new file mode 100644 index 000000000000..4588d730242c --- /dev/null +++ b/run/django/polls/test_polls.py @@ -0,0 +1,22 @@ +# Copyright 2020 Google LLC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from django.test import Client, TestCase # noqa: 401 + + +class PollViewTests(TestCase): + def test_index_view(self): + response = self.client.get('/') + assert response.status_code == 200 + assert 'Hello, world' in str(response.content) diff --git a/run/django/polls/tests.py b/run/django/polls/tests.py deleted file mode 100644 index 7ce503c2dd97..000000000000 --- a/run/django/polls/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/run/django/requirements-test.txt b/run/django/requirements-test.txt new file mode 100644 index 000000000000..eab71ebbbc62 --- /dev/null +++ b/run/django/requirements-test.txt @@ -0,0 +1,3 @@ +pytest==6.0.1 +pytest-django==3.10.0 +requests==2.24.0 \ No newline at end of file From a0c90a459df3139aae195f8d75fbe9e2bf0e82ab Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 19 Oct 2020 16:50:01 +1100 Subject: [PATCH 15/91] Update CODEOWNERS --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2ba586d80ca0..8be73c82814b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -54,6 +54,7 @@ /profiler/**/*.py @kalyanac @GoogleCloudPlatform/python-samples-owners /pubsub/**/*.py @anguillanneuf @hongalex @GoogleCloudPlatform/python-samples-owners /run/**/*.py @averikitsch @grant @GoogleCloudPlatform/python-samples-owners +/run/django/**/*.py @glasnt @GoogleCloudPlatform/python-samples-owners /scheduler/**/*.py @averikitsch @GoogleCloudPlatform/python-samples-owners /spanner/**/*.py @larkee @GoogleCloudPlatform/python-samples-owners /speech/**/*.py @telpirion @sirtorry @GoogleCloudPlatform/python-samples-owners From 083145ae1d1a90b70919a1b34475d16c0edc70ee Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 20 Oct 2020 08:52:07 +1100 Subject: [PATCH 16/91] Licence headers --- run/django/Dockerfile | 14 ++++++++++++++ run/django/cloudmigrate.yaml | 14 ++++++++++++++ run/django/manage.py | 15 +++++++++++++++ run/django/mysite/settings.py | 14 ++++++++++++++ run/django/mysite/urls.py | 14 ++++++++++++++ run/django/mysite/wsgi.py | 14 ++++++++++++++ run/django/polls/admin.py | 14 ++++++++++++++ run/django/polls/apps.py | 14 ++++++++++++++ run/django/polls/models.py | 14 ++++++++++++++ run/django/polls/urls.py | 14 ++++++++++++++ run/django/polls/views.py | 14 ++++++++++++++ 11 files changed, 155 insertions(+) diff --git a/run/django/Dockerfile b/run/django/Dockerfile index 17f54919cde7..7e435bca3772 100644 --- a/run/django/Dockerfile +++ b/run/django/Dockerfile @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Use an official lightweight Python image. # https://hub.docker.com/_/python FROM python:3.8-slim diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml index cad8eae78491..f24cd1175a81 100644 --- a/run/django/cloudmigrate.yaml +++ b/run/django/cloudmigrate.yaml @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + steps: # Build the image diff --git a/run/django/manage.py b/run/django/manage.py index 341863cf62fa..934371b8740a 100755 --- a/run/django/manage.py +++ b/run/django/manage.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Django's command-line utility for administrative tasks.""" import os import sys diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 9498fbfafe3e..64baafb8dd33 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ Django settings for mysite project. diff --git a/run/django/mysite/urls.py b/run/django/mysite/urls.py index ca908e72e3c2..ffef6e232864 100644 --- a/run/django/mysite/urls.py +++ b/run/django/mysite/urls.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from django.contrib import admin from django.urls import include, path diff --git a/run/django/mysite/wsgi.py b/run/django/mysite/wsgi.py index dbe7bb5fcc88..78282856146e 100644 --- a/run/django/mysite/wsgi.py +++ b/run/django/mysite/wsgi.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ WSGI config for mysite project. diff --git a/run/django/polls/admin.py b/run/django/polls/admin.py index 6af8ff674594..e90a167fb2a3 100644 --- a/run/django/polls/admin.py +++ b/run/django/polls/admin.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from django.contrib import admin from .models import Question diff --git a/run/django/polls/apps.py b/run/django/polls/apps.py index d0f109e60e45..f692aa20d159 100644 --- a/run/django/polls/apps.py +++ b/run/django/polls/apps.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from django.apps import AppConfig diff --git a/run/django/polls/models.py b/run/django/polls/models.py index 48780e636428..c1e2e9f7a913 100644 --- a/run/django/polls/models.py +++ b/run/django/polls/models.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from django.db import models diff --git a/run/django/polls/urls.py b/run/django/polls/urls.py index 88a9caca8ceb..1a67f0009d00 100644 --- a/run/django/polls/urls.py +++ b/run/django/polls/urls.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from django.urls import path from . import views diff --git a/run/django/polls/views.py b/run/django/polls/views.py index 963b6f7088b1..bda93457d1e5 100644 --- a/run/django/polls/views.py +++ b/run/django/polls/views.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from django.http import HttpResponse From 0508de36b8cb492cb0af4361c773ad2f5ad13734 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 20 Oct 2020 08:52:14 +1100 Subject: [PATCH 17/91] Remove envvar declaration --- run/django/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/run/django/Dockerfile b/run/django/Dockerfile index 7e435bca3772..986f2e3a34c1 100644 --- a/run/django/Dockerfile +++ b/run/django/Dockerfile @@ -29,10 +29,6 @@ RUN pip install -r requirements.txt # Copy local code to the container image. COPY . . -# Service must listen to $PORT environment variable. -# This default value facilitates local development. -ENV PORT 8080 - # Run the web service on container startup. Here we use the gunicorn # webserver, with one worker process and 8 threads. # For environments with multiple CPU cores, increase the number of workers From 4ae043ec9d5f21782c6c1fd10dbead3a68ced460 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 20 Oct 2020 08:52:30 +1100 Subject: [PATCH 18/91] cleanup readme --- run/django/README.md | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/run/django/README.md b/run/django/README.md index f27567ae40f5..564991ee8a4b 100644 --- a/run/django/README.md +++ b/run/django/README.md @@ -1,4 +1,4 @@ -# Getting started with Django on Google Cloud Platform on Cloud Run (fully managed) +# Getting started with Django on Cloud Run [![Open in Cloud Shell][shell_img]][shell_link] @@ -8,18 +8,8 @@ This repository is an example of how to run a [Django](https://www.djangoproject.com/) app on Google Cloud Run (fully managed). -The Django application used in this tutorial is the [Writing your first Django app](https://docs.djangoproject.com/en/3.0/#first-steps), after completing [Part 1](https://docs.djangoproject.com/en/3.0/intro/tutorial01/) and [Part 2](https://docs.djangoproject.com/en/3.0/intro/tutorial02/)). +The Django application used in this tutorial is the [Writing your first Django app](https://docs.djangoproject.com/en/3.0/#first-steps), after completing [Part 1](https://docs.djangoproject.com/en/3.0/intro/tutorial01/) and [Part 2](https://docs.djangoproject.com/en/3.0/intro/tutorial02/). # Tutorial -See our [Running Django on Cloud Run (fully managed)](https://cloud.google.com/python/django/run) tutorial for instructions for setting up and deploying this sample application. - - -## Contributing changes - -* See [CONTRIBUTING.md](CONTRIBUTING.md) - - -## Licensing - -* See [LICENSE](LICENSE) +See our [Running Django on Cloud Run (fully managed)](https://cloud.google.com/python/django/run) tutorial for instructions for setting up and deploying this sample application. \ No newline at end of file From 0fea1a44827939ff09811cb2016aacc1a34b6509 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 20 Oct 2020 08:52:43 +1100 Subject: [PATCH 19/91] Fix import names, region tags --- .../mysite/migrations/0001_createsuperuser.py | 27 ++++++++++++++++--- run/django/mysite/settings.py | 17 ++++++------ run/django/polls/test_polls.py | 4 +-- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index 25233a816e49..af4e51e5c830 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -1,13 +1,34 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from django.db import migrations import google.auth -from google.cloud import secretmanager_v1 as sm +from google.cloud import secretmanager_v1 def createsuperuser(apps, schema_editor): - # Retrieve secret from Secret Manager + """ + Dynamically create an admin user as part of a migration + Password is pulled from Secret Manger (previously created as part of tutorial) + """ + client = secretmanager_v1.SecretManagerServiceClient() + + # Get project value for identifying current context _, project = google.auth.default() - client = sm.SecretManagerServiceClient() + + # Retrieve the previously stored admin passowrd name = f"projects/{project}/secrets/superuser_password/versions/latest" admin_password = client.access_secret_version(name=name).payload.data.decode("UTF-8") diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 64baafb8dd33..03cc58be8bf6 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -25,20 +25,21 @@ """ import os -# [START secretconfig] +# [START run_secretconfig] import environ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) env_file = os.path.join(BASE_DIR, ".env") +# If no .env has been provided, pull it from Secret Manager, storing it locally if not os.path.isfile(".env"): import google.auth - from google.cloud import secretmanager_v1 as sm + from google.cloud import secretmanager_v1 _, project = google.auth.default() if project: - client = sm.SecretManagerServiceClient() + client = secretmanager_v1.SecretManagerServiceClient() SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") name = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" @@ -49,7 +50,7 @@ env = environ.Env() env.read_env(env_file) -# [END secretconfig] +# [END run_secretconfig] SECRET_KEY = env("SECRET_KEY") @@ -100,10 +101,10 @@ WSGI_APPLICATION = "mysite.wsgi.application" -# [START dbconfig] +# [START run_dbconfig] # Use django-environ to define the connection string DATABASES = {"default": env.db()} -# [END dbconfig] +# [END run_dbconfig] # Password validation # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators @@ -131,11 +132,11 @@ USE_TZ = True -# [START staticconfig] +# [START run_staticconfig] # Define static storage via django-storages[google] GS_BUCKET_NAME = env("GS_BUCKET_NAME") DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" STATICFILES_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" GS_DEFAULT_ACL = "publicRead" -# [END staticconfig] +# [END run_staticconfig] diff --git a/run/django/polls/test_polls.py b/run/django/polls/test_polls.py index 4588d730242c..7024a5001c23 100644 --- a/run/django/polls/test_polls.py +++ b/run/django/polls/test_polls.py @@ -17,6 +17,6 @@ class PollViewTests(TestCase): def test_index_view(self): - response = self.client.get('/') + response = self.client.get("/") assert response.status_code == 200 - assert 'Hello, world' in str(response.content) + assert "Hello, world" in str(response.content) From 59a690aec8136f244828cc52906b1de092b998d9 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 20 Oct 2020 08:53:50 +1100 Subject: [PATCH 20/91] black --- run/django/manage.py | 4 ++-- run/django/mysite/asgi.py | 2 +- run/django/mysite/settings.py | 12 +++++++++--- run/django/mysite/urls.py | 4 ++-- run/django/mysite/wsgi.py | 2 +- run/django/noxfile_config.py | 9 +++------ run/django/polls/apps.py | 2 +- run/django/polls/models.py | 2 +- run/django/polls/urls.py | 2 +- 9 files changed, 21 insertions(+), 18 deletions(-) diff --git a/run/django/manage.py b/run/django/manage.py index 934371b8740a..868c06851602 100755 --- a/run/django/manage.py +++ b/run/django/manage.py @@ -20,7 +20,7 @@ def main(): - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: @@ -32,5 +32,5 @@ def main(): execute_from_command_line(sys.argv) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/run/django/mysite/asgi.py b/run/django/mysite/asgi.py index 35d925e4c5a8..5b7d1a17eec1 100644 --- a/run/django/mysite/asgi.py +++ b/run/django/mysite/asgi.py @@ -11,6 +11,6 @@ from django.core.asgi import get_asgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") application = get_asgi_application() diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 03cc58be8bf6..431d7430b722 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -113,9 +113,15 @@ { "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, - {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",}, - {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",}, - {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",}, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, ] diff --git a/run/django/mysite/urls.py b/run/django/mysite/urls.py index ffef6e232864..7e7f336ce614 100644 --- a/run/django/mysite/urls.py +++ b/run/django/mysite/urls.py @@ -16,6 +16,6 @@ from django.urls import include, path urlpatterns = [ - path('', include('polls.urls')), - path('admin/', admin.site.urls), + path("", include("polls.urls")), + path("admin/", admin.site.urls), ] diff --git a/run/django/mysite/wsgi.py b/run/django/mysite/wsgi.py index 78282856146e..444d68f6e5df 100644 --- a/run/django/mysite/wsgi.py +++ b/run/django/mysite/wsgi.py @@ -25,6 +25,6 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") application = get_wsgi_application() diff --git a/run/django/noxfile_config.py b/run/django/noxfile_config.py index af47c4048cd6..696c09e6ce14 100644 --- a/run/django/noxfile_config.py +++ b/run/django/noxfile_config.py @@ -22,18 +22,15 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # We only run the cloud run tests in py38 session. - 'ignored_versions': ["2.7", "3.6", "3.7"], - + "ignored_versions": ["2.7", "3.6", "3.7"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - 'envs': {}, + "envs": {}, } diff --git a/run/django/polls/apps.py b/run/django/polls/apps.py index f692aa20d159..156a0cf65582 100644 --- a/run/django/polls/apps.py +++ b/run/django/polls/apps.py @@ -16,4 +16,4 @@ class PollsConfig(AppConfig): - name = 'polls' + name = "polls" diff --git a/run/django/polls/models.py b/run/django/polls/models.py index c1e2e9f7a913..9234289b05e4 100644 --- a/run/django/polls/models.py +++ b/run/django/polls/models.py @@ -17,7 +17,7 @@ class Question(models.Model): question_text = models.CharField(max_length=200) - pub_date = models.DateTimeField('date published') + pub_date = models.DateTimeField("date published") class Choice(models.Model): diff --git a/run/django/polls/urls.py b/run/django/polls/urls.py index 1a67f0009d00..daaf7a501182 100644 --- a/run/django/polls/urls.py +++ b/run/django/polls/urls.py @@ -17,5 +17,5 @@ from . import views urlpatterns = [ - path('', views.index, name='index'), + path("", views.index, name="index"), ] From 43311df1cab6335ffde89e21c17db57f2c69f4d9 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 4 May 2020 13:59:02 +1000 Subject: [PATCH 21/91] Cloud Run with Django tutorial --- run/django/.gcloudignore | 17 +++++ run/django/Dockerfile | 23 ++++++ run/django/README.md | 25 +++++++ run/django/cloudmigrate.yaml | 16 +++++ run/django/manage.py | 21 ++++++ run/django/mysite/__init__.py | 0 run/django/mysite/asgi.py | 16 +++++ run/django/mysite/settings.py | 131 ++++++++++++++++++++++++++++++++++ run/django/mysite/urls.py | 7 ++ run/django/mysite/wsgi.py | 16 +++++ run/django/polls/__init__.py | 0 run/django/polls/admin.py | 5 ++ run/django/polls/apps.py | 5 ++ run/django/polls/models.py | 12 ++++ run/django/polls/tests.py | 3 + run/django/polls/urls.py | 7 ++ run/django/polls/views.py | 5 ++ run/django/requirements.txt | 7 ++ 18 files changed, 316 insertions(+) create mode 100644 run/django/.gcloudignore create mode 100644 run/django/Dockerfile create mode 100644 run/django/README.md create mode 100644 run/django/cloudmigrate.yaml create mode 100755 run/django/manage.py create mode 100644 run/django/mysite/__init__.py create mode 100644 run/django/mysite/asgi.py create mode 100644 run/django/mysite/settings.py create mode 100644 run/django/mysite/urls.py create mode 100644 run/django/mysite/wsgi.py create mode 100644 run/django/polls/__init__.py create mode 100644 run/django/polls/admin.py create mode 100644 run/django/polls/apps.py create mode 100644 run/django/polls/models.py create mode 100644 run/django/polls/tests.py create mode 100644 run/django/polls/urls.py create mode 100644 run/django/polls/views.py create mode 100644 run/django/requirements.txt diff --git a/run/django/.gcloudignore b/run/django/.gcloudignore new file mode 100644 index 000000000000..c6be208a110f --- /dev/null +++ b/run/django/.gcloudignore @@ -0,0 +1,17 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore + +# Python pycache: +__pycache__/ diff --git a/run/django/Dockerfile b/run/django/Dockerfile new file mode 100644 index 000000000000..95e7953f86d9 --- /dev/null +++ b/run/django/Dockerfile @@ -0,0 +1,23 @@ +# Use an official lightweight Python image. +# https://hub.docker.com/_/python +FROM python:3.8-slim + +ENV APP_HOME /app +WORKDIR $APP_HOME + +# Install dependencies. +COPY requirements.txt . +RUN pip install -r requirements.txt + +# Copy local code to the container image. +COPY . . + +# Service must listen to $PORT environment variable. +# This default value facilitates local development. +ENV PORT 8080 + +# Run the web service on container startup. Here we use the gunicorn +# webserver, with one worker process and 8 threads. +# For environments with multiple CPU cores, increase the number of workers +# to be equal to the cores available. +CMD exec gunicorn --bind 0.0.0.0:$PORT --workers 1 --threads 8 --timeout 0 mysite.wsgi:application diff --git a/run/django/README.md b/run/django/README.md new file mode 100644 index 000000000000..c7e277de6676 --- /dev/null +++ b/run/django/README.md @@ -0,0 +1,25 @@ +# Getting started with Django on Google Cloud Platform on Cloud Run (fully managed) + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=run/django/README.md + +This repository is an example of how to run a [Django](https://www.djangoproject.com/) +app on Google Cloud Run (fully managed). It uses the +[Writing your first Django app](https://docs.djangoproject.com/en/3.0/#first-steps) ([Part 1](https://docs.djangoproject.com/en/3.0/intro/tutorial01/) and [Part 2](https://docs.djangoproject.com/en/3.0/intro/tutorial02/)) as the +example app to deploy. + + +# Tutorial +See our [Running Django on Cloud Run (fully managed)](https://cloud.google.com/python/django/run) tutorial for instructions for setting up and deploying this sample application. + + +## Contributing changes + +* See [CONTRIBUTING.md](CONTRIBUTING.md) + + +## Licensing + +* See [LICENSE](LICENSE) diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml new file mode 100644 index 000000000000..01a2173fb4a6 --- /dev/null +++ b/run/django/cloudmigrate.yaml @@ -0,0 +1,16 @@ +steps: +- name: "gcr.io/cloud-builders/docker" + args: ["build", "-t", "gcr.io/${PROJECT_ID}/polls", "."] + +- name: "gcr.io/cloud-builders/docker" + args: ["push", "gcr.io/${PROJECT_ID}/polls"] + +- name: "gcr.io/google-appengine/exec-wrapper" + args: ["-i", "gcr.io/$PROJECT_ID/polls", + "-s", "${PROJECT_ID}:us-central1:django-instance", + "--", "python", "manage.py", "migrate"] + +- name: "gcr.io/google-appengine/exec-wrapper" + args: ["-i", "gcr.io/$PROJECT_ID/polls", + "-s", "${PROJECT_ID}:us-central1:django-instance", + "--", "python", "manage.py", "collectstatic", "--no-input"] diff --git a/run/django/manage.py b/run/django/manage.py new file mode 100755 index 000000000000..341863cf62fa --- /dev/null +++ b/run/django/manage.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/run/django/mysite/__init__.py b/run/django/mysite/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/run/django/mysite/asgi.py b/run/django/mysite/asgi.py new file mode 100644 index 000000000000..35d925e4c5a8 --- /dev/null +++ b/run/django/mysite/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for mysite project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + +application = get_asgi_application() diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py new file mode 100644 index 000000000000..224f1671a2e5 --- /dev/null +++ b/run/django/mysite/settings.py @@ -0,0 +1,131 @@ +""" +Django settings for mysite project. + +Generated by 'django-admin startproject' using Django 3.0.5. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.0/ref/settings/ +""" +import os + +# [START secretconfig] +import environ +import google.auth +from google.cloud import secretmanager_v1beta1 as sm + +# Import settings with django-environ +env = environ.Env() + +# Import settings from Secret Manager +secrets = {} +_, project = google.auth.default() +client = sm.SecretManagerServiceClient() +parent = client.project_path(project) + +for secret in ["DATABASE_URL", "GS_BUCKET_NAME", "SECRET_KEY"]: + path = client.secret_version_path(project, secret, "latest") + payload = client.access_secret_version(path).payload.data.decode("UTF-8") + secrets[secret] = payload + +os.environ.update(secrets) +# [END secretconfig] + + +SECRET_KEY = env("SECRET_KEY") + +ALLOWED_HOSTS = ["*"] + + +# Application definition + +INSTALLED_APPS = [ + 'polls.apps.PollsConfig', + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'mysite', + 'storages', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'mysite.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'mysite.wsgi.application' + + +# [START dbconfig] +# Use django-environ to define the connection string +DATABASES = {"default": env.db()} +# [END dbconfig] + +# Password validation +# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +# [START staticconfig] +# Define static storage via django-storages[google] +GS_BUCKET_NAME = env("GS_BUCKET_NAME") + +DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" +STATICFILES_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" +GS_DEFAULT_ACL = "publicRead" +# [END staticconfig] diff --git a/run/django/mysite/urls.py b/run/django/mysite/urls.py new file mode 100644 index 000000000000..ca908e72e3c2 --- /dev/null +++ b/run/django/mysite/urls.py @@ -0,0 +1,7 @@ +from django.contrib import admin +from django.urls import include, path + +urlpatterns = [ + path('', include('polls.urls')), + path('admin/', admin.site.urls), +] diff --git a/run/django/mysite/wsgi.py b/run/django/mysite/wsgi.py new file mode 100644 index 000000000000..dbe7bb5fcc88 --- /dev/null +++ b/run/django/mysite/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for mysite project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + +application = get_wsgi_application() diff --git a/run/django/polls/__init__.py b/run/django/polls/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/run/django/polls/admin.py b/run/django/polls/admin.py new file mode 100644 index 000000000000..6af8ff674594 --- /dev/null +++ b/run/django/polls/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin + +from .models import Question + +admin.site.register(Question) diff --git a/run/django/polls/apps.py b/run/django/polls/apps.py new file mode 100644 index 000000000000..d0f109e60e45 --- /dev/null +++ b/run/django/polls/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class PollsConfig(AppConfig): + name = 'polls' diff --git a/run/django/polls/models.py b/run/django/polls/models.py new file mode 100644 index 000000000000..48780e636428 --- /dev/null +++ b/run/django/polls/models.py @@ -0,0 +1,12 @@ +from django.db import models + + +class Question(models.Model): + question_text = models.CharField(max_length=200) + pub_date = models.DateTimeField('date published') + + +class Choice(models.Model): + question = models.ForeignKey(Question, on_delete=models.CASCADE) + choice_text = models.CharField(max_length=200) + votes = models.IntegerField(default=0) diff --git a/run/django/polls/tests.py b/run/django/polls/tests.py new file mode 100644 index 000000000000..7ce503c2dd97 --- /dev/null +++ b/run/django/polls/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/run/django/polls/urls.py b/run/django/polls/urls.py new file mode 100644 index 000000000000..88a9caca8ceb --- /dev/null +++ b/run/django/polls/urls.py @@ -0,0 +1,7 @@ +from django.urls import path + +from . import views + +urlpatterns = [ + path('', views.index, name='index'), +] diff --git a/run/django/polls/views.py b/run/django/polls/views.py new file mode 100644 index 000000000000..963b6f7088b1 --- /dev/null +++ b/run/django/polls/views.py @@ -0,0 +1,5 @@ +from django.http import HttpResponse + + +def index(request): + return HttpResponse("Hello, world. You're at the polls index.") diff --git a/run/django/requirements.txt b/run/django/requirements.txt new file mode 100644 index 000000000000..cee43bab78c5 --- /dev/null +++ b/run/django/requirements.txt @@ -0,0 +1,7 @@ +django==3.0.5 +django-storages[google]==1.9.1 +django-environ==0.4.5 +# mysqlclient==1.4.1 # Uncomment this line if using MySQL +psycopg2-binary==2.8.4 +gunicorn +google-cloud-secret-manager==0.2.0 From 192547a4a6835719af91c7115235e16b61f6cdea Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 11 May 2020 13:53:56 +1000 Subject: [PATCH 22/91] black, .env file update --- run/django/mysite/settings.py | 105 +++++++++++++++++----------------- 1 file changed, 51 insertions(+), 54 deletions(-) diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 224f1671a2e5..df3205e23246 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -13,26 +13,29 @@ # [START secretconfig] import environ -import google.auth -from google.cloud import secretmanager_v1beta1 as sm -# Import settings with django-environ -env = environ.Env() +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +env_file = os.path.join(BASE_DIR, ".env") -# Import settings from Secret Manager -secrets = {} -_, project = google.auth.default() -client = sm.SecretManagerServiceClient() -parent = client.project_path(project) +if not os.path.isfile(".env"): + import google.auth + from google.cloud import secretmanager_v1beta1 as sm -for secret in ["DATABASE_URL", "GS_BUCKET_NAME", "SECRET_KEY"]: - path = client.secret_version_path(project, secret, "latest") - payload = client.access_secret_version(path).payload.data.decode("UTF-8") - secrets[secret] = payload + _, project = google.auth.default() -os.environ.update(secrets) -# [END secretconfig] + if project: + client = sm.SecretManagerServiceClient() + + SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") + path = client.secret_version_path(project, SETTINGS_NAME, "latest") + payload = client.access_secret_version(path).payload.data.decode("UTF-8") + with open(env_file, "w") as f: + f.write(payload) + +env = environ.Env() +env.read_env(env_file) +# [END secretconfig] SECRET_KEY = env("SECRET_KEY") @@ -42,46 +45,46 @@ # Application definition INSTALLED_APPS = [ - 'polls.apps.PollsConfig', - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'mysite', - 'storages', + "polls.apps.PollsConfig", + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "mysite", + "storages", ] MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", ] -ROOT_URLCONF = 'mysite.urls' +ROOT_URLCONF = "mysite.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", ], }, }, ] -WSGI_APPLICATION = 'mysite.wsgi.application' +WSGI_APPLICATION = "mysite.wsgi.application" # [START dbconfig] @@ -94,26 +97,20 @@ AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, + {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",}, + {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",}, + {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",}, ] # Internationalization # https://docs.djangoproject.com/en/3.0/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" USE_I18N = True From 4d8ec6f7ec0e83e0d6282f8585157d184fc119c2 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 15 Jun 2020 12:21:02 +1000 Subject: [PATCH 23/91] Document Dockerfile, add PYTHONUNBUFFERED --- run/django/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/run/django/Dockerfile b/run/django/Dockerfile index 95e7953f86d9..17f54919cde7 100644 --- a/run/django/Dockerfile +++ b/run/django/Dockerfile @@ -5,7 +5,10 @@ FROM python:3.8-slim ENV APP_HOME /app WORKDIR $APP_HOME -# Install dependencies. +# Removes output stream buffering, allowing for more efficient logging +ENV PYTHONUNBUFFERED 1 + +# Install dependencies COPY requirements.txt . RUN pip install -r requirements.txt From 9fb3094de80dfd4831bb879c73815b768407c682 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 15 Jun 2020 12:21:16 +1000 Subject: [PATCH 24/91] Document cloudmigrate.yaml --- run/django/cloudmigrate.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml index 01a2173fb4a6..d49f42bc7ecf 100644 --- a/run/django/cloudmigrate.yaml +++ b/run/django/cloudmigrate.yaml @@ -1,16 +1,24 @@ steps: + +# Build the image - name: "gcr.io/cloud-builders/docker" args: ["build", "-t", "gcr.io/${PROJECT_ID}/polls", "."] +# Push the image to the Container Registry - name: "gcr.io/cloud-builders/docker" args: ["push", "gcr.io/${PROJECT_ID}/polls"] +# Apply the Django database migration against the Cloud SQL database - name: "gcr.io/google-appengine/exec-wrapper" args: ["-i", "gcr.io/$PROJECT_ID/polls", "-s", "${PROJECT_ID}:us-central1:django-instance", "--", "python", "manage.py", "migrate"] +# Apply the static migrations against the Cloud Storage bucket - name: "gcr.io/google-appengine/exec-wrapper" args: ["-i", "gcr.io/$PROJECT_ID/polls", "-s", "${PROJECT_ID}:us-central1:django-instance", "--", "python", "manage.py", "collectstatic", "--no-input"] + +# This Cloud Build configuration could be extended to include the gcloud run deploy +# step, see https://cloud.google.com/run/docs/continuous-deployment-with-cloud-build From 885d44f8ce2146cbea4b0b1900e2a59f529ceb2d Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 15 Jun 2020 12:21:27 +1000 Subject: [PATCH 25/91] Formatiting --- run/django/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run/django/README.md b/run/django/README.md index c7e277de6676..f27567ae40f5 100644 --- a/run/django/README.md +++ b/run/django/README.md @@ -6,9 +6,9 @@ [shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=run/django/README.md This repository is an example of how to run a [Django](https://www.djangoproject.com/) -app on Google Cloud Run (fully managed). It uses the -[Writing your first Django app](https://docs.djangoproject.com/en/3.0/#first-steps) ([Part 1](https://docs.djangoproject.com/en/3.0/intro/tutorial01/) and [Part 2](https://docs.djangoproject.com/en/3.0/intro/tutorial02/)) as the -example app to deploy. +app on Google Cloud Run (fully managed). + +The Django application used in this tutorial is the [Writing your first Django app](https://docs.djangoproject.com/en/3.0/#first-steps), after completing [Part 1](https://docs.djangoproject.com/en/3.0/intro/tutorial01/) and [Part 2](https://docs.djangoproject.com/en/3.0/intro/tutorial02/)). # Tutorial From b56ecdf20baf70fd30ff178292c3364118066523 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 15 Jun 2020 12:21:37 +1000 Subject: [PATCH 26/91] Bump django, remove mysql --- run/django/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/run/django/requirements.txt b/run/django/requirements.txt index cee43bab78c5..4758bfb5e0c7 100644 --- a/run/django/requirements.txt +++ b/run/django/requirements.txt @@ -1,7 +1,6 @@ -django==3.0.5 +django==3.0.7 django-storages[google]==1.9.1 django-environ==0.4.5 -# mysqlclient==1.4.1 # Uncomment this line if using MySQL psycopg2-binary==2.8.4 gunicorn google-cloud-secret-manager==0.2.0 From 3897badd1e459f34cf94bda11c42a1868f938958 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 2 Oct 2020 12:54:01 +1000 Subject: [PATCH 27/91] Update secretmanager --- .../mysite/migrations/0001_createsuperuser.py | 28 +++++++++++++++++++ run/django/mysite/settings.py | 4 +-- 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 run/django/mysite/migrations/0001_createsuperuser.py diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py new file mode 100644 index 000000000000..ceaa6a59bc8a --- /dev/null +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -0,0 +1,28 @@ +from django.db import migrations + +import google.auth +from google.cloud import secretmanager_v1 as sm + + +def createsuperuser(apps, schema_editor): + # Retrieve secret from Secret Manager + _, project = google.auth.default() + client = sm.SecretManagerServiceClient() + path = f"projects/{project}/secrets/superuser_password/versions/latest" + admin_password = client.access_secret_version(path).payload.data.decode("UTF-8") + + # Create a new user using acquired password + from django.contrib.auth.models import User + User.objects.create_superuser("admin", password=admin_password) + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.RunPython(createsuperuser) + ] diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index df3205e23246..4c4a7b497e07 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -19,7 +19,7 @@ if not os.path.isfile(".env"): import google.auth - from google.cloud import secretmanager_v1beta1 as sm + from google.cloud import secretmanager_v1 as sm _, project = google.auth.default() @@ -27,7 +27,7 @@ client = sm.SecretManagerServiceClient() SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") - path = client.secret_version_path(project, SETTINGS_NAME, "latest") + path = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" payload = client.access_secret_version(path).payload.data.decode("UTF-8") with open(env_file, "w") as f: From a453f02ed52a989bb7f845b7eec11902150bd6a0 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 2 Oct 2020 12:54:14 +1000 Subject: [PATCH 28/91] Update pinned dependencies --- run/django/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/run/django/requirements.txt b/run/django/requirements.txt index 4758bfb5e0c7..b6960e3a5144 100644 --- a/run/django/requirements.txt +++ b/run/django/requirements.txt @@ -1,6 +1,6 @@ -django==3.0.7 -django-storages[google]==1.9.1 +Django==3.1.2 +django-storages[google]==1.10.1 django-environ==0.4.5 psycopg2-binary==2.8.4 -gunicorn -google-cloud-secret-manager==0.2.0 +gunicorn==20.0.4 +google-cloud-secret-manager==2.0.0 From 7981375ad4a9ab555d3e34048c66e8aa2f9f6bc3 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 2 Oct 2020 14:00:57 +1000 Subject: [PATCH 29/91] ensure path is set correctly with api update --- run/django/mysite/migrations/0001_createsuperuser.py | 4 ++-- run/django/mysite/settings.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index ceaa6a59bc8a..25233a816e49 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -8,8 +8,8 @@ def createsuperuser(apps, schema_editor): # Retrieve secret from Secret Manager _, project = google.auth.default() client = sm.SecretManagerServiceClient() - path = f"projects/{project}/secrets/superuser_password/versions/latest" - admin_password = client.access_secret_version(path).payload.data.decode("UTF-8") + name = f"projects/{project}/secrets/superuser_password/versions/latest" + admin_password = client.access_secret_version(name=name).payload.data.decode("UTF-8") # Create a new user using acquired password from django.contrib.auth.models import User diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 4c4a7b497e07..a80b50b23fff 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -27,8 +27,8 @@ client = sm.SecretManagerServiceClient() SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") - path = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" - payload = client.access_secret_version(path).payload.data.decode("UTF-8") + name = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" + payload = client.access_secret_version(name=name).payload.data.decode("UTF-8") with open(env_file, "w") as f: f.write(payload) From 756daf97b16424395e182f6a884035cf9108abe3 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 2 Oct 2020 14:01:23 +1000 Subject: [PATCH 30/91] format --- run/django/mysite/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index a80b50b23fff..9498fbfafe3e 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -41,7 +41,6 @@ ALLOWED_HOSTS = ["*"] - # Application definition INSTALLED_APPS = [ From 86573fe6889848a4bebe58b0585915fd2b1244a1 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 12 Oct 2020 14:11:53 +1100 Subject: [PATCH 31/91] Generalise cloudmigrate file --- run/django/cloudmigrate.yaml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml index d49f42bc7ecf..eeff439cbe83 100644 --- a/run/django/cloudmigrate.yaml +++ b/run/django/cloudmigrate.yaml @@ -2,23 +2,28 @@ steps: # Build the image - name: "gcr.io/cloud-builders/docker" - args: ["build", "-t", "gcr.io/${PROJECT_ID}/polls", "."] + args: ["build", "-t", "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}", "."] # Push the image to the Container Registry - name: "gcr.io/cloud-builders/docker" - args: ["push", "gcr.io/${PROJECT_ID}/polls"] + args: ["push", "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}"] # Apply the Django database migration against the Cloud SQL database - name: "gcr.io/google-appengine/exec-wrapper" - args: ["-i", "gcr.io/$PROJECT_ID/polls", - "-s", "${PROJECT_ID}:us-central1:django-instance", + args: ["-i", "gcr.io/$PROJECT_ID/${_SERVICE_NAME}", + "-s", "${PROJECT_ID}:${_REGION}:${_INSTANCE_NAME}", "--", "python", "manage.py", "migrate"] # Apply the static migrations against the Cloud Storage bucket - name: "gcr.io/google-appengine/exec-wrapper" - args: ["-i", "gcr.io/$PROJECT_ID/polls", - "-s", "${PROJECT_ID}:us-central1:django-instance", + args: ["-i", "gcr.io/$PROJECT_ID/${_SERVICE_NAME}", + "-s", "${PROJECT_ID}:${_REGION}:${_INSTANCE_NAME}", "--", "python", "manage.py", "collectstatic", "--no-input"] -# This Cloud Build configuration could be extended to include the gcloud run deploy -# step, see https://cloud.google.com/run/docs/continuous-deployment-with-cloud-build +substitutions: + _INSTANCE_NAME: django-instance + _REGION: us-central1 + _SERVICE_NAME: polls + +images: + - "gcr.io/${PROJECT_ID}/${_SERVICE_NAME} From 9a94cb265ae462a6f106f698eceaca9f077c1bdb Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 12 Oct 2020 14:13:49 +1100 Subject: [PATCH 32/91] update service default --- run/django/cloudmigrate.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml index eeff439cbe83..93d3e7d34b60 100644 --- a/run/django/cloudmigrate.yaml +++ b/run/django/cloudmigrate.yaml @@ -23,7 +23,7 @@ steps: substitutions: _INSTANCE_NAME: django-instance _REGION: us-central1 - _SERVICE_NAME: polls + _SERVICE_NAME: polls-service images: - "gcr.io/${PROJECT_ID}/${_SERVICE_NAME} From 6426d01f783fb8368c509527ea0b2b68fe4cf8a9 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 19 Oct 2020 16:47:15 +1100 Subject: [PATCH 33/91] add tests --- run/django/.gcloudignore | 3 + run/django/cloudmigrate.yaml | 2 +- run/django/e2e_test.py | 409 +++++++++++++++++++++++++++++++ run/django/noxfile_config.py | 39 +++ run/django/polls/test_polls.py | 22 ++ run/django/polls/tests.py | 3 - run/django/requirements-test.txt | 3 + 7 files changed, 477 insertions(+), 4 deletions(-) create mode 100644 run/django/e2e_test.py create mode 100644 run/django/noxfile_config.py create mode 100644 run/django/polls/test_polls.py delete mode 100644 run/django/polls/tests.py create mode 100644 run/django/requirements-test.txt diff --git a/run/django/.gcloudignore b/run/django/.gcloudignore index c6be208a110f..c0abe06014ba 100644 --- a/run/django/.gcloudignore +++ b/run/django/.gcloudignore @@ -15,3 +15,6 @@ # Python pycache: __pycache__/ + +venv/ +.env diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml index 93d3e7d34b60..cad8eae78491 100644 --- a/run/django/cloudmigrate.yaml +++ b/run/django/cloudmigrate.yaml @@ -26,4 +26,4 @@ substitutions: _SERVICE_NAME: polls-service images: - - "gcr.io/${PROJECT_ID}/${_SERVICE_NAME} + - "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}" \ No newline at end of file diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py new file mode 100644 index 000000000000..896381605f49 --- /dev/null +++ b/run/django/e2e_test.py @@ -0,0 +1,409 @@ +# Copyright 2020 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This test creates a Cloud SQL instance, a Cloud Storage bucket, associated +# secrets, and deploys a Django service + +import os +import subprocess +import uuid + +import pytest +import requests +from google.cloud import secretmanager_v1 as sm + +# Unique suffix to create distinct service names +SUFFIX = uuid.uuid4().hex[:10] + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +DATABASE_VERISON = "POSTGRES_12" +REGION = "us-central1" +DATABASE_TIER = "db-f1-micro" + +CLOUDSQL_INSTANCE = f"instance-{SUFFIX}" +STORAGE_BUCKET = f"{PROJECT}-media-{SUFFIX}" +DATABASE_NAME = f"polls-{SUFFIX}" +DATABASE_USERNAME = f"django-{SUFFIX}" +DATABASE_PASSWORD = uuid.uuid4().hex[:26] + +ADMIN_NAME = "admin" +ADMIN_PASSWORD = uuid.uuid4().hex[:26] + +# These are hardcoded elsewhere in the code +SECRET_SETTINGS_NAME = "django_settings" +SECRET_ADMINPASS_NAME = "superuser_password" + + +@pytest.fixture +def project_number(): + projectnum = ( + subprocess.run( + [ + "gcloud", + "projects", + "list", + "--filter", + PROJECT, + "--format", + "value(projectNumber)", + ], + stdout=subprocess.PIPE, + check=True, + ) + .stdout.strip() + .decode() + ) + + yield projectnum + + +@pytest.fixture +def postgres_host(project_number): + # Create database instance + subprocess.run( + [ + "gcloud", + "sql", + "instances", + "create", + CLOUDSQL_INSTANCE, + "--database-version", + DATABASE_VERISON, + "--tier", + DATABASE_TIER, + "--region", + REGION, + "--project", + PROJECT, + ], + check=True, + ) + + # Create database + subprocess.run( + [ + "gcloud", + "sql", + "databases", + "create", + DATABASE_NAME, + "--instance", + CLOUDSQL_INSTANCE, + "--project", + PROJECT, + ], + check=True, + ) + # Create User + # NOTE Creating a user via the tutorial method is not automatable. + subprocess.run( + [ + "gcloud", + "sql", + "users", + "create", + DATABASE_USERNAME, + "--password", + DATABASE_PASSWORD, + "--instance", + CLOUDSQL_INSTANCE, + "--project", + PROJECT, + ], + check=True, + ) + + # Allow access to database from Cloud Build + subprocess.run( + [ + "gcloud", + "projects", + "add-iam-policy-binding", + PROJECT, + "--member", + f"serviceAccount:{project_number}@cloudbuild.gserviceaccount.com", + "--role", + "roles/cloudsql.client", + ], + check=True, + ) + + yield CLOUDSQL_INSTANCE + + subprocess.run( + [ + "gcloud", + "sql", + "databases", + "delete", + DATABASE_NAME, + "--instance", + CLOUDSQL_INSTANCE, + "--quiet", + ], + check=True, + ) + + subprocess.run( + [ + "gcloud", + "sql", + "users", + "delete", + DATABASE_USERNAME, + "--instance", + CLOUDSQL_INSTANCE, + "--quiet", + ], + check=True, + ) + + subprocess.run( + ["gcloud", "sql", "instances" "delete", CLOUDSQL_INSTANCE, "--quiet"], + check=True, + ) + + +@pytest.fixture +def media_bucket(): + # Create storage bucket + subprocess.run(["gsutil", "mb", "-l", REGION, f"gs://{STORAGE_BUCKET}"], check=True) + + yield STORAGE_BUCKET + + # Delete storage bucket contents, delete bucket + subprocess.run(["gsutil", "-m", "rm", "-r", f"gs://{STORAGE_BUCKET}"], check=True) + subprocess.run(["gsutil", "rb", f"gs://{STORAGE_BUCKET}"], check=True) + + +@pytest.fixture +def secrets(project_number): + # Create a number of secrets and allow Google Cloud services access to them + + def create_secret(name, value): + secret = client.create_secret( + request={ + "parent": f"projects/{PROJECT}", + "secret": {"replication": {"automatic": {}}}, + "secret_id": name, + } + ) + + client.add_secret_version( + request={"parent": secret.name, "payload": {"data": value.encode("UTF-8")}} + ) + print(f"DEBUG: {name}\n{value}") + + def allow_access(name, member): + subprocess.run( + [ + "gcloud", + "secrets", + "add-iam-policy-binding", + name, + "--member", + member, + "--role", + "roles/secretmanager.secretAccessor", + ], + check=True, + ) + + client = sm.SecretManagerServiceClient() + secret_key = uuid.uuid4().hex[:56] + settings = f""" +DATABASE_URL=postgres://{DATABASE_USERNAME}:{DATABASE_PASSWORD}@//cloudsql/{PROJECT}:{REGION}:{CLOUDSQL_INSTANCE}/{DATABASE_NAME} +GS_BUCKET_NAME={STORAGE_BUCKET} +SECRET_KEY={secret_key} + """ + + create_secret(SECRET_SETTINGS_NAME, settings) + allow_access( + SECRET_SETTINGS_NAME, + f"serviceAccount:{project_number}-compute@developer.gserviceaccount.com", + ) + allow_access( + SECRET_SETTINGS_NAME, + f"serviceAccount:{project_number}@cloudbuild.gserviceaccount.com", + ) + + create_secret(SECRET_ADMINPASS_NAME, ADMIN_PASSWORD) + allow_access( + SECRET_ADMINPASS_NAME, + f"serviceAccount:{project_number}@cloudbuild.gserviceaccount.com", + ) + + yield SECRET_SETTINGS_NAME + + # delete secrets + subprocess.run( + ["gcloud", "secrets", "delete", SECRET_ADMINPASS_NAME, "--quiet"], check=True + ) + subprocess.run( + ["gcloud", "secrets", "delete", SECRET_SETTINGS_NAME, "--quiet"], check=True + ) + + +@pytest.fixture +def container_image(postgres_host, media_bucket, secrets): + # Build container image for Cloud Run deployment + image_name = f"gcr.io/{PROJECT}/polls-{SUFFIX}" + service_name = f"polls-{SUFFIX}" + cloudbuild_config = "cloudmigrate.yaml" + subprocess.run( + [ + "gcloud", + "builds", + "submit", + "--config", + cloudbuild_config, + "--substitutions", + f"_INSTANCE_NAME={postgres_host},_REGION={REGION},_SERVICE_NAME={service_name}", + "--project", + PROJECT, + ], + check=True, + ) + yield image_name + + # Delete container image + subprocess.run( + [ + "gcloud", + "container", + "images", + "delete", + image_name, + "--quiet", + "--project", + PROJECT, + ], + check=True, + ) + + +@pytest.fixture +def deployed_service(container_image): + # Deploy image to Cloud Run + service_name = f"polls-{SUFFIX}" + subprocess.run( + [ + "gcloud", + "run", + "deploy", + service_name, + "--image", + container_image, + "--platform=managed", + "--no-allow-unauthenticated", + "--region", + REGION, + "--add-cloudsql-instances", + f"{PROJECT}:{REGION}:{CLOUDSQL_INSTANCE}", + "--project", + PROJECT, + ], + check=True, + ) + yield service_name + + # Delete Cloud Run service + subprocess.run( + [ + "gcloud", + "run", + "services", + "delete", + service_name, + "--platform=managed", + "--region=us-central1", + "--quiet", + "--project", + PROJECT, + ], + check=True, + ) + + +@pytest.fixture +def service_url_auth_token(deployed_service): + # Get Cloud Run service URL and auth token + service_url = ( + subprocess.run( + [ + "gcloud", + "run", + "services", + "describe", + deployed_service, + "--platform", + "managed", + "--region", + REGION, + "--format", + "value(status.url)", + "--project", + PROJECT, + ], + stdout=subprocess.PIPE, + check=True, + ) + .stdout.strip() + .decode() + ) + auth_token = ( + subprocess.run( + ["gcloud", "auth", "print-identity-token"], + stdout=subprocess.PIPE, + check=True, + ) + .stdout.strip() + .decode() + ) + + yield service_url, auth_token + + # no deletion needed + + +def test_end_to_end(service_url_auth_token): + service_url, auth_token = service_url_auth_token + headers = {"Authorization": f"Bearer {auth_token}"} + login_slug = "/admin/login/?next=/admin/" + client = requests.session() + + # Check homepage + response = client.get(service_url, headers=headers) + body = response.text + + assert response.status_code == 200 + assert "Hello, world" in body + + # Load login page, collecting csrf token + client.get(service_url + login_slug, headers=headers) + csrftoken = client.cookies["csrftoken"] + + # Log into Django admin + payload = { + "username": ADMIN_NAME, + "password": ADMIN_PASSWORD, + "csrfmiddlewaretoken": csrftoken, + } + response = client.post(service_url + login_slug, data=payload, headers=headers) + body = response.text + + # Check Django admin landing page + assert response.status_code == 200 + assert "Site administration" in body + assert "Polls" in body diff --git a/run/django/noxfile_config.py b/run/django/noxfile_config.py new file mode 100644 index 000000000000..af47c4048cd6 --- /dev/null +++ b/run/django/noxfile_config.py @@ -0,0 +1,39 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# You can copy this file into your directory, then it will be imported from +# the noxfile.py. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + + # We only run the cloud run tests in py38 session. + 'ignored_versions': ["2.7", "3.6", "3.7"], + + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + 'envs': {}, +} diff --git a/run/django/polls/test_polls.py b/run/django/polls/test_polls.py new file mode 100644 index 000000000000..4588d730242c --- /dev/null +++ b/run/django/polls/test_polls.py @@ -0,0 +1,22 @@ +# Copyright 2020 Google LLC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from django.test import Client, TestCase # noqa: 401 + + +class PollViewTests(TestCase): + def test_index_view(self): + response = self.client.get('/') + assert response.status_code == 200 + assert 'Hello, world' in str(response.content) diff --git a/run/django/polls/tests.py b/run/django/polls/tests.py deleted file mode 100644 index 7ce503c2dd97..000000000000 --- a/run/django/polls/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/run/django/requirements-test.txt b/run/django/requirements-test.txt new file mode 100644 index 000000000000..eab71ebbbc62 --- /dev/null +++ b/run/django/requirements-test.txt @@ -0,0 +1,3 @@ +pytest==6.0.1 +pytest-django==3.10.0 +requests==2.24.0 \ No newline at end of file From 8fa8c19b572cd1bb90781da0b0d53356b0453c9f Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 19 Oct 2020 16:50:01 +1100 Subject: [PATCH 34/91] Update CODEOWNERS --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 765aabef9e87..a3c46b9fc06e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -54,6 +54,7 @@ /profiler/**/*.py @kalyanac @GoogleCloudPlatform/python-samples-owners /pubsub/**/*.py @anguillanneuf @hongalex @GoogleCloudPlatform/python-samples-owners /run/**/*.py @averikitsch @grant @GoogleCloudPlatform/python-samples-owners +/run/django/**/*.py @glasnt @GoogleCloudPlatform/python-samples-owners /scheduler/**/*.py @averikitsch @GoogleCloudPlatform/python-samples-owners /spanner/**/*.py @larkee @GoogleCloudPlatform/python-samples-owners /speech/**/*.py @telpirion @sirtorry @GoogleCloudPlatform/python-samples-owners From be5911a57e954cc5b9d19fb9a28e8e922ead8b1a Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 20 Oct 2020 08:52:07 +1100 Subject: [PATCH 35/91] Licence headers --- run/django/Dockerfile | 14 ++++++++++++++ run/django/cloudmigrate.yaml | 14 ++++++++++++++ run/django/manage.py | 15 +++++++++++++++ run/django/mysite/settings.py | 14 ++++++++++++++ run/django/mysite/urls.py | 14 ++++++++++++++ run/django/mysite/wsgi.py | 14 ++++++++++++++ run/django/polls/admin.py | 14 ++++++++++++++ run/django/polls/apps.py | 14 ++++++++++++++ run/django/polls/models.py | 14 ++++++++++++++ run/django/polls/urls.py | 14 ++++++++++++++ run/django/polls/views.py | 14 ++++++++++++++ 11 files changed, 155 insertions(+) diff --git a/run/django/Dockerfile b/run/django/Dockerfile index 17f54919cde7..7e435bca3772 100644 --- a/run/django/Dockerfile +++ b/run/django/Dockerfile @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Use an official lightweight Python image. # https://hub.docker.com/_/python FROM python:3.8-slim diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml index cad8eae78491..f24cd1175a81 100644 --- a/run/django/cloudmigrate.yaml +++ b/run/django/cloudmigrate.yaml @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + steps: # Build the image diff --git a/run/django/manage.py b/run/django/manage.py index 341863cf62fa..934371b8740a 100755 --- a/run/django/manage.py +++ b/run/django/manage.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Django's command-line utility for administrative tasks.""" import os import sys diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 9498fbfafe3e..64baafb8dd33 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ Django settings for mysite project. diff --git a/run/django/mysite/urls.py b/run/django/mysite/urls.py index ca908e72e3c2..ffef6e232864 100644 --- a/run/django/mysite/urls.py +++ b/run/django/mysite/urls.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from django.contrib import admin from django.urls import include, path diff --git a/run/django/mysite/wsgi.py b/run/django/mysite/wsgi.py index dbe7bb5fcc88..78282856146e 100644 --- a/run/django/mysite/wsgi.py +++ b/run/django/mysite/wsgi.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ WSGI config for mysite project. diff --git a/run/django/polls/admin.py b/run/django/polls/admin.py index 6af8ff674594..e90a167fb2a3 100644 --- a/run/django/polls/admin.py +++ b/run/django/polls/admin.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from django.contrib import admin from .models import Question diff --git a/run/django/polls/apps.py b/run/django/polls/apps.py index d0f109e60e45..f692aa20d159 100644 --- a/run/django/polls/apps.py +++ b/run/django/polls/apps.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from django.apps import AppConfig diff --git a/run/django/polls/models.py b/run/django/polls/models.py index 48780e636428..c1e2e9f7a913 100644 --- a/run/django/polls/models.py +++ b/run/django/polls/models.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from django.db import models diff --git a/run/django/polls/urls.py b/run/django/polls/urls.py index 88a9caca8ceb..1a67f0009d00 100644 --- a/run/django/polls/urls.py +++ b/run/django/polls/urls.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from django.urls import path from . import views diff --git a/run/django/polls/views.py b/run/django/polls/views.py index 963b6f7088b1..bda93457d1e5 100644 --- a/run/django/polls/views.py +++ b/run/django/polls/views.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from django.http import HttpResponse From cb5992e8bc972ee9831f06dfa239f2a0906caf72 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 20 Oct 2020 08:52:14 +1100 Subject: [PATCH 36/91] Remove envvar declaration --- run/django/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/run/django/Dockerfile b/run/django/Dockerfile index 7e435bca3772..986f2e3a34c1 100644 --- a/run/django/Dockerfile +++ b/run/django/Dockerfile @@ -29,10 +29,6 @@ RUN pip install -r requirements.txt # Copy local code to the container image. COPY . . -# Service must listen to $PORT environment variable. -# This default value facilitates local development. -ENV PORT 8080 - # Run the web service on container startup. Here we use the gunicorn # webserver, with one worker process and 8 threads. # For environments with multiple CPU cores, increase the number of workers From 8c643d404d03e247f57392af02782ad4f2349da6 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 20 Oct 2020 08:52:30 +1100 Subject: [PATCH 37/91] cleanup readme --- run/django/README.md | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/run/django/README.md b/run/django/README.md index f27567ae40f5..564991ee8a4b 100644 --- a/run/django/README.md +++ b/run/django/README.md @@ -1,4 +1,4 @@ -# Getting started with Django on Google Cloud Platform on Cloud Run (fully managed) +# Getting started with Django on Cloud Run [![Open in Cloud Shell][shell_img]][shell_link] @@ -8,18 +8,8 @@ This repository is an example of how to run a [Django](https://www.djangoproject.com/) app on Google Cloud Run (fully managed). -The Django application used in this tutorial is the [Writing your first Django app](https://docs.djangoproject.com/en/3.0/#first-steps), after completing [Part 1](https://docs.djangoproject.com/en/3.0/intro/tutorial01/) and [Part 2](https://docs.djangoproject.com/en/3.0/intro/tutorial02/)). +The Django application used in this tutorial is the [Writing your first Django app](https://docs.djangoproject.com/en/3.0/#first-steps), after completing [Part 1](https://docs.djangoproject.com/en/3.0/intro/tutorial01/) and [Part 2](https://docs.djangoproject.com/en/3.0/intro/tutorial02/). # Tutorial -See our [Running Django on Cloud Run (fully managed)](https://cloud.google.com/python/django/run) tutorial for instructions for setting up and deploying this sample application. - - -## Contributing changes - -* See [CONTRIBUTING.md](CONTRIBUTING.md) - - -## Licensing - -* See [LICENSE](LICENSE) +See our [Running Django on Cloud Run (fully managed)](https://cloud.google.com/python/django/run) tutorial for instructions for setting up and deploying this sample application. \ No newline at end of file From 98837e1ac2d94cc53f1c61a153bf8d25bd65a67f Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 20 Oct 2020 08:52:43 +1100 Subject: [PATCH 38/91] Fix import names, region tags --- .../mysite/migrations/0001_createsuperuser.py | 27 ++++++++++++++++--- run/django/mysite/settings.py | 17 ++++++------ run/django/polls/test_polls.py | 4 +-- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index 25233a816e49..af4e51e5c830 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -1,13 +1,34 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from django.db import migrations import google.auth -from google.cloud import secretmanager_v1 as sm +from google.cloud import secretmanager_v1 def createsuperuser(apps, schema_editor): - # Retrieve secret from Secret Manager + """ + Dynamically create an admin user as part of a migration + Password is pulled from Secret Manger (previously created as part of tutorial) + """ + client = secretmanager_v1.SecretManagerServiceClient() + + # Get project value for identifying current context _, project = google.auth.default() - client = sm.SecretManagerServiceClient() + + # Retrieve the previously stored admin passowrd name = f"projects/{project}/secrets/superuser_password/versions/latest" admin_password = client.access_secret_version(name=name).payload.data.decode("UTF-8") diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 64baafb8dd33..03cc58be8bf6 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -25,20 +25,21 @@ """ import os -# [START secretconfig] +# [START run_secretconfig] import environ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) env_file = os.path.join(BASE_DIR, ".env") +# If no .env has been provided, pull it from Secret Manager, storing it locally if not os.path.isfile(".env"): import google.auth - from google.cloud import secretmanager_v1 as sm + from google.cloud import secretmanager_v1 _, project = google.auth.default() if project: - client = sm.SecretManagerServiceClient() + client = secretmanager_v1.SecretManagerServiceClient() SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") name = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" @@ -49,7 +50,7 @@ env = environ.Env() env.read_env(env_file) -# [END secretconfig] +# [END run_secretconfig] SECRET_KEY = env("SECRET_KEY") @@ -100,10 +101,10 @@ WSGI_APPLICATION = "mysite.wsgi.application" -# [START dbconfig] +# [START run_dbconfig] # Use django-environ to define the connection string DATABASES = {"default": env.db()} -# [END dbconfig] +# [END run_dbconfig] # Password validation # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators @@ -131,11 +132,11 @@ USE_TZ = True -# [START staticconfig] +# [START run_staticconfig] # Define static storage via django-storages[google] GS_BUCKET_NAME = env("GS_BUCKET_NAME") DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" STATICFILES_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" GS_DEFAULT_ACL = "publicRead" -# [END staticconfig] +# [END run_staticconfig] diff --git a/run/django/polls/test_polls.py b/run/django/polls/test_polls.py index 4588d730242c..7024a5001c23 100644 --- a/run/django/polls/test_polls.py +++ b/run/django/polls/test_polls.py @@ -17,6 +17,6 @@ class PollViewTests(TestCase): def test_index_view(self): - response = self.client.get('/') + response = self.client.get("/") assert response.status_code == 200 - assert 'Hello, world' in str(response.content) + assert "Hello, world" in str(response.content) From 5e1cb0a4cb697ad1c8a38f9956b544cb805146c2 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 20 Oct 2020 08:53:50 +1100 Subject: [PATCH 39/91] black --- run/django/manage.py | 4 ++-- run/django/mysite/asgi.py | 2 +- run/django/mysite/settings.py | 12 +++++++++--- run/django/mysite/urls.py | 4 ++-- run/django/mysite/wsgi.py | 2 +- run/django/noxfile_config.py | 9 +++------ run/django/polls/apps.py | 2 +- run/django/polls/models.py | 2 +- run/django/polls/urls.py | 2 +- 9 files changed, 21 insertions(+), 18 deletions(-) diff --git a/run/django/manage.py b/run/django/manage.py index 934371b8740a..868c06851602 100755 --- a/run/django/manage.py +++ b/run/django/manage.py @@ -20,7 +20,7 @@ def main(): - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: @@ -32,5 +32,5 @@ def main(): execute_from_command_line(sys.argv) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/run/django/mysite/asgi.py b/run/django/mysite/asgi.py index 35d925e4c5a8..5b7d1a17eec1 100644 --- a/run/django/mysite/asgi.py +++ b/run/django/mysite/asgi.py @@ -11,6 +11,6 @@ from django.core.asgi import get_asgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") application = get_asgi_application() diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 03cc58be8bf6..431d7430b722 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -113,9 +113,15 @@ { "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, - {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",}, - {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",}, - {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",}, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, ] diff --git a/run/django/mysite/urls.py b/run/django/mysite/urls.py index ffef6e232864..7e7f336ce614 100644 --- a/run/django/mysite/urls.py +++ b/run/django/mysite/urls.py @@ -16,6 +16,6 @@ from django.urls import include, path urlpatterns = [ - path('', include('polls.urls')), - path('admin/', admin.site.urls), + path("", include("polls.urls")), + path("admin/", admin.site.urls), ] diff --git a/run/django/mysite/wsgi.py b/run/django/mysite/wsgi.py index 78282856146e..444d68f6e5df 100644 --- a/run/django/mysite/wsgi.py +++ b/run/django/mysite/wsgi.py @@ -25,6 +25,6 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") application = get_wsgi_application() diff --git a/run/django/noxfile_config.py b/run/django/noxfile_config.py index af47c4048cd6..696c09e6ce14 100644 --- a/run/django/noxfile_config.py +++ b/run/django/noxfile_config.py @@ -22,18 +22,15 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # We only run the cloud run tests in py38 session. - 'ignored_versions': ["2.7", "3.6", "3.7"], - + "ignored_versions": ["2.7", "3.6", "3.7"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - 'envs': {}, + "envs": {}, } diff --git a/run/django/polls/apps.py b/run/django/polls/apps.py index f692aa20d159..156a0cf65582 100644 --- a/run/django/polls/apps.py +++ b/run/django/polls/apps.py @@ -16,4 +16,4 @@ class PollsConfig(AppConfig): - name = 'polls' + name = "polls" diff --git a/run/django/polls/models.py b/run/django/polls/models.py index c1e2e9f7a913..9234289b05e4 100644 --- a/run/django/polls/models.py +++ b/run/django/polls/models.py @@ -17,7 +17,7 @@ class Question(models.Model): question_text = models.CharField(max_length=200) - pub_date = models.DateTimeField('date published') + pub_date = models.DateTimeField("date published") class Choice(models.Model): diff --git a/run/django/polls/urls.py b/run/django/polls/urls.py index 1a67f0009d00..daaf7a501182 100644 --- a/run/django/polls/urls.py +++ b/run/django/polls/urls.py @@ -17,5 +17,5 @@ from . import views urlpatterns = [ - path('', views.index, name='index'), + path("", views.index, name="index"), ] From 911ca507348b2fdc04ee078ecb65e73339dedee7 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 20 Oct 2020 08:59:12 +1100 Subject: [PATCH 40/91] black --- .../mysite/migrations/0001_createsuperuser.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index af4e51e5c830..1e461a6af97d 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -24,16 +24,19 @@ def createsuperuser(apps, schema_editor): Password is pulled from Secret Manger (previously created as part of tutorial) """ client = secretmanager_v1.SecretManagerServiceClient() - + # Get project value for identifying current context _, project = google.auth.default() # Retrieve the previously stored admin passowrd name = f"projects/{project}/secrets/superuser_password/versions/latest" - admin_password = client.access_secret_version(name=name).payload.data.decode("UTF-8") + admin_password = client.access_secret_version(name=name).payload.data.decode( + "UTF-8" + ) # Create a new user using acquired password from django.contrib.auth.models import User + User.objects.create_superuser("admin", password=admin_password) @@ -41,9 +44,6 @@ class Migration(migrations.Migration): initial = True - dependencies = [ - ] + dependencies = [] - operations = [ - migrations.RunPython(createsuperuser) - ] + operations = [migrations.RunPython(createsuperuser)] From 877c65881f43f1fcac10ce1f3b43dd148414f3dd Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 20 Oct 2020 12:53:48 +1100 Subject: [PATCH 41/91] Format YAML --- run/django/cloudmigrate.yaml | 56 +++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml index f24cd1175a81..ba1596c45a92 100644 --- a/run/django/cloudmigrate.yaml +++ b/run/django/cloudmigrate.yaml @@ -13,31 +13,47 @@ # limitations under the License. steps: + # Build the image + - name: "gcr.io/cloud-builders/docker" + args: ["build", "-t", "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}", "."] -# Build the image -- name: "gcr.io/cloud-builders/docker" - args: ["build", "-t", "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}", "."] + # Push the image to the Container Registry + - name: "gcr.io/cloud-builders/docker" + args: ["push", "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}"] -# Push the image to the Container Registry -- name: "gcr.io/cloud-builders/docker" - args: ["push", "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}"] + # Apply the Django database migration against the Cloud SQL database + - name: "gcr.io/google-appengine/exec-wrapper" + args: + [ + "-i", + "gcr.io/$PROJECT_ID/${_SERVICE_NAME}", + "-s", + "${PROJECT_ID}:${_REGION}:${_INSTANCE_NAME}", + "--", + "python", + "manage.py", + "migrate", + ] -# Apply the Django database migration against the Cloud SQL database -- name: "gcr.io/google-appengine/exec-wrapper" - args: ["-i", "gcr.io/$PROJECT_ID/${_SERVICE_NAME}", - "-s", "${PROJECT_ID}:${_REGION}:${_INSTANCE_NAME}", - "--", "python", "manage.py", "migrate"] + # Apply the static migrations against the Cloud Storage bucket + - name: "gcr.io/google-appengine/exec-wrapper" + args: + [ + "-i", + "gcr.io/$PROJECT_ID/${_SERVICE_NAME}", + "-s", + "${PROJECT_ID}:${_REGION}:${_INSTANCE_NAME}", + "--", + "python", + "manage.py", + "collectstatic", + "--no-input", + ] -# Apply the static migrations against the Cloud Storage bucket -- name: "gcr.io/google-appengine/exec-wrapper" - args: ["-i", "gcr.io/$PROJECT_ID/${_SERVICE_NAME}", - "-s", "${PROJECT_ID}:${_REGION}:${_INSTANCE_NAME}", - "--", "python", "manage.py", "collectstatic", "--no-input"] - -substitutions: +substitutions: _INSTANCE_NAME: django-instance _REGION: us-central1 _SERVICE_NAME: polls-service -images: - - "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}" \ No newline at end of file +images: + - "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}" From 466a801b844ed005fd6df7a71764952fea899b4b Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 5 Nov 2020 13:55:42 +1100 Subject: [PATCH 42/91] bump dependencies --- run/django/requirements-test.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run/django/requirements-test.txt b/run/django/requirements-test.txt index eab71ebbbc62..28da80f113fa 100644 --- a/run/django/requirements-test.txt +++ b/run/django/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==6.0.1 +pytest==6.1.1 pytest-django==3.10.0 -requests==2.24.0 \ No newline at end of file +requests==2.24.0 From c188733d2188f97fed1fdb206b0e71ccdbf8dcd7 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 5 Nov 2020 13:55:54 +1100 Subject: [PATCH 43/91] Update var names, remove sql instance creation --- run/django/e2e_test.py | 67 ++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 45 deletions(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 896381605f49..14aaebe8614d 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -27,15 +27,15 @@ SUFFIX = uuid.uuid4().hex[:10] PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] -DATABASE_VERISON = "POSTGRES_12" REGION = "us-central1" -DATABASE_TIER = "db-f1-micro" + +CLOUD_STORAGE_BUCKET = f"{PROJECT}-media-{SUFFIX}" CLOUDSQL_INSTANCE = f"instance-{SUFFIX}" -STORAGE_BUCKET = f"{PROJECT}-media-{SUFFIX}" -DATABASE_NAME = f"polls-{SUFFIX}" -DATABASE_USERNAME = f"django-{SUFFIX}" -DATABASE_PASSWORD = uuid.uuid4().hex[:26] +POSTGRES_INSTANCE = "test-instance-pg" +POSTGRES_DATABASE = f"polls-{SUFFIX}" +POSTGRES_USER = f"django-{SUFFIX}" +POSTGRES_PASSWORD = uuid.uuid4().hex[:26] ADMIN_NAME = "admin" ADMIN_PASSWORD = uuid.uuid4().hex[:26] @@ -70,25 +70,7 @@ def project_number(): @pytest.fixture def postgres_host(project_number): - # Create database instance - subprocess.run( - [ - "gcloud", - "sql", - "instances", - "create", - CLOUDSQL_INSTANCE, - "--database-version", - DATABASE_VERISON, - "--tier", - DATABASE_TIER, - "--region", - REGION, - "--project", - PROJECT, - ], - check=True, - ) + # Presume instance already exists # Create database subprocess.run( @@ -97,9 +79,9 @@ def postgres_host(project_number): "sql", "databases", "create", - DATABASE_NAME, + POSTGRES_DATABASE, "--instance", - CLOUDSQL_INSTANCE, + POSTGRES_INSTANCE, "--project", PROJECT, ], @@ -113,11 +95,11 @@ def postgres_host(project_number): "sql", "users", "create", - DATABASE_USERNAME, + POSTGRES_USERNAME, "--password", - DATABASE_PASSWORD, + POSTGRES_PASSWORD, "--instance", - CLOUDSQL_INSTANCE, + POSTGRES_INSTANCE, "--project", PROJECT, ], @@ -147,9 +129,9 @@ def postgres_host(project_number): "sql", "databases", "delete", - DATABASE_NAME, + POSTGRES_DATABASE, "--instance", - CLOUDSQL_INSTANCE, + POSTGRES_INSTANCE, "--quiet", ], check=True, @@ -161,30 +143,25 @@ def postgres_host(project_number): "sql", "users", "delete", - DATABASE_USERNAME, + POSTGRES_USERNAME, "--instance", - CLOUDSQL_INSTANCE, + POSTGRES_INSTANCE, "--quiet", ], check=True, ) - subprocess.run( - ["gcloud", "sql", "instances" "delete", CLOUDSQL_INSTANCE, "--quiet"], - check=True, - ) - @pytest.fixture def media_bucket(): # Create storage bucket - subprocess.run(["gsutil", "mb", "-l", REGION, f"gs://{STORAGE_BUCKET}"], check=True) + subprocess.run(["gsutil", "mb", "-l", REGION, f"gs://{CLOUD_STORAGE_BUCKET}"], check=True) - yield STORAGE_BUCKET + yield CLOUD_STORAGE_BUCKET # Delete storage bucket contents, delete bucket - subprocess.run(["gsutil", "-m", "rm", "-r", f"gs://{STORAGE_BUCKET}"], check=True) - subprocess.run(["gsutil", "rb", f"gs://{STORAGE_BUCKET}"], check=True) + subprocess.run(["gsutil", "-m", "rm", "-r", f"gs://{CLOUD_STORAGE_BUCKET}"], check=True) + subprocess.run(["gsutil", "rb", f"gs://{CLOUD_STORAGE_BUCKET}"], check=True) @pytest.fixture @@ -223,8 +200,8 @@ def allow_access(name, member): client = sm.SecretManagerServiceClient() secret_key = uuid.uuid4().hex[:56] settings = f""" -DATABASE_URL=postgres://{DATABASE_USERNAME}:{DATABASE_PASSWORD}@//cloudsql/{PROJECT}:{REGION}:{CLOUDSQL_INSTANCE}/{DATABASE_NAME} -GS_BUCKET_NAME={STORAGE_BUCKET} +DATABASE_URL=postgres://{POSTGRES_USER}:{POSTGRES_PASSWORD}@//cloudsql/{PROJECT}:{REGION}:{POSTGRES_INSTANCE}/{POSTGRES_DATABASE} +GS_BUCKET_NAME={CLOUD_STORAGE_BUCKET} SECRET_KEY={secret_key} """ From 4d33c687de8a3791dcdab004003a7f8633b74dc6 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 9 Nov 2020 09:32:58 +1100 Subject: [PATCH 44/91] PR Feedback --- run/README.md | 1 + run/django/e2e_test.py | 34 ++++++++++++++++++++++++++------ run/django/polls/test_polls.py | 2 +- run/django/requirements-test.txt | 2 +- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/run/README.md b/run/README.md index c3b24af5b1ce..fbb41d21ca53 100644 --- a/run/README.md +++ b/run/README.md @@ -15,6 +15,7 @@ This directory contains samples for [Google Cloud Run](https://cloud.run). [Clou |[Cloud Pub/Sub][pubsub] | Handling Pub/Sub push messages | [Run on Google Cloud][run_button_pubsub] | |[Cloud SQL (MySQL)][mysql] | Use MySQL with Cloud Run | - | |[Cloud SQL (Postgres)][postgres] | Use Postgres with Cloud Run | - | +|[Django][django] | Deploy Django on Cloud Run | - | For more Cloud Run samples beyond Python, see the main list in the [Cloud Run Samples repository](https://github.com/GoogleCloudPlatform/cloud-run-samples). diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 14aaebe8614d..edb266680bc3 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -19,9 +19,9 @@ import subprocess import uuid +from google.cloud import secretmanager_v1 as sm import pytest import requests -from google.cloud import secretmanager_v1 as sm # Unique suffix to create distinct service names SUFFIX = uuid.uuid4().hex[:10] @@ -95,7 +95,7 @@ def postgres_host(project_number): "sql", "users", "create", - POSTGRES_USERNAME, + POSTGRES_USER, "--password", POSTGRES_PASSWORD, "--instance", @@ -106,7 +106,8 @@ def postgres_host(project_number): check=True, ) - # Allow access to database from Cloud Build + # TODO: Make static fixture + # Allow access to Cloud SQL from Cloud Build subprocess.run( [ "gcloud", @@ -143,7 +144,7 @@ def postgres_host(project_number): "sql", "users", "delete", - POSTGRES_USERNAME, + POSTGRES_USER, "--instance", POSTGRES_INSTANCE, "--quiet", @@ -151,16 +152,37 @@ def postgres_host(project_number): check=True, ) + # TODO: remove when binding is static + # Remove policy binding + subprocess.run( + [ + "gcloud", + "projects", + "remove-iam-policy-binding", + PROJECT, + "--member", + f"serviceAccount:{project_number}@cloudbuild.gserviceaccount.com", + "--role", + "roles/cloudsql.client", + ], + check=True, + ) + @pytest.fixture def media_bucket(): # Create storage bucket - subprocess.run(["gsutil", "mb", "-l", REGION, f"gs://{CLOUD_STORAGE_BUCKET}"], check=True) + subprocess.run( + ["gsutil", "mb", "-l", REGION, f"gs://{CLOUD_STORAGE_BUCKET}"], + check=True, + ) yield CLOUD_STORAGE_BUCKET # Delete storage bucket contents, delete bucket - subprocess.run(["gsutil", "-m", "rm", "-r", f"gs://{CLOUD_STORAGE_BUCKET}"], check=True) + subprocess.run( + ["gsutil", "-m", "rm", "-r", f"gs://{CLOUD_STORAGE_BUCKET}"], check=True + ) subprocess.run(["gsutil", "rb", f"gs://{CLOUD_STORAGE_BUCKET}"], check=True) diff --git a/run/django/polls/test_polls.py b/run/django/polls/test_polls.py index 7024a5001c23..3e2d35ec37ca 100644 --- a/run/django/polls/test_polls.py +++ b/run/django/polls/test_polls.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from django.test import Client, TestCase # noqa: 401 +from django.test import TestCase class PollViewTests(TestCase): diff --git a/run/django/requirements-test.txt b/run/django/requirements-test.txt index 28da80f113fa..e771ac52b18f 100644 --- a/run/django/requirements-test.txt +++ b/run/django/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==6.1.1 +pytest==6.1.2 pytest-django==3.10.0 requests==2.24.0 From 3ed2a6e011daf8f8fb3a60e2fa0707bd79000af2 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 9 Nov 2020 09:40:16 +1100 Subject: [PATCH 45/91] Debug failing command --- run/django/e2e_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index edb266680bc3..4cbb960d9e08 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -119,7 +119,7 @@ def postgres_host(project_number): "--role", "roles/cloudsql.client", ], - check=True, + check=True, capture_output=True ) yield CLOUDSQL_INSTANCE From 8f854e9e7046e91e3a47fcdd1aba8bc6daf4dca5 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 9 Nov 2020 10:05:25 +1100 Subject: [PATCH 46/91] Remove IAM calls - possible permissions issues --- run/django/e2e_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 4cbb960d9e08..1215608d5816 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -108,6 +108,7 @@ def postgres_host(project_number): # TODO: Make static fixture # Allow access to Cloud SQL from Cloud Build + """ subprocess.run( [ "gcloud", @@ -121,6 +122,7 @@ def postgres_host(project_number): ], check=True, capture_output=True ) + """ yield CLOUDSQL_INSTANCE @@ -154,6 +156,7 @@ def postgres_host(project_number): # TODO: remove when binding is static # Remove policy binding + """ subprocess.run( [ "gcloud", @@ -167,6 +170,7 @@ def postgres_host(project_number): ], check=True, ) + """ @pytest.fixture From 1809ab5278c36209fe11302299d4cd2f71247427 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 9 Nov 2020 10:44:03 +1100 Subject: [PATCH 47/91] debug --- run/django/mysite/migrations/0001_createsuperuser.py | 2 +- run/django/mysite/settings.py | 1 - run/django/polls/test_polls.py | 6 +++--- run/django/requirements-test.txt | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index 1e461a6af97d..bbe7fa48b4ef 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -28,7 +28,7 @@ def createsuperuser(apps, schema_editor): # Get project value for identifying current context _, project = google.auth.default() - # Retrieve the previously stored admin passowrd + # Retrieve the previously stored admin password name = f"projects/{project}/secrets/superuser_password/versions/latest" admin_password = client.access_secret_version(name=name).payload.data.decode( "UTF-8" diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 431d7430b722..036b5120e4ce 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -97,7 +97,6 @@ }, }, ] - WSGI_APPLICATION = "mysite.wsgi.application" diff --git a/run/django/polls/test_polls.py b/run/django/polls/test_polls.py index 3e2d35ec37ca..cbb1e97262e9 100644 --- a/run/django/polls/test_polls.py +++ b/run/django/polls/test_polls.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from django.test import TestCase - +from django.test import Client, TestCase +client = Client() class PollViewTests(TestCase): def test_index_view(self): - response = self.client.get("/") + response = client.get("/") assert response.status_code == 200 assert "Hello, world" in str(response.content) diff --git a/run/django/requirements-test.txt b/run/django/requirements-test.txt index e771ac52b18f..cae45e2c3280 100644 --- a/run/django/requirements-test.txt +++ b/run/django/requirements-test.txt @@ -1,3 +1,3 @@ pytest==6.1.2 -pytest-django==3.10.0 +pytest-django==4.1.0 requests==2.24.0 From cefb39a67ac3b6afc152b757e00ed330bbbab2c0 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 9 Nov 2020 12:57:15 +1100 Subject: [PATCH 48/91] force local settings if in nox/test --- .../mysite/migrations/0001_createsuperuser.py | 25 +++++++++++-------- run/django/mysite/settings.py | 24 ++++++++++-------- run/django/polls/test_polls.py | 1 + 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index bbe7fa48b4ef..8a3313654a86 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys + from django.db import migrations import google.auth @@ -23,16 +25,19 @@ def createsuperuser(apps, schema_editor): Dynamically create an admin user as part of a migration Password is pulled from Secret Manger (previously created as part of tutorial) """ - client = secretmanager_v1.SecretManagerServiceClient() - - # Get project value for identifying current context - _, project = google.auth.default() - - # Retrieve the previously stored admin password - name = f"projects/{project}/secrets/superuser_password/versions/latest" - admin_password = client.access_secret_version(name=name).payload.data.decode( - "UTF-8" - ) + if 'test' in sys.argv or 'test_coverage' in sys.argv or 'nox' in sys.argv: + admin_password = "test" + else: + client = secretmanager_v1.SecretManagerServiceClient() + + # Get project value for identifying current context + _, project = google.auth.default() + + # Retrieve the previously stored admin password + name = f"projects/{project}/secrets/superuser_password/versions/latest" + admin_password = client.access_secret_version(name=name).payload.data.decode( + "UTF-8" + ) # Create a new user using acquired password from django.contrib.auth.models import User diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 036b5120e4ce..ffcae8c580f2 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -24,6 +24,7 @@ https://docs.djangoproject.com/en/3.0/ref/settings/ """ import os +import sys # [START run_secretconfig] import environ @@ -33,20 +34,23 @@ # If no .env has been provided, pull it from Secret Manager, storing it locally if not os.path.isfile(".env"): - import google.auth - from google.cloud import secretmanager_v1 + if 'test' in sys.argv or 'test_coverage' in sys.argv or 'nox' in sys.argv: + payload = "SECRET_KEY=a\nDATABASE_URL=sqlite:////sqlite.db\nGS_BUCKET_NAME=none" + else: + import google.auth + from google.cloud import secretmanager_v1 - _, project = google.auth.default() + _, project = google.auth.default() - if project: - client = secretmanager_v1.SecretManagerServiceClient() + if project: + client = secretmanager_v1.SecretManagerServiceClient() - SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") - name = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" - payload = client.access_secret_version(name=name).payload.data.decode("UTF-8") + SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") + name = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" + payload = client.access_secret_version(name=name).payload.data.decode("UTF-8") - with open(env_file, "w") as f: - f.write(payload) + with open(env_file, "w") as f: + f.write(payload) env = environ.Env() env.read_env(env_file) diff --git a/run/django/polls/test_polls.py b/run/django/polls/test_polls.py index cbb1e97262e9..0a1435fc1d6d 100644 --- a/run/django/polls/test_polls.py +++ b/run/django/polls/test_polls.py @@ -15,6 +15,7 @@ from django.test import Client, TestCase client = Client() + class PollViewTests(TestCase): def test_index_view(self): response = client.get("/") From 2b613100161ceef9e858c8053191e6ac89c6310b Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 13:17:16 +1100 Subject: [PATCH 49/91] add IDs to cloud build --- run/django/cloudmigrate.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml index ba1596c45a92..dec9877e6703 100644 --- a/run/django/cloudmigrate.yaml +++ b/run/django/cloudmigrate.yaml @@ -13,16 +13,16 @@ # limitations under the License. steps: - # Build the image - - name: "gcr.io/cloud-builders/docker" + - id: "build image" + name: "gcr.io/cloud-builders/docker" args: ["build", "-t", "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}", "."] - # Push the image to the Container Registry - - name: "gcr.io/cloud-builders/docker" + - id: "push image" + name: "gcr.io/cloud-builders/docker" args: ["push", "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}"] - # Apply the Django database migration against the Cloud SQL database - - name: "gcr.io/google-appengine/exec-wrapper" + - id: "apply migrations" + name: "gcr.io/google-appengine/exec-wrapper" args: [ "-i", @@ -35,8 +35,8 @@ steps: "migrate", ] - # Apply the static migrations against the Cloud Storage bucket - - name: "gcr.io/google-appengine/exec-wrapper" + - id: "collect static" + name: "gcr.io/google-appengine/exec-wrapper" args: [ "-i", From cd0b8d198c00cfa0fb0fc6ed6e8fe6582e7688ae Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 13:17:59 +1100 Subject: [PATCH 50/91] Update region tags --- run/django/mysite/settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index ffcae8c580f2..f1102b5a7fab 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -26,7 +26,7 @@ import os import sys -# [START run_secretconfig] +# [START cloudrun_secretconfig] import environ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -104,7 +104,7 @@ WSGI_APPLICATION = "mysite.wsgi.application" -# [START run_dbconfig] +# [START cloudrun_dbconfig] # Use django-environ to define the connection string DATABASES = {"default": env.db()} # [END run_dbconfig] @@ -141,7 +141,7 @@ USE_TZ = True -# [START run_staticconfig] +# [START cloudrun_staticconfig] # Define static storage via django-storages[google] GS_BUCKET_NAME = env("GS_BUCKET_NAME") From 9492d6707d1887fcbf10b98201a3ba8c45909b63 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 13:23:30 +1100 Subject: [PATCH 51/91] Get instance name from envvar --- run/django/e2e_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 1215608d5816..0cbaf5aabe7c 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -27,12 +27,12 @@ SUFFIX = uuid.uuid4().hex[:10] PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +POSTGRES_INSTANCE = os.environ["POSTGRES_INSTANCE"] REGION = "us-central1" CLOUD_STORAGE_BUCKET = f"{PROJECT}-media-{SUFFIX}" CLOUDSQL_INSTANCE = f"instance-{SUFFIX}" -POSTGRES_INSTANCE = "test-instance-pg" POSTGRES_DATABASE = f"polls-{SUFFIX}" POSTGRES_USER = f"django-{SUFFIX}" POSTGRES_PASSWORD = uuid.uuid4().hex[:26] From 2601b223ff33202a5c413b3e54408bfd270c50ca Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 13:24:02 +1100 Subject: [PATCH 52/91] Make secrets dynamic, optionally overload names by envvar --- run/django/e2e_test.py | 13 +++++++------ .../mysite/migrations/0001_createsuperuser.py | 3 ++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 0cbaf5aabe7c..057a207379dd 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -40,9 +40,8 @@ ADMIN_NAME = "admin" ADMIN_PASSWORD = uuid.uuid4().hex[:26] -# These are hardcoded elsewhere in the code -SECRET_SETTINGS_NAME = "django_settings" -SECRET_ADMINPASS_NAME = "superuser_password" +SECRET_SETTINGS_NAME = f"django_settings-{SUFFIX}" +SECRET_PASSWORD_NAME = f"superuser_password-{SUFFIX}" @pytest.fixture @@ -229,6 +228,8 @@ def allow_access(name, member): DATABASE_URL=postgres://{POSTGRES_USER}:{POSTGRES_PASSWORD}@//cloudsql/{PROJECT}:{REGION}:{POSTGRES_INSTANCE}/{POSTGRES_DATABASE} GS_BUCKET_NAME={CLOUD_STORAGE_BUCKET} SECRET_KEY={secret_key} +SETTINGS_NAME={SECRET_SETTINGS_NAME} +PASSWORD_NAME={SECRET_PASSWORD_NAME} """ create_secret(SECRET_SETTINGS_NAME, settings) @@ -241,9 +242,9 @@ def allow_access(name, member): f"serviceAccount:{project_number}@cloudbuild.gserviceaccount.com", ) - create_secret(SECRET_ADMINPASS_NAME, ADMIN_PASSWORD) + create_secret(SECRET_PASSWORD_NAME, ADMIN_PASSWORD) allow_access( - SECRET_ADMINPASS_NAME, + SECRET_PASSWORD_NAME, f"serviceAccount:{project_number}@cloudbuild.gserviceaccount.com", ) @@ -251,7 +252,7 @@ def allow_access(name, member): # delete secrets subprocess.run( - ["gcloud", "secrets", "delete", SECRET_ADMINPASS_NAME, "--quiet"], check=True + ["gcloud", "secrets", "delete", SECRET_PASSWORD_NAME, "--quiet"], check=True ) subprocess.run( ["gcloud", "secrets", "delete", SECRET_SETTINGS_NAME, "--quiet"], check=True diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index 8a3313654a86..7dc9682d887a 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -34,7 +34,8 @@ def createsuperuser(apps, schema_editor): _, project = google.auth.default() # Retrieve the previously stored admin password - name = f"projects/{project}/secrets/superuser_password/versions/latest" + PASSWORD_NAME = os.environ.get("PASSWORD_NAME", "superuser_password") + name = f"projects/{project}/secrets/{PASSWORD_NAME}/versions/latest" admin_password = client.access_secret_version(name=name).payload.data.decode( "UTF-8" ) From 2ca34270750bac87400f80746d46894b3cab9889 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 13:24:27 +1100 Subject: [PATCH 53/91] Remove create/destroy of static setting --- run/django/e2e_test.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 057a207379dd..826d87a296a6 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -104,25 +104,6 @@ def postgres_host(project_number): ], check=True, ) - - # TODO: Make static fixture - # Allow access to Cloud SQL from Cloud Build - """ - subprocess.run( - [ - "gcloud", - "projects", - "add-iam-policy-binding", - PROJECT, - "--member", - f"serviceAccount:{project_number}@cloudbuild.gserviceaccount.com", - "--role", - "roles/cloudsql.client", - ], - check=True, capture_output=True - ) - """ - yield CLOUDSQL_INSTANCE subprocess.run( @@ -153,24 +134,6 @@ def postgres_host(project_number): check=True, ) - # TODO: remove when binding is static - # Remove policy binding - """ - subprocess.run( - [ - "gcloud", - "projects", - "remove-iam-policy-binding", - PROJECT, - "--member", - f"serviceAccount:{project_number}@cloudbuild.gserviceaccount.com", - "--role", - "roles/cloudsql.client", - ], - check=True, - ) - """ - @pytest.fixture def media_bucket(): From f1b79c2c11b38b8a20ec47815283153ea43d1f15 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 13:26:08 +1100 Subject: [PATCH 54/91] Consolidate envvars --- run/django/e2e_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 826d87a296a6..6584c6fc3647 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -32,7 +32,6 @@ CLOUD_STORAGE_BUCKET = f"{PROJECT}-media-{SUFFIX}" -CLOUDSQL_INSTANCE = f"instance-{SUFFIX}" POSTGRES_DATABASE = f"polls-{SUFFIX}" POSTGRES_USER = f"django-{SUFFIX}" POSTGRES_PASSWORD = uuid.uuid4().hex[:26] @@ -104,7 +103,7 @@ def postgres_host(project_number): ], check=True, ) - yield CLOUDSQL_INSTANCE + yield POSTGRES_INSTANCE subprocess.run( [ @@ -277,7 +276,7 @@ def deployed_service(container_image): "--region", REGION, "--add-cloudsql-instances", - f"{PROJECT}:{REGION}:{CLOUDSQL_INSTANCE}", + f"{PROJECT}:{REGION}:{POSTGRES_INSTANCE}", "--project", PROJECT, ], From 738a9b5fbc07144c4490299788ab74e4f0e1f971 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 13:35:46 +1100 Subject: [PATCH 55/91] fix region tags --- run/django/mysite/settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index f1102b5a7fab..035b1d001107 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -54,7 +54,7 @@ env = environ.Env() env.read_env(env_file) -# [END run_secretconfig] +# [END cloudrun_secretconfig] SECRET_KEY = env("SECRET_KEY") @@ -107,7 +107,7 @@ # [START cloudrun_dbconfig] # Use django-environ to define the connection string DATABASES = {"default": env.db()} -# [END run_dbconfig] +# [END cloudrun_dbconfig] # Password validation # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators @@ -148,4 +148,4 @@ DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" STATICFILES_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" GS_DEFAULT_ACL = "publicRead" -# [END run_staticconfig] +# [END cloudrun_staticconfig] From e62e4d39b519600edd2067944ddaa5034025050a Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 13:39:16 +1100 Subject: [PATCH 56/91] lint, import error --- .../mysite/migrations/0001_createsuperuser.py | 4 ++-- run/django/mysite/settings.py | 22 +++++++------------ 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index 7dc9682d887a..8118ac79c231 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import sys from django.db import migrations - import google.auth from google.cloud import secretmanager_v1 @@ -25,7 +25,7 @@ def createsuperuser(apps, schema_editor): Dynamically create an admin user as part of a migration Password is pulled from Secret Manger (previously created as part of tutorial) """ - if 'test' in sys.argv or 'test_coverage' in sys.argv or 'nox' in sys.argv: + if "test" in sys.argv or "test_coverage" in sys.argv or "nox" in sys.argv: admin_password = "test" else: client = secretmanager_v1.SecretManagerServiceClient() diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 035b1d001107..83a4ff9eb0c6 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -34,7 +34,7 @@ # If no .env has been provided, pull it from Secret Manager, storing it locally if not os.path.isfile(".env"): - if 'test' in sys.argv or 'test_coverage' in sys.argv or 'nox' in sys.argv: + if "test" in sys.argv or "test_coverage" in sys.argv or "nox" in sys.argv: payload = "SECRET_KEY=a\nDATABASE_URL=sqlite:////sqlite.db\nGS_BUCKET_NAME=none" else: import google.auth @@ -47,7 +47,9 @@ SETTINGS_NAME = os.environ.get("SETTINGS_NAME", "django_settings") name = f"projects/{project}/secrets/{SETTINGS_NAME}/versions/latest" - payload = client.access_secret_version(name=name).payload.data.decode("UTF-8") + payload = client.access_secret_version(name=name).payload.data.decode( + "UTF-8" + ) with open(env_file, "w") as f: f.write(payload) @@ -113,18 +115,10 @@ # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ - { - "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", - }, - { - "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", - }, - { - "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", - }, - { - "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", - }, + {"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, + {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, + {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, + {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, ] From e51cd2b922926c78ec6930d289ca079bd52085d9 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 13:51:04 +1100 Subject: [PATCH 57/91] Testing instance name is fqdn, need just name for gcloud sql calls --- run/django/e2e_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 6584c6fc3647..cb70d4f5c7cf 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -28,6 +28,8 @@ PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] POSTGRES_INSTANCE = os.environ["POSTGRES_INSTANCE"] +if ":" in POSTGRES_INSTANCE: + POSTGRES_INSTANCE = POSTGRES_INSTANCE.split(":")[-1] REGION = "us-central1" CLOUD_STORAGE_BUCKET = f"{PROJECT}-media-{SUFFIX}" From 783761176e64544955bdb821b86cddcae329fc21 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 13:51:35 +1100 Subject: [PATCH 58/91] Ensure project specified on gcloud calls --- run/django/e2e_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index cb70d4f5c7cf..5fc0567afcf0 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -116,6 +116,8 @@ def postgres_host(project_number): POSTGRES_DATABASE, "--instance", POSTGRES_INSTANCE, + "--project", + PROJECT, "--quiet", ], check=True, @@ -130,6 +132,8 @@ def postgres_host(project_number): POSTGRES_USER, "--instance", POSTGRES_INSTANCE, + "--project", + PROJECT, "--quiet", ], check=True, From 460b5ff207940b907dd8581211d1759c6bec7a89 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 14:47:55 +1100 Subject: [PATCH 59/91] set project when using gsutil --- run/django/e2e_test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 5fc0567afcf0..421de79a467f 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -70,8 +70,6 @@ def project_number(): @pytest.fixture def postgres_host(project_number): - # Presume instance already exists - # Create database subprocess.run( [ @@ -144,7 +142,7 @@ def postgres_host(project_number): def media_bucket(): # Create storage bucket subprocess.run( - ["gsutil", "mb", "-l", REGION, f"gs://{CLOUD_STORAGE_BUCKET}"], + ["gsutil", "mb", "-l", REGION, f"gs://{CLOUD_STORAGE_BUCKET}", "-p", PROJECT], check=True, ) @@ -152,9 +150,12 @@ def media_bucket(): # Delete storage bucket contents, delete bucket subprocess.run( - ["gsutil", "-m", "rm", "-r", f"gs://{CLOUD_STORAGE_BUCKET}"], check=True + ["gsutil", "-m", "rm", "-r", f"gs://{CLOUD_STORAGE_BUCKET}", "-p", PROJECT], + check=True, + ) + subprocess.run( + ["gsutil", "rb", f"gs://{CLOUD_STORAGE_BUCKET}", "-p", PROJECT], check=True ) - subprocess.run(["gsutil", "rb", f"gs://{CLOUD_STORAGE_BUCKET}"], check=True) @pytest.fixture From 6fce68c5b33107df36c0b129dfee975e493c8808 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 14:57:24 +1100 Subject: [PATCH 60/91] order of flags in gsutil matters --- run/django/e2e_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 421de79a467f..823229667822 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -142,7 +142,7 @@ def postgres_host(project_number): def media_bucket(): # Create storage bucket subprocess.run( - ["gsutil", "mb", "-l", REGION, f"gs://{CLOUD_STORAGE_BUCKET}", "-p", PROJECT], + ["gsutil", "mb", "-l", REGION, "-p", PROJECT, f"gs://{CLOUD_STORAGE_BUCKET}"], check=True, ) @@ -150,11 +150,11 @@ def media_bucket(): # Delete storage bucket contents, delete bucket subprocess.run( - ["gsutil", "-m", "rm", "-r", f"gs://{CLOUD_STORAGE_BUCKET}", "-p", PROJECT], + ["gsutil", "-m", "rm", "-r", "-p", PROJECT, f"gs://{CLOUD_STORAGE_BUCKET}"], check=True, ) subprocess.run( - ["gsutil", "rb", f"gs://{CLOUD_STORAGE_BUCKET}", "-p", PROJECT], check=True + ["gsutil", "rb", "-p", PROJECT, f"gs://{CLOUD_STORAGE_BUCKET}"], check=True ) From 2b09698fccfded18c29fcb3822f6cdcd3628e858 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 15:10:44 +1100 Subject: [PATCH 61/91] ensure all subprocesses include project causes issues otherwise --- run/django/e2e_test.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 823229667822..f10b68deaf21 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -187,6 +187,8 @@ def allow_access(name, member): member, "--role", "roles/secretmanager.secretAccessor", + "--project", + PROJECT, ], check=True, ) @@ -221,10 +223,28 @@ def allow_access(name, member): # delete secrets subprocess.run( - ["gcloud", "secrets", "delete", SECRET_PASSWORD_NAME, "--quiet"], check=True + [ + "gcloud", + "secrets", + "delete", + SECRET_PASSWORD_NAME, + "--project", + PROJECT, + "--quiet", + ], + check=True, ) subprocess.run( - ["gcloud", "secrets", "delete", SECRET_SETTINGS_NAME, "--quiet"], check=True + [ + "gcloud", + "secrets", + "delete", + SECRET_SETTINGS_NAME, + "--project", + PROJECT, + "--quiet", + ], + check=True, ) From cb5ea1e2678432ac8db48a962bb2f276da3b9cb2 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 15:19:59 +1100 Subject: [PATCH 62/91] Add and remove more project tags --- run/django/e2e_test.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index f10b68deaf21..fb6da5db28b4 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -148,14 +148,11 @@ def media_bucket(): yield CLOUD_STORAGE_BUCKET - # Delete storage bucket contents, delete bucket + # Recursively delete assets and bucket (does not take a -p flag, apparently) subprocess.run( - ["gsutil", "-m", "rm", "-r", "-p", PROJECT, f"gs://{CLOUD_STORAGE_BUCKET}"], + ["gsutil", "-m", "rm", "-r", f"gs://{CLOUD_STORAGE_BUCKET}"], check=True, ) - subprocess.run( - ["gsutil", "rb", "-p", PROJECT, f"gs://{CLOUD_STORAGE_BUCKET}"], check=True - ) @pytest.fixture @@ -357,7 +354,7 @@ def service_url_auth_token(deployed_service): ) auth_token = ( subprocess.run( - ["gcloud", "auth", "print-identity-token"], + ["gcloud", "auth", "print-identity-token", "--project", PROJECT], stdout=subprocess.PIPE, check=True, ) From 9877d70fc940c72e4066a2cbeb05526e6aefae4e Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 15:37:32 +1100 Subject: [PATCH 63/91] Update comments --- run/django/e2e_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index fb6da5db28b4..f8eab73cdb92 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -27,10 +27,12 @@ SUFFIX = uuid.uuid4().hex[:10] PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +REGION = "us-central1" POSTGRES_INSTANCE = os.environ["POSTGRES_INSTANCE"] + +# Most commands in this test require the short instance form if ":" in POSTGRES_INSTANCE: POSTGRES_INSTANCE = POSTGRES_INSTANCE.split(":")[-1] -REGION = "us-central1" CLOUD_STORAGE_BUCKET = f"{PROJECT}-media-{SUFFIX}" @@ -171,7 +173,6 @@ def create_secret(name, value): client.add_secret_version( request={"parent": secret.name, "payload": {"data": value.encode("UTF-8")}} ) - print(f"DEBUG: {name}\n{value}") def allow_access(name, member): subprocess.run( From 04c0900c143198d186dec86ce9b93a22a69f3203 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 15:37:40 +1100 Subject: [PATCH 64/91] match on exact project name, not substring --- run/django/e2e_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index f8eab73cdb92..67b746cd914e 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -56,7 +56,7 @@ def project_number(): "projects", "list", "--filter", - PROJECT, + f"name={PROJECT}", "--format", "value(projectNumber)", ], @@ -66,7 +66,7 @@ def project_number(): .stdout.strip() .decode() ) - + print("DEBUG:", projectnum) yield projectnum From 895f8fb190338aff7c5fdd0175c5eae10d3e6ba7 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 15:38:56 +1100 Subject: [PATCH 65/91] lint --- run/django/e2e_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 67b746cd914e..24cd105b11b9 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -152,8 +152,7 @@ def media_bucket(): # Recursively delete assets and bucket (does not take a -p flag, apparently) subprocess.run( - ["gsutil", "-m", "rm", "-r", f"gs://{CLOUD_STORAGE_BUCKET}"], - check=True, + ["gsutil", "-m", "rm", "-r", f"gs://{CLOUD_STORAGE_BUCKET}"], check=True, ) From 05aa42cf79b197e60b1b10033e24edf0552992f6 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 16:03:06 +1100 Subject: [PATCH 66/91] =?UTF-8?q?Can't=20hide=20the=20settings=20name=20in?= =?UTF-8?q?=20the=20setting=20itself=20=F0=9F=A7=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- run/django/cloudmigrate.yaml | 7 ++++++- run/django/e2e_test.py | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml index dec9877e6703..16816f2b90a4 100644 --- a/run/django/cloudmigrate.yaml +++ b/run/django/cloudmigrate.yaml @@ -21,7 +21,7 @@ steps: name: "gcr.io/cloud-builders/docker" args: ["push", "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}"] - - id: "apply migrations" + - id: "apply migrations" name: "gcr.io/google-appengine/exec-wrapper" args: [ @@ -29,6 +29,8 @@ steps: "gcr.io/$PROJECT_ID/${_SERVICE_NAME}", "-s", "${PROJECT_ID}:${_REGION}:${_INSTANCE_NAME}", + "-e", + "SETTINGS_NAME=${_SETTINGS_NAME}", "--", "python", "manage.py", @@ -43,6 +45,8 @@ steps: "gcr.io/$PROJECT_ID/${_SERVICE_NAME}", "-s", "${PROJECT_ID}:${_REGION}:${_INSTANCE_NAME}", + "-e", + "SETTINGS_NAME=${_SECRET_SETTINGS_NAME}", "--", "python", "manage.py", @@ -54,6 +58,7 @@ substitutions: _INSTANCE_NAME: django-instance _REGION: us-central1 _SERVICE_NAME: polls-service + _SECRET_SETTINGS_NAME: django-settings images: - "gcr.io/${PROJECT_ID}/${_SERVICE_NAME}" diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 24cd105b11b9..60f40ba3a5db 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -196,7 +196,6 @@ def allow_access(name, member): DATABASE_URL=postgres://{POSTGRES_USER}:{POSTGRES_PASSWORD}@//cloudsql/{PROJECT}:{REGION}:{POSTGRES_INSTANCE}/{POSTGRES_DATABASE} GS_BUCKET_NAME={CLOUD_STORAGE_BUCKET} SECRET_KEY={secret_key} -SETTINGS_NAME={SECRET_SETTINGS_NAME} PASSWORD_NAME={SECRET_PASSWORD_NAME} """ @@ -259,7 +258,12 @@ def container_image(postgres_host, media_bucket, secrets): "--config", cloudbuild_config, "--substitutions", - f"_INSTANCE_NAME={postgres_host},_REGION={REGION},_SERVICE_NAME={service_name}", + ( + f"_INSTANCE_NAME={postgres_host}," + f"_REGION={REGION}," + f"_SERVICE_NAME={service_name}," + f"_SECRET_SETTINGS_NAME={SECRET_SETTINGS_NAME}" + ), "--project", PROJECT, ], From 1db86b59dcf3f3840a94c0117d376c433f19febc Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 16:13:45 +1100 Subject: [PATCH 67/91] big oof --- run/django/cloudmigrate.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/django/cloudmigrate.yaml b/run/django/cloudmigrate.yaml index 16816f2b90a4..bef3f789a329 100644 --- a/run/django/cloudmigrate.yaml +++ b/run/django/cloudmigrate.yaml @@ -30,7 +30,7 @@ steps: "-s", "${PROJECT_ID}:${_REGION}:${_INSTANCE_NAME}", "-e", - "SETTINGS_NAME=${_SETTINGS_NAME}", + "SETTINGS_NAME=${_SECRET_SETTINGS_NAME}", "--", "python", "manage.py", From d8a557f1b3c0247178a26c65e9ac80eabe8cc384 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 16:27:28 +1100 Subject: [PATCH 68/91] migrations require __init__.py to be detected --- run/django/mysite/migrations/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 run/django/mysite/migrations/__init__.py diff --git a/run/django/mysite/migrations/__init__.py b/run/django/mysite/migrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 From 87ebb84b72b8a8b934c25a6d16c6692cc52be123 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 11 Nov 2020 16:27:49 +1100 Subject: [PATCH 69/91] Bump overnight patch --- run/django/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/django/requirements.txt b/run/django/requirements.txt index b6960e3a5144..4cb5f30bf24c 100644 --- a/run/django/requirements.txt +++ b/run/django/requirements.txt @@ -1,4 +1,4 @@ -Django==3.1.2 +Django==3.1.3 django-storages[google]==1.10.1 django-environ==0.4.5 psycopg2-binary==2.8.4 From adc6fda6405e1ef3fd37b11719bbcedd516e64b4 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 12 Nov 2020 10:24:47 +1100 Subject: [PATCH 70/91] Debug 503 --- run/django/e2e_test.py | 1 + run/django/mysite/settings.py | 1 + 2 files changed, 2 insertions(+) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 60f40ba3a5db..35d98837a5d2 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -398,6 +398,7 @@ def test_end_to_end(service_url_auth_token): body = response.text # Check Django admin landing page + print(body) assert response.status_code == 200 assert "Site administration" in body assert "Polls" in body diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 83a4ff9eb0c6..e1e20ea35211 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -128,6 +128,7 @@ LANGUAGE_CODE = "en-us" TIME_ZONE = "UTC" +DEBUG = True USE_I18N = True From 4a956fbd711e085a17764b6d34fae2f1d7ce1272 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 12 Nov 2020 11:12:29 +1100 Subject: [PATCH 71/91] Service requires custom settings envvar --- run/django/e2e_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 35d98837a5d2..0d04a13785e2 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -305,6 +305,8 @@ def deployed_service(container_image): REGION, "--add-cloudsql-instances", f"{PROJECT}:{REGION}:{POSTGRES_INSTANCE}", + "--set-env-var", + f"_SETTINGS_NAME={SECRET_SETTINGS_NAME}", "--project", PROJECT, ], From 9a48ab4caac17c03b9293d9bbf55f4b9f93dfa9a Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 12 Nov 2020 14:13:25 +1100 Subject: [PATCH 72/91] attempt fixing local tests --- appengine/flexible/django_cloudsql/mysite/settings.py | 2 +- run/django/mysite/migrations/0001_createsuperuser.py | 4 ++-- run/django/mysite/settings.py | 7 ++++--- run/django/noxfile_config.py | 11 +++++++---- run/django/polls/test_polls.py | 5 ++--- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/appengine/flexible/django_cloudsql/mysite/settings.py b/appengine/flexible/django_cloudsql/mysite/settings.py index c124f3cc1879..40602f568ce6 100644 --- a/appengine/flexible/django_cloudsql/mysite/settings.py +++ b/appengine/flexible/django_cloudsql/mysite/settings.py @@ -148,4 +148,4 @@ STATIC_URL = '/static/' # [END staticurl] -STATIC_ROOT = 'static/' +STATIC_ROOT = 'static/' \ No newline at end of file diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index 8118ac79c231..d9ecd85d1bd3 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -25,7 +25,7 @@ def createsuperuser(apps, schema_editor): Dynamically create an admin user as part of a migration Password is pulled from Secret Manger (previously created as part of tutorial) """ - if "test" in sys.argv or "test_coverage" in sys.argv or "nox" in sys.argv: + if os.getenv('TRAMPOLINE_CI', None): admin_password = "test" else: client = secretmanager_v1.SecretManagerServiceClient() @@ -39,7 +39,7 @@ def createsuperuser(apps, schema_editor): admin_password = client.access_secret_version(name=name).payload.data.decode( "UTF-8" ) - + # Create a new user using acquired password from django.contrib.auth.models import User diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index e1e20ea35211..0188d0499506 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -26,7 +26,7 @@ import os import sys -# [START cloudrun_secretconfig] + import environ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -34,9 +34,10 @@ # If no .env has been provided, pull it from Secret Manager, storing it locally if not os.path.isfile(".env"): - if "test" in sys.argv or "test_coverage" in sys.argv or "nox" in sys.argv: - payload = "SECRET_KEY=a\nDATABASE_URL=sqlite:////sqlite.db\nGS_BUCKET_NAME=none" + if os.getenv('TRAMPOLINE_CI', None): + payload = f"SECRET_KEY=a\nGS_BUCKET_NAME=none\nDATABASE_URL=sqlite://{os.path.join(BASE_DIR, 'db.sqlite3')}" else: + # [START cloudrun_secretconfig] import google.auth from google.cloud import secretmanager_v1 diff --git a/run/django/noxfile_config.py b/run/django/noxfile_config.py index 696c09e6ce14..3932ee4ed310 100644 --- a/run/django/noxfile_config.py +++ b/run/django/noxfile_config.py @@ -14,7 +14,7 @@ # Default TEST_CONFIG_OVERRIDE for python repos. -# You can copy this file into your directory, then it will be imported from +# You can copy this file into your directory, then it will be inported from # the noxfile.py. # The source of truth: @@ -22,15 +22,18 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # We only run the cloud run tests in py38 session. "ignored_versions": ["2.7", "3.6", "3.7"], + # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - "envs": {}, + 'envs': { + 'DJANGO_SETTINGS_MODULE': 'mysite.settings' + }, } diff --git a/run/django/polls/test_polls.py b/run/django/polls/test_polls.py index 0a1435fc1d6d..4877d7f78489 100644 --- a/run/django/polls/test_polls.py +++ b/run/django/polls/test_polls.py @@ -13,11 +13,10 @@ # limitations under the License. from django.test import Client, TestCase -client = Client() class PollViewTests(TestCase): def test_index_view(self): - response = client.get("/") + response = self.client.get('/') assert response.status_code == 200 - assert "Hello, world" in str(response.content) + assert 'Hello, world' in str(response.content) From 6d544f2f346e2377b242bdef29f8f998b80d7741 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 12 Nov 2020 14:23:15 +1100 Subject: [PATCH 73/91] lint --- run/django/mysite/migrations/0001_createsuperuser.py | 5 ++--- run/django/mysite/settings.py | 2 -- run/django/polls/test_polls.py | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index d9ecd85d1bd3..31af3e8a1ea7 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -13,7 +13,6 @@ # limitations under the License. import os -import sys from django.db import migrations import google.auth @@ -25,7 +24,7 @@ def createsuperuser(apps, schema_editor): Dynamically create an admin user as part of a migration Password is pulled from Secret Manger (previously created as part of tutorial) """ - if os.getenv('TRAMPOLINE_CI', None): + if os.getenv("TRAMPOLINE_CI", None): admin_password = "test" else: client = secretmanager_v1.SecretManagerServiceClient() @@ -39,7 +38,7 @@ def createsuperuser(apps, schema_editor): admin_password = client.access_secret_version(name=name).payload.data.decode( "UTF-8" ) - + # Create a new user using acquired password from django.contrib.auth.models import User diff --git a/run/django/mysite/settings.py b/run/django/mysite/settings.py index 0188d0499506..50d01032863f 100644 --- a/run/django/mysite/settings.py +++ b/run/django/mysite/settings.py @@ -24,8 +24,6 @@ https://docs.djangoproject.com/en/3.0/ref/settings/ """ import os -import sys - import environ diff --git a/run/django/polls/test_polls.py b/run/django/polls/test_polls.py index 4877d7f78489..68b385e6df24 100644 --- a/run/django/polls/test_polls.py +++ b/run/django/polls/test_polls.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from django.test import Client, TestCase +from django.test import TestCase class PollViewTests(TestCase): From 7ffff2d494d9e2d57c2ce1e8e0e90782b2732219 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 12 Nov 2020 14:37:31 +1100 Subject: [PATCH 74/91] revert accidental appengine change --- appengine/flexible/django_cloudsql/mysite/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appengine/flexible/django_cloudsql/mysite/settings.py b/appengine/flexible/django_cloudsql/mysite/settings.py index 40602f568ce6..240a6cf8e01b 100644 --- a/appengine/flexible/django_cloudsql/mysite/settings.py +++ b/appengine/flexible/django_cloudsql/mysite/settings.py @@ -148,4 +148,5 @@ STATIC_URL = '/static/' # [END staticurl] -STATIC_ROOT = 'static/' \ No newline at end of file +STATIC_ROOT = 'static/' + From d529d19a2f76989aad65a812ae95cb36f401e800 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 12 Nov 2020 14:38:05 +1100 Subject: [PATCH 75/91] envvars --- run/django/e2e_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 0d04a13785e2..2bd41dcaaa20 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -305,7 +305,7 @@ def deployed_service(container_image): REGION, "--add-cloudsql-instances", f"{PROJECT}:{REGION}:{POSTGRES_INSTANCE}", - "--set-env-var", + "--set-env-vars", f"_SETTINGS_NAME={SECRET_SETTINGS_NAME}", "--project", PROJECT, From 1edf555d04e4bed164e3bad18a08ce8afb1036c2 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 12 Nov 2020 15:04:38 +1100 Subject: [PATCH 76/91] ohno --- appengine/flexible/django_cloudsql/mysite/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/appengine/flexible/django_cloudsql/mysite/settings.py b/appengine/flexible/django_cloudsql/mysite/settings.py index 240a6cf8e01b..c124f3cc1879 100644 --- a/appengine/flexible/django_cloudsql/mysite/settings.py +++ b/appengine/flexible/django_cloudsql/mysite/settings.py @@ -149,4 +149,3 @@ # [END staticurl] STATIC_ROOT = 'static/' - From d95a6982bbabe735c3f8429fe8fe51f030e5fbf2 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 12 Nov 2020 15:20:29 +1100 Subject: [PATCH 77/91] wrong varname --- run/django/e2e_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 2bd41dcaaa20..2a6989858ef9 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -306,7 +306,7 @@ def deployed_service(container_image): "--add-cloudsql-instances", f"{PROJECT}:{REGION}:{POSTGRES_INSTANCE}", "--set-env-vars", - f"_SETTINGS_NAME={SECRET_SETTINGS_NAME}", + f"SETTINGS_NAME={SECRET_SETTINGS_NAME}", "--project", PROJECT, ], From c7034b159fb8b189f4911f556951958a75626c3b Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 18 Nov 2020 09:50:24 +1100 Subject: [PATCH 78/91] Update run/django/noxfile_config.py Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com> --- run/django/noxfile_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/run/django/noxfile_config.py b/run/django/noxfile_config.py index 3932ee4ed310..9555e1335533 100644 --- a/run/django/noxfile_config.py +++ b/run/django/noxfile_config.py @@ -23,7 +23,9 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. "ignored_versions": ["2.7", "3.6", "3.7"], - +# Old samples are opted out of enforcing Python type hints +# All new samples should feature the +'enforce_type_hints': True, # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string From 14371a4951bf39614d542da6760eb8ed0301eda9 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 18 Nov 2020 09:51:15 +1100 Subject: [PATCH 79/91] Add licence header --- run/django/mysite/asgi.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/run/django/mysite/asgi.py b/run/django/mysite/asgi.py index 5b7d1a17eec1..ed252fbc2b29 100644 --- a/run/django/mysite/asgi.py +++ b/run/django/mysite/asgi.py @@ -1,3 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ ASGI config for mysite project. From e31e69239a64e63b8cbd06f37809ffb9b5b31567 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 18 Nov 2020 10:13:58 +1100 Subject: [PATCH 80/91] Formatting --- run/django/noxfile_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run/django/noxfile_config.py b/run/django/noxfile_config.py index 9555e1335533..a2f92a214d2d 100644 --- a/run/django/noxfile_config.py +++ b/run/django/noxfile_config.py @@ -23,9 +23,9 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. "ignored_versions": ["2.7", "3.6", "3.7"], -# Old samples are opted out of enforcing Python type hints -# All new samples should feature the -'enforce_type_hints': True, + # Old samples are opted out of enforcing Python type hints + # All new samples should feature the + "enforce_type_hints": True, # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string From e4dd31fb8a990ec17bc38e48fed1d4752a146727 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 18 Nov 2020 10:14:05 +1100 Subject: [PATCH 81/91] black --- run/django/e2e_test.py | 3 ++- run/django/noxfile_config.py | 7 ++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 2a6989858ef9..97be5a369959 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -152,7 +152,8 @@ def media_bucket(): # Recursively delete assets and bucket (does not take a -p flag, apparently) subprocess.run( - ["gsutil", "-m", "rm", "-r", f"gs://{CLOUD_STORAGE_BUCKET}"], check=True, + ["gsutil", "-m", "rm", "-r", f"gs://{CLOUD_STORAGE_BUCKET}"], + check=True, ) diff --git a/run/django/noxfile_config.py b/run/django/noxfile_config.py index a2f92a214d2d..118cf9fa5c5f 100644 --- a/run/django/noxfile_config.py +++ b/run/django/noxfile_config.py @@ -30,12 +30,9 @@ # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - 'envs': { - 'DJANGO_SETTINGS_MODULE': 'mysite.settings' - }, + "envs": {"DJANGO_SETTINGS_MODULE": "mysite.settings"}, } From 8712b3f2c00b82c8956bd8ab5ee919e9bc72d5b7 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 18 Nov 2020 15:19:24 +1100 Subject: [PATCH 82/91] Add typehinting, remove debugging --- run/django/e2e_test.py | 23 +++++++++---------- run/django/manage.py | 2 +- .../mysite/migrations/0001_createsuperuser.py | 4 +++- run/django/polls/test_polls.py | 4 +++- run/django/polls/views.py | 4 ++-- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/run/django/e2e_test.py b/run/django/e2e_test.py index 97be5a369959..9a161d5a1180 100644 --- a/run/django/e2e_test.py +++ b/run/django/e2e_test.py @@ -17,6 +17,7 @@ import os import subprocess +from typing import Iterator, List, Tuple import uuid from google.cloud import secretmanager_v1 as sm @@ -48,7 +49,7 @@ @pytest.fixture -def project_number(): +def project_number() -> Iterator[str]: projectnum = ( subprocess.run( [ @@ -66,12 +67,11 @@ def project_number(): .stdout.strip() .decode() ) - print("DEBUG:", projectnum) yield projectnum @pytest.fixture -def postgres_host(project_number): +def postgres_host() -> Iterator[str]: # Create database subprocess.run( [ @@ -141,7 +141,7 @@ def postgres_host(project_number): @pytest.fixture -def media_bucket(): +def media_bucket() -> Iterator[str]: # Create storage bucket subprocess.run( ["gsutil", "mb", "-l", REGION, "-p", PROJECT, f"gs://{CLOUD_STORAGE_BUCKET}"], @@ -158,10 +158,10 @@ def media_bucket(): @pytest.fixture -def secrets(project_number): +def secrets(project_number: str) -> Iterator[str]: # Create a number of secrets and allow Google Cloud services access to them - def create_secret(name, value): + def create_secret(name: str, value: str) -> None: secret = client.create_secret( request={ "parent": f"projects/{PROJECT}", @@ -174,7 +174,7 @@ def create_secret(name, value): request={"parent": secret.name, "payload": {"data": value.encode("UTF-8")}} ) - def allow_access(name, member): + def allow_access(name: str, member: str) -> None: subprocess.run( [ "gcloud", @@ -246,7 +246,7 @@ def allow_access(name, member): @pytest.fixture -def container_image(postgres_host, media_bucket, secrets): +def container_image(postgres_host: str, media_bucket: str, secrets: str) -> Iterator[str]: # Build container image for Cloud Run deployment image_name = f"gcr.io/{PROJECT}/polls-{SUFFIX}" service_name = f"polls-{SUFFIX}" @@ -289,7 +289,7 @@ def container_image(postgres_host, media_bucket, secrets): @pytest.fixture -def deployed_service(container_image): +def deployed_service(container_image: str) -> Iterator[str]: # Deploy image to Cloud Run service_name = f"polls-{SUFFIX}" subprocess.run( @@ -334,7 +334,7 @@ def deployed_service(container_image): @pytest.fixture -def service_url_auth_token(deployed_service): +def service_url_auth_token(deployed_service: str) -> Iterator[Tuple[str, str]]: # Get Cloud Run service URL and auth token service_url = ( subprocess.run( @@ -374,7 +374,7 @@ def service_url_auth_token(deployed_service): # no deletion needed -def test_end_to_end(service_url_auth_token): +def test_end_to_end(service_url_auth_token: List[str]) -> None: service_url, auth_token = service_url_auth_token headers = {"Authorization": f"Bearer {auth_token}"} login_slug = "/admin/login/?next=/admin/" @@ -401,7 +401,6 @@ def test_end_to_end(service_url_auth_token): body = response.text # Check Django admin landing page - print(body) assert response.status_code == 200 assert "Site administration" in body assert "Polls" in body diff --git a/run/django/manage.py b/run/django/manage.py index 868c06851602..627ae5a9f14f 100755 --- a/run/django/manage.py +++ b/run/django/manage.py @@ -19,7 +19,7 @@ import sys -def main(): +def main() -> None: os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") try: from django.core.management import execute_from_command_line diff --git a/run/django/mysite/migrations/0001_createsuperuser.py b/run/django/mysite/migrations/0001_createsuperuser.py index 31af3e8a1ea7..e9b2c473513b 100644 --- a/run/django/mysite/migrations/0001_createsuperuser.py +++ b/run/django/mysite/migrations/0001_createsuperuser.py @@ -15,11 +15,13 @@ import os from django.db import migrations +from django.db.backends.postgresql.schema import DatabaseSchemaEditor +from django.db.migrations.state import StateApps import google.auth from google.cloud import secretmanager_v1 -def createsuperuser(apps, schema_editor): +def createsuperuser(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: """ Dynamically create an admin user as part of a migration Password is pulled from Secret Manger (previously created as part of tutorial) diff --git a/run/django/polls/test_polls.py b/run/django/polls/test_polls.py index 68b385e6df24..e02f4a34f44b 100644 --- a/run/django/polls/test_polls.py +++ b/run/django/polls/test_polls.py @@ -12,11 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + from django.test import TestCase class PollViewTests(TestCase): - def test_index_view(self): + def test_index_view(self: PollViewTests) -> None: response = self.client.get('/') assert response.status_code == 200 assert 'Hello, world' in str(response.content) diff --git a/run/django/polls/views.py b/run/django/polls/views.py index bda93457d1e5..66a49f990cb3 100644 --- a/run/django/polls/views.py +++ b/run/django/polls/views.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from django.http import HttpResponse +from django.http import HttpRequest, HttpResponse -def index(request): +def index(request: HttpRequest) -> HttpResponse: return HttpResponse("Hello, world. You're at the polls index.") From 652d5eea1ddc95532157e842fdb0df55ecb41a5d Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 23 Nov 2020 15:11:26 +1100 Subject: [PATCH 83/91] =?UTF-8?q?=E2=9A=A0=EF=B8=8F=20=20noxfile=20change?= =?UTF-8?q?=20-=20add=20typehints?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- noxfile-template.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/noxfile-template.py b/noxfile-template.py index e7ad325db370..706c214d6ebf 100644 --- a/noxfile-template.py +++ b/noxfile-template.py @@ -15,8 +15,9 @@ from __future__ import print_function import os -from pathlib import Path import sys +from pathlib import Path +from typing import Callable, Dict, List, Optional import nox @@ -68,7 +69,7 @@ TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) -def get_pytest_env_vars(): +def get_pytest_env_vars() -> Dict[str]: """Returns a dict for pytest invocation.""" ret = {} @@ -98,7 +99,7 @@ def get_pytest_env_vars(): # -def _determine_local_import_names(start_dir): +def _determine_local_import_names(start_dir: str) -> List[str]: """Determines all import names that should be considered "local". This is used when running the linter to insure that import order is @@ -136,7 +137,7 @@ def _determine_local_import_names(start_dir): @nox.session -def lint(session): +def lint(session: nox.Session) -> None: if not TEST_CONFIG['enforce_type_hints']: session.install("flake8", "flake8-import-order") else: @@ -156,7 +157,7 @@ def lint(session): # @nox.session -def blacken(session): +def blacken(session: nox.Session) -> None: session.install("black") python_files = [path for path in os.listdir(".") if path.endswith(".py")] @@ -171,7 +172,7 @@ def blacken(session): PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] -def _session_tests(session, post_install=None): +def _session_tests(session: nox.Session, post_install: Callable = None) -> None: """Runs py.test for a particular project.""" if os.path.exists("requirements.txt"): session.install("-r", "requirements.txt") @@ -197,7 +198,7 @@ def _session_tests(session, post_install=None): @nox.session(python=ALL_VERSIONS) -def py(session): +def py(session: nox.Session) -> None: """Runs py.test for a sample using the specified version of Python.""" if session.python in TESTED_VERSIONS: _session_tests(session) @@ -212,9 +213,10 @@ def py(session): # -def _get_repo_root(): +def _get_repo_root() -> Optional[str]: """ Returns the root folder of the project. """ - # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + # Get root of this repository. + # Assume we don't have directories nested deeper than 10 items. p = Path(os.getcwd()) for i in range(10): if p is None: @@ -230,7 +232,7 @@ def _get_repo_root(): @nox.session @nox.parametrize("path", GENERATED_READMES) -def readmegen(session, path): +def readmegen(session: nox.Session, path: str) -> None: """(Re-)generates the readme for a sample.""" session.install("jinja2", "pyyaml") dir_ = os.path.dirname(path) From 08d29617a5282c46127c45681014964053c23aef Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 23 Nov 2020 15:21:12 +1100 Subject: [PATCH 84/91] =?UTF-8?q?=E2=9A=A0=EF=B8=8F=20noxfile=20change=20-?= =?UTF-8?q?=20fix=20method=20signature?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- noxfile-template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile-template.py b/noxfile-template.py index 706c214d6ebf..a7bbfcad1a10 100644 --- a/noxfile-template.py +++ b/noxfile-template.py @@ -69,7 +69,7 @@ TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) -def get_pytest_env_vars() -> Dict[str]: +def get_pytest_env_vars() -> Dict[str, str]: """Returns a dict for pytest invocation.""" ret = {} From e441b59b005fe1b0f8c1fbc838f1ab5a76b51cad Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 24 Nov 2020 10:54:42 +1100 Subject: [PATCH 85/91] =?UTF-8?q?=E2=9A=A0=EF=B8=8F=20noxfile=20change=20-?= =?UTF-8?q?=20fix=20nox=20session=20type?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- noxfile-template.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/noxfile-template.py b/noxfile-template.py index a7bbfcad1a10..2b644f4f4cf7 100644 --- a/noxfile-template.py +++ b/noxfile-template.py @@ -137,7 +137,7 @@ def _determine_local_import_names(start_dir: str) -> List[str]: @nox.session -def lint(session: nox.Session) -> None: +def lint(session: nox.sessions.Session) -> None: if not TEST_CONFIG['enforce_type_hints']: session.install("flake8", "flake8-import-order") else: @@ -157,7 +157,7 @@ def lint(session: nox.Session) -> None: # @nox.session -def blacken(session: nox.Session) -> None: +def blacken(session: nox.sessions.Session) -> None: session.install("black") python_files = [path for path in os.listdir(".") if path.endswith(".py")] @@ -172,7 +172,7 @@ def blacken(session: nox.Session) -> None: PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] -def _session_tests(session: nox.Session, post_install: Callable = None) -> None: +def _session_tests(session: nox.sessions.Session, post_install: Callable = None) -> None: """Runs py.test for a particular project.""" if os.path.exists("requirements.txt"): session.install("-r", "requirements.txt") @@ -198,7 +198,7 @@ def _session_tests(session: nox.Session, post_install: Callable = None) -> None: @nox.session(python=ALL_VERSIONS) -def py(session: nox.Session) -> None: +def py(session: nox.sessions.Session) -> None: """Runs py.test for a sample using the specified version of Python.""" if session.python in TESTED_VERSIONS: _session_tests(session) @@ -232,7 +232,7 @@ def _get_repo_root() -> Optional[str]: @nox.session @nox.parametrize("path", GENERATED_READMES) -def readmegen(session: nox.Session, path: str) -> None: +def readmegen(session: nox.sessions.Session, path: str) -> None: """(Re-)generates the readme for a sample.""" session.install("jinja2", "pyyaml") dir_ = os.path.dirname(path) From 763ad5b68dc5d2b9b92ae3251680ff47d9d4278a Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 24 Nov 2020 11:10:58 +1100 Subject: [PATCH 86/91] =?UTF-8?q?=E2=9A=A0=EF=B8=8F=20noxfile=20change=20-?= =?UTF-8?q?=20import=20order?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- noxfile-template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile-template.py b/noxfile-template.py index 2b644f4f4cf7..9c28d5a0050e 100644 --- a/noxfile-template.py +++ b/noxfile-template.py @@ -15,8 +15,8 @@ from __future__ import print_function import os -import sys from pathlib import Path +import sys from typing import Callable, Dict, List, Optional import nox From 82df36ff8b5730257835aa60ead47746a77b55a0 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 24 Nov 2020 11:25:54 +1100 Subject: [PATCH 87/91] hadolint Dockerfile --- run/django/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run/django/Dockerfile b/run/django/Dockerfile index 986f2e3a34c1..670f871c1dd7 100644 --- a/run/django/Dockerfile +++ b/run/django/Dockerfile @@ -24,7 +24,7 @@ ENV PYTHONUNBUFFERED 1 # Install dependencies COPY requirements.txt . -RUN pip install -r requirements.txt +RUN pip install --no-cache-dir -r requirements.txt # Copy local code to the container image. COPY . . @@ -33,4 +33,4 @@ COPY . . # webserver, with one worker process and 8 threads. # For environments with multiple CPU cores, increase the number of workers # to be equal to the cores available. -CMD exec gunicorn --bind 0.0.0.0:$PORT --workers 1 --threads 8 --timeout 0 mysite.wsgi:application +CMD ["exec", "gunicorn", "--bind", "0.0.0.0:$PORT", "--workers", "1", "--threads", "8", "--timeout", "0", "mysite.wsgi:application"] From 5395697ff69aa69a6e7b3eb310c4b2613775940d Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 24 Nov 2020 13:17:28 +1100 Subject: [PATCH 88/91] revert hadolint --- run/django/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run/django/Dockerfile b/run/django/Dockerfile index 670f871c1dd7..f85c7118c3a6 100644 --- a/run/django/Dockerfile +++ b/run/django/Dockerfile @@ -24,7 +24,7 @@ ENV PYTHONUNBUFFERED 1 # Install dependencies COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir -r requirements.txt # Copy local code to the container image. COPY . . @@ -33,4 +33,4 @@ COPY . . # webserver, with one worker process and 8 threads. # For environments with multiple CPU cores, increase the number of workers # to be equal to the cores available. -CMD ["exec", "gunicorn", "--bind", "0.0.0.0:$PORT", "--workers", "1", "--threads", "8", "--timeout", "0", "mysite.wsgi:application"] +CMD exec gunicorn --bind 0.0.0.0:$PORT --workers 1 --threads 8 --timeout 0 mysite.wsgi:application From a18d3feef7c08db842adc41ce19e6b3ddbbbe410 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 3 Dec 2020 08:49:47 +1100 Subject: [PATCH 89/91] Add linking --- run/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/run/README.md b/run/README.md index fbb41d21ca53..4b6bb5223f72 100644 --- a/run/README.md +++ b/run/README.md @@ -110,6 +110,7 @@ for more information. [pubsub]: pubsub/ [mysql]: ../cloud-sql/mysql/sqlalchemy [postgres]: ../cloud-sql/postgres/sqlalchemy +[django]: django/ [run_button_helloworld]: https://deploy.cloud.run/?git_repo=https://github.com/knative/docs&dir=docs/serving/samples/hello-world/helloworld-python [run_button_pubsub]: https://deploy.cloud.run/?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&dir=run/pubsub [testing]: https://cloud.google.com/run/docs/testing/local#running_locally_using_docker_with_access_to_services From 5e78fdbf38073500b32efe9b5fe84d96eeb7be07 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 3 Dec 2020 08:49:54 +1100 Subject: [PATCH 90/91] Remove gcloudignore --- run/django/.gcloudignore | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 run/django/.gcloudignore diff --git a/run/django/.gcloudignore b/run/django/.gcloudignore deleted file mode 100644 index c0abe06014ba..000000000000 --- a/run/django/.gcloudignore +++ /dev/null @@ -1,20 +0,0 @@ -# This file specifies files that are *not* uploaded to Google Cloud Platform -# using gcloud. It follows the same syntax as .gitignore, with the addition of -# "#!include" directives (which insert the entries of the given .gitignore-style -# file at that point). -# -# For more information, run: -# $ gcloud topic gcloudignore -# -.gcloudignore -# If you would like to upload your .git directory, .gitignore file or files -# from your .gitignore file, remove the corresponding line -# below: -.git -.gitignore - -# Python pycache: -__pycache__/ - -venv/ -.env From 60699a94dd50e167bb27c967fe2d91b7ce10229e Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 3 Dec 2020 09:46:57 +1100 Subject: [PATCH 91/91] Testing: i have a theory this is why tests started failing --- run/django/.gcloudignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 run/django/.gcloudignore diff --git a/run/django/.gcloudignore b/run/django/.gcloudignore new file mode 100644 index 000000000000..4c49bd78f1d0 --- /dev/null +++ b/run/django/.gcloudignore @@ -0,0 +1 @@ +.env