Add the CI tests and watch it burn

.github/actions/build-docker/action.yml

@@ -10,9 +10,6 @@ inputs:
   version:
     required: true
     description: The image version to tag with
-  target:
-    required: true
-    description: The stage to target in the build
   push:
     required: false
     description: Push the image?
@@ -57,7 +54,8 @@ runs:
     - name: Create .env and version.json files
       shell: bash
       run: |
-        echo "DOCKER_TARGET=${{ inputs.target }}" >> $GITHUB_ENV
+        # We can only build the production image in CI
+        echo "DOCKER_TARGET=production" >> $GITHUB_ENV
         echo "DOCKER_VERSION=${{ steps.meta.outputs.version }}" >> $GITHUB_ENV
         echo "DOCKER_COMMIT=${{ steps.context.outputs.git_sha }}" >> $GITHUB_ENV
         echo "DOCKER_BUILD=${{ steps.context.outputs.git_build_url }}" >> $GITHUB_ENV

.github/actions/run-docker/action.yml

@@ -9,53 +9,56 @@ inputs:
     description: 'The build digest of the image to run. Overrides version.'
     required: true
     default: ''
+  target:
+    description: 'Docker target to run (development|production)'
+    required: false
+    default: 'development'
   run:
     description: 'Run command in container'
     required: true
-  compose_file:
-    description: 'The docker-compose file to use'
-    required: false
-    default: 'docker-compose.yml:docker-compose.ci.yml'
   logs:
     description: 'Show logs'
     required: false
-  data_backup_skip:
-    description: 'Skip data backup'
+  initialize:
+    description: 'Run the initialize script'
     required: false
-    default: 'true'
+    default: 'false'
   olympia_deps:
     description: 'Which dependencies to install at runtime? (development|production)'
     required: false
     default: 'development'
+  mount:
+    description: 'Name of the mount for the host directory (development|production)'
+    required: false
+    default: 'production'
 
 runs:
   using: 'composite'
   steps:
-    - id: id
-      shell: bash
-      run: |
-        echo "id=$(id -u)" >> $GITHUB_OUTPUT
-
     - name: Run Docker Container
+      id: run
+      continue-on-error: true
       shell: bash
       env:
         DOCKER_VERSION: ${{ inputs.version }}
         DOCKER_DIGEST: ${{ inputs.digest }}
-        COMPOSE_FILE: ${{ inputs.compose_file }}
-        HOST_UID: ${{ steps.id.outputs.id }}
-        DATA_BACKUP_SKIP: ${{ inputs.data_backup_skip }}
+        OLYMPIA_MOUNT_INPUT: ${{ inputs.mount }}
+        DATA_BACKUP_SKIP: ${{ inputs.initialize == 'true' && '' || 'true' }}
         OLYMPIA_DEPS: ${{ inputs.olympia_deps }}
+        DOCKER_TARGET: ${{ inputs.target }}
       run: |
         # Start the specified services
         make up
 
         # Exec the run command in the container
         # quoted 'EOF' to prevent variable expansion
-        cat <<'EOF' | docker compose exec --user olympia web sh
+        cat <<'EOF' | docker compose exec --user olympia web bash
           ${{ inputs.run }}
         EOF
 
     - name: Logs
       shell: bash
-      if: ${{ inputs.logs }}
-      run: docker compose logs
+      if: ${{ steps.run.outcome == 'failure' }}
+      run: |
+        docker compose logs
+        exit 1

.github/workflows/_test.yml

@@ -41,29 +41,24 @@ jobs:
           -
             name: Needs Locale Compilation
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: |
               make compile_locales
               make test_needs_locales_compilation
           -
             name: Static Assets
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_static_assets
           -
             name: Internal Routes
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_internal_routes_allowed
           -
             name: Elastic Search
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_es_tests
           -
             name: Codestyle
             services: web
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make lint-codestyle
     steps:
       - uses: actions/checkout@v4
@@ -73,5 +68,4 @@ jobs:
           version: ${{ inputs.version }}
           digest: ${{ inputs.digest }}
           services: ${{ matrix.services }}
-          compose_file: ${{ matrix.compose_file }}
           run: ${{ matrix.run }}

.github/workflows/_test_check.yml

@@ -45,17 +45,21 @@ jobs:
     runs-on: ubuntu-latest
     name: |
       version: '${{ matrix.version }}' |
-      compose_file: '${{ matrix.compose_file }}'
+      target: '${{ matrix.target }}' |
+      mount: '${{ matrix.mount }}' |
       olympia_deps: '${{ matrix.olympia_deps }}'
     strategy:
       fail-fast: false
       matrix:
         version:
           - local
           - ${{ inputs.version }}
-        compose_file:
-          - docker-compose.yml
-          - docker-compose.yml:docker-compose.ci.yml
+        target:
+          - development
+          - production
+        mount:
+          - development
+          - production
         olympia_deps:
           - development
           - production
@@ -67,7 +71,8 @@ jobs:
           cat <<EOF
             Values passed to the action:
             version: ${{ matrix.version }}
-            compose_file: ${{ matrix.compose_file }}
+            target: ${{ matrix.target }}
+            mount: ${{ matrix.mount }}
             olympia_deps: ${{ matrix.olympia_deps }}
           EOF
       - uses: ./.github/actions/run-docker
@@ -77,7 +82,8 @@ jobs:
           DOCKER_VERSION: 'not-expected'
         with:
           version: ${{ matrix.version }}
-          compose_file: ${{ matrix.compose_file }}
+          target: ${{ matrix.target }}
+          mount: ${{ matrix.mount }}
           olympia_deps: ${{ matrix.olympia_deps }}
           run: make check
 

.github/workflows/_test_main.yml

@@ -88,7 +88,7 @@ jobs:
           services: ''
           digest: ${{ inputs.digest }}
           version: ${{ inputs.version }}
-          compose_file: docker-compose.yml
+          mount: development
           run: |
             split="--splits ${{ needs.test_config.outputs.splits }}"
             group="--group ${{ matrix.group }}"

.github/workflows/ci.yml

@@ -72,7 +72,6 @@ jobs:
           registry: ${{ steps.docker_hub.outputs.registry }}
           image: ${{ steps.docker_hub.outputs.image }}
           version: ci-${{ needs.context.outputs.docker_version }}
-          target: development
           push: true
 
   docs_build:
@@ -88,7 +87,8 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          compose_file: docker-compose.yml
+          mount: development
+          target: development
           run: |
             make docs
 
@@ -139,7 +139,7 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          compose_file: docker-compose.yml
+          mount: development
           run: make extract_locales
 
       - name: Push Locales
@@ -213,7 +213,6 @@ jobs:
           registry: ${{ steps.docker_hub.outputs.registry }}
           image: ${{ steps.docker_hub.outputs.image }}
           version: ${{ needs.context.outputs.docker_version }}
-          target: production
           push: true
 
   push_gar:
@@ -245,5 +244,4 @@ jobs:
           registry: ${{ steps.docker_gar.outputs.registry }}
           image: ${{ steps.docker_gar.outputs.image }}
           version: ${{ needs.context.outputs.docker_version }}
-          target: production
           push: true

.gitignore

@@ -53,6 +53,5 @@ tmp/*
 # End of .gitignore. Please keep this in sync with the top section of .dockerignore
 
 # do not ignore the following files
-!docker-compose.ci.yml
 !docker-compose.private.yml
 !private/README.md

Dockerfile

@@ -46,7 +46,28 @@ RUN localedef -i en_US -f UTF-8 en_US.UTF-8
 ENV LANG=en_US.UTF-8
 ENV LC_ALL=en_US.UTF-8
 
+# Build args that determine the tag and stage of the build
+# These are passed to docker via the bake.hcl file
+ARG DOCKER_COMMIT
+ARG DOCKER_VERSION
+ARG DOCKER_BUILD
+ARG DOCKER_TARGET
+
+# Expose these variables via hard coded file to prevent them from being
+# overridden by the environment when running the container. These
+# values represent the build and should not be changable afterwards.
+ENV BUILD_INFO=/build-info
 RUN <<EOF
+# Create the build file hard coding build variables to the image
+cat <<INNEREOF > ${BUILD_INFO}
+commit="${DOCKER_COMMIT}"
+version="${DOCKER_VERSION}"
+build="${DOCKER_BUILD}"
+target="${DOCKER_TARGET}"
+INNEREOF
+# Set permissions to make the file readable by all but only writable by root
+chmod 644 ${BUILD_INFO}
+
 # Create directory for dependencies
 mkdir /deps
 chown -R olympia:olympia /deps
@@ -95,6 +116,9 @@ EOF
 ARG DOCKER_TARGET
 ENV DOCKER_TARGET=${DOCKER_TARGET}
 
+# Add our custom mime types (required for for ts/json/md files)
+COPY docker/etc/mime.types /etc/mime.types
+
 # Define production dependencies as a single layer
 # let's the rest of the stages inherit prod dependencies
 # and makes copying the /deps dir to the final layer easy.
@@ -114,6 +138,8 @@ RUN \
 ${HOME}/scripts/install_deps.py prod
 EOF
 
+FROM base AS development
+
 FROM base AS locales
 ARG LOCALE_DIR=${HOME}/locale
 # Compile locales
@@ -149,32 +175,20 @@ echo "from olympia.lib.settings_base import *" > settings_local.py
 DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
 EOF
 
-FROM base AS sources
+FROM base AS production
 
 ARG DOCKER_BUILD DOCKER_COMMIT DOCKER_VERSION
 
 ENV DOCKER_BUILD=${DOCKER_BUILD}
 ENV DOCKER_COMMIT=${DOCKER_COMMIT}
 ENV DOCKER_VERSION=${DOCKER_VERSION}
 
-# Add our custom mime types (required for for ts/json/md files)
-COPY docker/etc/mime.types /etc/mime.types
 # Copy the rest of the source files from the host
 COPY --chown=olympia:olympia . ${HOME}
+# Copy compiled locales from builder
+COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy assets from assets
 COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
 COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
-
-# Set shell back to sh until we can prove we can use bash at runtime
-SHELL ["/bin/sh", "-c"]
-
-FROM sources AS development
-
-FROM sources AS production
-
-# Copy compiled locales from builder
-COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy dependencies from `pip_production`
 COPY --from=pip_production --chown=olympia:olympia /deps /deps
-
-

Makefile-docker

@@ -69,7 +69,7 @@ data_dump:
 
 .PHONY: data_load
 data_load:
-	./manage.py data_load $(ARGS)
+	./manage.py data_load --name $(LOAD)
 
 .PHONY: update_assets
 update_assets:

Makefile-os

@@ -7,43 +7,50 @@
 DOCKER_PROGRESS ?= auto
 DOCKER_METADATA_FILE ?= buildx-bake-metadata.json
 DOCKER_PUSH ?=
+# Values that are not saved to .env
+# but should be set in the docker image
+# default to static values to prevent
+# invalidating docker build cache
 export DOCKER_COMMIT ?=
 export DOCKER_BUILD ?=
 export DOCKER_VERSION ?=
 export DATA_BACKUP_SKIP ?=
 override DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld
 
 INITIALIZE_ARGS ?=
-INIT_CLEAN ?=
-INIT_LOAD ?=
-
-ifneq ($(INIT_CLEAN),)
-	INITIALIZE_ARGS += --clean
-endif
-
-ifneq ($(INIT_LOAD),)
-	INITIALIZE_ARGS += --load $(INIT_LOAD)
-endif
-
-
+export CLEAN ?=
+export LOAD ?=
 
 DOCKER_BAKE_ARGS := \
 	--file docker-bake.hcl \
 	--file .env \
 	--progress $(DOCKER_PROGRESS) \
 	--metadata-file $(DOCKER_METADATA_FILE) \
 
-ifeq ($(DOCKER_PUSH), true)
-	DOCKER_BAKE_ARGS += --push
-endif
-
 DOCKER_COMPOSE_ARGS := \
 	-d \
 	--wait \
 	--remove-orphans \
 	--no-build \
 	--quiet-pull \
-	--renew-anon-volumes
+	--timestamps
+
+ifneq ($(CLEAN),)
+	# Build without cache
+	DOCKER_COMPOSE_ARGS += --no-cache
+	# Force recreate all containers
+	DOCKER_COMPOSE_ARGS += --force-recreate
+	# Clean the database and all other data when Initializing
+	INITIALIZE_ARGS += --clean
+endif
+
+ifneq ($(LOAD),)
+	INITIALIZE_ARGS += --load $(LOAD)
+endif
+
+ifeq ($(DOCKER_PUSH), true)
+	DOCKER_BAKE_ARGS += --push
+endif
 
 # Paths should be cleaned before mounting .:/data/olympia
 # These are files which should be sourced from the container