Merge pull request #5 from aodn/features/cicd-pipeline-integration

Add github actions files

.deepsource.toml

@@ -0,0 +1,7 @@
+version = 1
+
+[[analyzers]]
+name = "python"
+
+  [analyzers.meta]
+  runtime_version = "3.x.x"

.github/environment/README.md

@@ -0,0 +1,13 @@
+## Github Deployment Environments
+Github deployment environments are used to define unique settings for each environment i.e. staging and production
+
+The build and push workflows need to know which AWS account to push updated docker images to.
+
+### DotEnv Files
+The .env files in this directory are here as a record of the "variables" and their values.
+
+The variables can be updated from these files using the following command:
+```bash
+gh variable set -R aodn/<repo name> -e <environment name> -f <environment>.env
+
+```

.github/environment/central.env

@@ -0,0 +1,4 @@
+AWS_REGION=ap-southeast-2
+AWS_ROLE_ARN=arn:aws:iam::851725428481:role/AodnGitHubActionsRole
+ECR_REGISTRY=851725428481.dkr.ecr.ap-southeast-2.amazonaws.com
+ECR_REPOSITORY=data-access-service

.github/environment/edge.env

@@ -0,0 +1,2 @@
+AWS_REGION=ap-southeast-2
+AWS_ROLE_ARN=arn:aws:iam::704910415367:role/AodnGitHubActionsRole

.github/environment/production.env

@@ -0,0 +1,2 @@
+AWS_REGION=ap-southeast-2
+AWS_ROLE_ARN=arn:aws:iam::211125304466:role/AodnGitHubActionsRole

.github/environment/staging.env

@@ -0,0 +1,2 @@
+AWS_REGION=ap-southeast-2
+AWS_ROLE_ARN=arn:aws:iam::905418367757:role/AodnGitHubActionsRole

.github/workflows/build_deploy_edge.yml

@@ -0,0 +1,54 @@
+name: Build/Deploy Edge
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '**/*.md'
+      - '.github/environment/**'
+permissions:
+  id-token: write
+  contents: read
+jobs:
+  build_push:
+    runs-on: ubuntu-latest
+    environment: central
+    outputs:
+      digest: ${{ steps.build_and_push.outputs.digest }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Configure AWS Credentials
+        id: aws_auth
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
+      - name: Login to ECR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ vars.ECR_REGISTRY }}
+      - name: Build and Push Docker Image
+        id: build_and_push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          #          Only building for AMD64 for now
+          #          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ github.sha }}
+            ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:latest
+  trigger_edge_deploy:
+    needs: [build_push]
+    uses: ./.github/workflows/trigger_deploy.yml
+    with:
+      app_name: data-access-service
+      environment: edge
+      digest: ${{ needs.build_push.outputs.digest }}
+    secrets: inherit

.github/workflows/semantic-release-ci.yml

@@ -0,0 +1,100 @@
+name: Semantic Release CI
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '**/*.md'
+      - '.github/environment/**'
+permissions:
+  contents: write
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  pre-commit-and-pytest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Set up Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: "latest"
+      - name: Install dependencies
+        run: |
+          poetry lock --no-update
+          poetry install
+      - name: Run pre-commit checks
+        run: |
+          poetry run pre-commit run --all-files
+      - name: Run tests
+        run: |
+          if [ -z "$(find . -name 'test_*.py')" ]; then echo "No tests found"; else poetry run pytest; fi
+  release:
+    runs-on: ubuntu-latest
+    needs: pre-commit-and-pytest
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads/main')
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+      - name: Install dependencies
+        run: npm install semantic-release @semantic-release/exec @semantic-release/commit-analyzer @semantic-release/release-notes-generator @semantic-release/github
+      - name: Run semantic-release and extract changelog
+        id: release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Generate changelog with Markdown formatting by running semantic-release in dry-run mode
+          npx semantic-release --dry-run | tee semantic_output.log
+          # Capture the changelog markdown from semantic-release output for the latest version
+          changelog=$(awk '/Release note for version/ {flag=1; next} flag && !/^#[^#]/' semantic_output.log)
+          # Insert "v" in front of version numbers in `##` headers
+          changelog=$(echo "$changelog" | sed -E 's/^## ([0-9]+\.[0-9]+\.[0-9]+)/## v\1/')
+          # Trim leading spaces before each bullet point (lines starting with `* `)
+          changelog=$(echo "$changelog" | sed -E 's/^[[:space:]]+\*/\*/')
+          # Write the cleaned changelog to a file
+          echo "$changelog" > changelog.md
+          # Set the changelog as an output variable for use in the GitHub release step
+          echo "changelog<<EOF" >> $GITHUB_ENV
+          echo "$changelog" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      - name: Push new tag
+        if: ${{ steps.release.outputs.version }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          version="${{ steps.release.outputs.version }}"
+          # Create and push the new tag
+          git tag "$version"
+          git push origin "$version"
+      - name: Create GitHub Release with Changelog
+        if: ${{ steps.release.outputs.version }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          version="${{ steps.release.outputs.version }}"
+          tag_name="$version"
+
+          # Read changelog from the markdown file to preserve formatting
+          release_notes=$(<changelog.md)
+
+          # Create GitHub release with formatted markdown changelog
+          gh release create "$tag_name" \
+            --title "$version" \
+            --notes "$release_notes" \
+            --target main \
+            --draft

.github/workflows/trigger_deploy.yml

@@ -0,0 +1,80 @@
+name: Trigger Deploy
+on:
+  workflow_dispatch:
+    inputs:
+      app_name:
+        required: true
+        description: The short-name of the app corresponding to the folder in appdeploy.
+        type: string
+      environment:
+        required: true
+        description: The environment to use for the deploy job.
+        type: choice
+        options:
+          - edge
+          - staging
+          - production
+      digest:
+        required: false
+        description: The image digest to pass to the deploy job.
+        type: string
+  workflow_call:
+    inputs:
+      app_name:
+        required: true
+        type: string
+      digest:
+        required: false
+        type: string
+      environment:
+        required: true
+        type: string
+permissions:
+  id-token: write
+  contents: read
+jobs:
+  trigger_deploy:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
+      - name: Push Image Digest to SSM
+        if: ${{ inputs.digest != '' }}
+        run: |
+          aws ssm put-parameter \
+            --name "/apps/${{ inputs.app_name }}/${{ inputs.environment }}/image_digest" \
+            --type "String" \
+            --value "$digest" \
+            --overwrite
+        env:
+          digest: ${{ inputs.digest }}
+      - name: Generate App Token
+        uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.DEPLOY_APP_ID }}
+          private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: "appdeploy"
+      - name: Trigger Deploy Workflow
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ steps.app-token.outputs.token }}
+          retries: 3
+          retry-exempt-status-codes: 204
+          script: |
+            github.rest.actions.createWorkflowDispatch({
+              owner: 'aodn',
+              repo: 'appdeploy',
+              workflow_id: 'deploy.yml',
+              ref: 'main',
+              inputs: {
+                app_name: '${{ inputs.app_name }}',
+                environment: '${{ inputs.environment }}'
+              }
+            })

.gitignore

@@ -77,3 +77,4 @@ dmypy.json
 cython_debug/
 .idea
 *.iml
+node_modules/

.releaserc

@@ -0,0 +1,18 @@
+{
+    "branches": [
+        "main"
+    ],
+    "plugins": [
+        "@semantic-release/commit-analyzer",
+        "@semantic-release/release-notes-generator",
+        "@semantic-release/changelog",
+        "@semantic-release/github",
+        [
+            "@semantic-release/exec", {
+                "verifyReleaseCmd": "echo \"version=v${nextRelease.version}\" >> $GITHUB_OUTPUT"
+            }
+        ]
+    ],
+    "preset": "angular",
+    "tagFormat": "v${version}"
+}

Dockerfile

@@ -1,10 +1,8 @@
-# Use the official Python base image
 FROM python:3.10-slim
 
-# Set the working directory in the container
 WORKDIR /app
+RUN useradd -l -m -s /bin/bash appuser
 
-# Copy the pyproject.toml and poetry.lock files into the container
 COPY pyproject.toml poetry.lock ./
 
 # For Docker build to understand the possible env
@@ -16,8 +14,10 @@ RUN apt update && \
     poetry lock && \
     poetry install
 
-# Expose the port the app runs on
-EXPOSE 8000
 
-# Copy the rest of the application code into the container
 COPY . /app
+
+RUN chown -R appuser:appuser /app
+USER appuser
+
+EXPOSE 8000

README.md

@@ -85,6 +85,18 @@ The configurations for pre-commit hooks are defined in `.pre-commit-config.yaml`
 pre-commit run --all-files
 ```
 
+### Versioning
+
+This project uses **semantic versioning** with automated releases managed by `semantic-release`.
+
+Every code change with commits following [Conventional Commits](https://github.com/conventional-changelog/commitlint/tree/master/%40commitlint/config-conventional) will trigger a version update and create a GitHub release.
+
+**Commit Guidelines**
+
+- `feat:` For new features
+- `fix:` For bug fixes
+- `BREAKING CHANGE:` For any breaking changes
+
 ## Environment variables
 
 In the root directory of the project, create a `.env` file.
@@ -108,6 +120,6 @@ PROFILE=edge
 
 | Description        | Endpoints                          | Param                                                                                          | Environment                                                                   |
 |--------------------|----------------------------------------|------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------|
-| Formatted metadata | /api/v1/das/metadata/{uuid}  | none                                                                                           | ALL                                                                           | 
-| Raw metadata       | /api/v1/das/metadata/{uuid}/raw | none                                                                                           | ALL                                                                      | 
+| Formatted metadata | /api/v1/das/metadata/{uuid}  | none                                                                                           | ALL                                                                           |
+| Raw metadata       | /api/v1/das/metadata/{uuid}/raw | none                                                                                           | ALL                                                                      |
 | Subsetting         | /api/v1/das/data/{uuid} | start_date=2023-12-25T14:30:00 end_date=2024-02-25T14:30:00 start_depth=-0.06 f=netcdf or json | ALL |

data_access_service/core/api.py

@@ -94,7 +94,8 @@ def has_data(self, uuid: str, start_date: datetime, end_date: datetime):
                     start_date, period_end, None, None, None, None, None
                 ).empty:
                     return True
-                else: start_date = period_end + timedelta(days=1)
+                else:
+                    start_date = period_end + timedelta(days=1)
         return False
 
     def get_dataset_data(