Merge pull request #6919 from opencrvs/optimise_docker_builds

Optimise Docker builds

.dockerignore

@@ -5,4 +5,4 @@
 **/*.log
 .secrets/*
 data
-grafana
+grafana

.github/workflows/build-images-from-branch.yml

@@ -32,70 +32,70 @@ jobs:
       - uses: actions/checkout@v4
         if: github.event_name == 'push'
 
-      - name: Set version
-        id: set-version
+      - name: Set version and branch
+        id: set-version-and-branch
         run: |
           export VERSION=`git log -1 --pretty=format:%h`
           echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
 
       - name: Get list of services
         id: get-services
         run: |
-          services=$(grep "^  [^ ]" docker-compose.yml | grep -v '#' | awk -F: '{print $1}' | sed -e 's/^ *//')
+          services=$(grep "^  [^ ]" docker-compose.yml | grep -v base| grep -v '#' | awk -F: '{print $1}' | sed -e 's/^ *//')
           services_json=$(echo $services | tr '\n' ',' | sed 's/,$//' | jq -R 'split(" ")' | tr -d '\n')
           # Set the list of service names as an output variable
           echo "services=$services_json" >> $GITHUB_OUTPUT
           echo "services=$services_json"
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push base image
+        uses: docker/build-push-action@v5
+        with:
+          file: packages/Dockerfile.base
+          context: .
+          push: true
+          tags: 'opencrvs/ocrvs-base:${{ steps.set-version-and-branch.outputs.branch }}'
+          cache-from: type=registry,ref=opencrvs/ocrvs-base:${{ steps.set-version-and-branch.outputs.branch }}
+          cache-to: type=inline
 
     outputs:
       services: ${{ steps.get-services.outputs.services }}
-      version: ${{ steps.set-version.outputs.version }}
+      version: ${{ steps.set-version-and-branch.outputs.version }}
+      branch: ${{ steps.set-version-and-branch.outputs.branch }}
 
   build:
     needs: base
     strategy:
+      fail-fast: false
       matrix:
         service: ${{ fromJSON(needs.base.outputs.services) }}
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
-        if: github.event_name == 'workflow_dispatch'
-        with:
-          ref: '${{ github.event.inputs.branch_name }}'
-
-      - uses: actions/checkout@v4
-        if: github.event_name == 'push'
-
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Build ${{ matrix.service }}
-        run: |
-          export VERSION="${{ needs.base.outputs.version }}"
-          echo ${{ matrix.service }}
-          docker compose build ${{ matrix.service }}
-          docker compose push ${{ matrix.service }}
-
-  security-scans:
-    needs: [base, build]
-    strategy:
-      fail-fast: false
-      matrix:
-        service: ${{ fromJSON(needs.base.outputs.services) }}
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
+      - name: Build and push
+        uses: docker/build-push-action@v5
         with:
-          sparse-checkout: |
-            trivy.yaml
-            .trivyignore.yaml
-          sparse-checkout-cone-mode: false
+          file: packages/${{ matrix.service }}/Dockerfile
+          build-args: |
+            VERSION=${{ needs.base.outputs.branch }}
+          push: true
+          context: .
+          tags: |
+            opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version }}
+            opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.branch }}
+          cache-from: type=registry,ref=opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.branch }}
+          cache-to: type=inline
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
+        uses: aquasecurity/trivy-action@0.22.0
         with:
           image-ref: 'opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version }}'
           trivy-config: trivy.yaml

.github/workflows/lint-and-test.yml

@@ -130,7 +130,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Run Trivy vulnerability scanner in fs mode
-        uses: aquasecurity/trivy-action@master
+        uses: aquasecurity/trivy-action@0.22.0
         with:
           scan-type: 'fs'
           scan-ref: '.'

.github/workflows/publish-release.yml

@@ -102,7 +102,7 @@ jobs:
             .trivyignore.yaml
           sparse-checkout-cone-mode: false
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
+        uses: aquasecurity/trivy-action@0.22.0
         with:
           image-ref: 'opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version }}'
           trivy-config: trivy.yaml

docker-compose.ci.yml

@@ -6,7 +6,6 @@
 # & Healthcare Disclaimer located at http://opencrvs.org/license.
 #
 # Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
-version: '3.3'
 
 services:
   client:

docker-compose.demo-deps.yml

@@ -6,7 +6,6 @@
 # & Healthcare Disclaimer located at http://opencrvs.org/license.
 #
 # Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
-version: '3.3'
 
 services:
   # For dependencies, expose ports locally for dev

docker-compose.deps.yml

@@ -6,7 +6,6 @@
 # & Healthcare Disclaimer located at http://opencrvs.org/license.
 #
 # Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
-version: '3.3'
 volumes:
   metricbeat:
 services:

docker-compose.dev-deps.yml

@@ -6,7 +6,6 @@
 # & Healthcare Disclaimer located at http://opencrvs.org/license.
 #
 # Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
-version: '3.3'
 
 services:
   # For dependencies, expose ports locally for dev

docker-compose.dev.yml

@@ -6,7 +6,6 @@
 # & Healthcare Disclaimer located at http://opencrvs.org/license.
 #
 # Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
-version: '3.3'
 
 services:
   # Expose dev secrets as a plain volume - these will use docker secrets in staging and prod

docker-compose.yml

@@ -6,41 +6,63 @@
 # & Healthcare Disclaimer located at http://opencrvs.org/license.
 #
 # Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
-version: '3.3'
 
 services:
+  base:
+    image: opencrvs/ocrvs-base:${VERSION}
+    #platform: linux/amd64
+    build:
+      context: .
+      dockerfile: ./packages/Dockerfile.base
+    deploy:
+      replicas: 0
   client:
     image: opencrvs/ocrvs-client:${VERSION}
+    #platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/client/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
 
   dashboards:
     image: opencrvs/ocrvs-dashboards:${VERSION}
+    #platform: linux/amd64
     build:
-      context: ./packages/dashboards
-      dockerfile: ./Dockerfile
+      context: .
+      dockerfile: ./packages/dashboards/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
 
   components:
     image: opencrvs/ocrvs-components:${VERSION}
+    #platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/components/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
   login:
     image: opencrvs/ocrvs-login:${VERSION}
+    #platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/login/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
   gateway:
     image: opencrvs/ocrvs-gateway:${VERSION}
+    #platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/gateway/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - REDIS_HOST=redis
       - CONFIG_SMS_CODE_EXPIRY_SECONDS=600
@@ -62,10 +84,13 @@ services:
   # User facing services
   workflow:
     image: opencrvs/ocrvs-workflow:${VERSION}
+    #platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/workflow/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - NODE_ENV=development
       - SEARCH_URL=http://search:9090/
@@ -80,10 +105,13 @@ services:
       - COUNTRY=${COUNTRY:-bgd} # PEN_TEST change to gbr
   search:
     image: opencrvs/ocrvs-search:${VERSION}
+    #platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/search/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - HEARTH_MONGO_URL=mongodb://mongo1/hearth-dev
       - NODE_ENV=development
@@ -93,10 +121,13 @@ services:
       - APPLICATION_CONFIG_URL=http://config:2021/
   metrics:
     image: opencrvs/ocrvs-metrics:${VERSION}
+    #platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/metrics/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - INFLUX_HOST=influxdb
       - INFLUX_PORT=8086
@@ -113,19 +144,25 @@ services:
   # END User facing services
   scheduler:
     image: opencrvs/ocrvs-scheduler:${VERSION}
+    #platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/scheduler/Dockerfile
+    depends_on:
+      - base
     environment:
       - NODE_ENV=development
       - METRICS_URL=http://metrics:1050
       - OPENHIM_MONGO_URL=mongodb://mongo1/openhim-dev
   auth:
     image: opencrvs/ocrvs-auth:${VERSION}
+    #platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/auth/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - REDIS_HOST=redis
       - USER_MANAGEMENT_URL=http://user-mgnt:3030/
@@ -135,10 +172,13 @@ services:
       - METRICS_URL=http://metrics:1050
   user-mgnt:
     image: opencrvs/ocrvs-user-mgnt:${VERSION}
+    #platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/user-mgnt/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - MONGO_URL=mongodb://mongo1/user-mgnt
       - NOTIFICATION_SERVICE_URL=http://notification:2020/
@@ -148,10 +188,13 @@ services:
       - APPLICATION_CONFIG_URL=http://config:2021/
   webhooks:
     image: opencrvs/ocrvs-webhooks:${VERSION}
+    #platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/webhooks/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - REDIS_HOST=redis://redis:6379
       - MONGO_URL=mongodb://mongo1/webhooks
@@ -162,20 +205,26 @@ services:
       - FHIR_URL=http://hearth:3447/fhir
   notification:
     image: opencrvs/ocrvs-notification:${VERSION}
+    #platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/notification/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - COUNTRY=${COUNTRY:-bgd}
       - HOST=0.0.0.0
       - COUNTRY_CONFIG_URL=http://countryconfig:3040
   config:
     image: opencrvs/ocrvs-config:${VERSION}
+    #platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/config/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - AUTH_URL=http://auth:4040
       - COUNTRY_CONFIG_URL=http://countryconfig:3040
@@ -191,10 +240,13 @@ services:
       - CHECK_INVALID_TOKEN=true
   migration:
     image: opencrvs/ocrvs-migration:${VERSION}
+    #platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/migration/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - USER_MGNT_MONGO_URL=mongodb://mongo1/user-mgnt
       - APPLICATION_CONFIG_MONGO_URL=mongodb://mongo1/application-config
@@ -208,10 +260,13 @@ services:
       - WAIT_HOSTS=mongo1:27017,influxdb:8086,minio:9000,elasticsearch:9200
   documents:
     image: opencrvs/ocrvs-documents:${VERSION}
+    #platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/documents/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - NODE_ENV=development
       - COUNTRY_CONFIG_URL=http://countryconfig:3040

lerna.json

@@ -1,4 +1,6 @@
 {
-  "version": "1.5.0",
+  "lerna": "2.11.0",
+  "packages": ["packages/*"],
+  "version": "0.0.0",
   "npmClient": "yarn"
 }