Merge branch 'main' into BAI-1463-update-reaction-hover-show-friendly…

…-dn-instead-full-dn
gchq · Dec 16, 2024 · 89c02a6 · 89c02a6
2 parents 4a2e023 + 4762935
commit 89c02a6
Show file tree

Hide file tree

Showing 89 changed files with 2,771 additions and 1,730 deletions.
diff --git a/.github/workflows/build.js.yml b/.github/workflows/build.js.yml
@@ -63,6 +63,34 @@ jobs:
           name: bailo_frontend
           path: /tmp/bailo_frontend.tar
 
+  build_modelscan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      # Clone repository
+      - uses: actions/checkout@v3
+
+      # Setup BuildX
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build modelscan image
+        uses: docker/build-push-action@v3
+        with:
+          context: lib/modelscan_api
+          file: lib/modelscan_api/Dockerfile
+          tags: bailo_modelscan:latest
+          cache-from: type=gha,scope=modelscan
+          cache-to: type=gha,mode=max,scope=modelscan
+          outputs: type=docker,dest=/tmp/bailo_modelscan.tar
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: bailo_modelscan
+          path: /tmp/bailo_modelscan.tar
+
   unit_testing:
     runs-on: ubuntu-latest
 
@@ -107,7 +135,7 @@ jobs:
       - run: npm run lint
 
   kubernetes:
-    needs: [build_backend, build_frontend]
+    needs: [build_backend, build_frontend, build_modelscan]
     runs-on: ubuntu-latest
 
     steps:
@@ -134,6 +162,11 @@ jobs:
         with:
           name: bailo_frontend
           path: /tmp
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: bailo_modelscan
+          path: /tmp
 
       - name: Start minikube
         uses: medyagh/setup-minikube@master
@@ -146,6 +179,7 @@ jobs:
           eval $(minikube -p minikube docker-env)
           docker load -i /tmp/bailo_backend.tar
           docker load -i /tmp/bailo_frontend.tar
+          docker load -i /tmp/bailo_modelscan.tar
 
       # Install dependencies
       - run: npm ci
@@ -193,6 +227,8 @@ jobs:
             frontendTag: latest
             backendRepository: "bailo_backend"
             backendTag: latest
+            modelscanRepository: "bailo_modelscan"
+            modelscanTag: latest
           # Used for k8s not openshift
           ingress:
             enabled: false
@@ -259,7 +295,7 @@ jobs:
         run: kubectl logs -l app.kubernetes.io/instance=bailo --tail=-1
 
   end_to_end:
-    needs: [build_backend, build_frontend]
+    needs: [build_backend, build_frontend, build_modelscan]
     runs-on: ubuntu-latest
 
     steps:
@@ -287,14 +323,20 @@ jobs:
         with:
           name: bailo_frontend
           path: /tmp
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: bailo_modelscan
+          path: /tmp
 
       - name: Load image
         run: |
           docker load --input /tmp/bailo_frontend.tar
           docker load --input /tmp/bailo_backend.tar
+          docker load --input /tmp/bailo_modelscan.tar
           docker image ls -a
 
-          rm -rf /tmp/bailo_frontend.tar /tmp/bailo_backend.tar
+          rm -rf /tmp/bailo_frontend.tar /tmp/bailo_backend.tar /tmp/bailo_modelscan.tar
 
       # Create logs directory
       - run: mkdir logs

diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml
@@ -108,7 +108,7 @@ jobs:
         env:
           BASE_PATH: "/Bailo"
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v1
+        uses: actions/upload-pages-artifact@v3
         with:
           path: ./lib/landing/out
 
@@ -128,4 +128,4 @@ jobs:
     steps:
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v4
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,19 +1,37 @@
 stages:
+  - git_sync
   - os_build
   - aws_build
 
 variables:
   AWS_TAG: latest
   OS_TAG: $CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA
 
-os-backend-build:
-  stage: os_build
+opensrc-bailols:
+  stage: git_sync
   rules:
-    - if: '$CI_COMMIT_BRANCH == "main"'
+    - if: '$CI_COMMIT_BRANCH == "main" && $BFG == "true"'
       when: always
       allow_failure: true
     - when: manual
       allow_failure: true
+  image:
+    name: openjdk:latest
+  script:
+    - microdnf install -y git wget
+    - git clone --mirror "https://$OPENSRC_TOKEN_NAME:$OPENSRC_TOKEN@$OPENSRC_URL"
+    - wget https://repo1.maven.org/maven2/com/madgag/bfg/1.14.0/bfg-1.14.0.jar
+    - java -jar bfg-1.14.0.jar --strip-blobs-bigger-than 40M $OPENSRC_GIT_NAME
+    - cd $OPENSRC_GIT_NAME
+    - git reflog expire --expire=now --all && git gc --prune=now --aggressive
+    - git remote add $REPO_NAME "https://$LS_TOKEN_NAME:$LS_TOKEN@$LS_URL"
+    - git push -u $REPO_NAME main --force
+
+os-backend-build:
+  stage: os_build
+  rules:
+    - when: manual
+      allow_failure: true
   image:
     name: docker:latest
   variables:
@@ -35,9 +53,6 @@ os-backend-build:
 os-frontend-build:
   stage: os_build
   rules:
-    - if: '$CI_COMMIT_BRANCH == "main"'
-      when: always
-      allow_failure: true
     - when: manual
       allow_failure: true
   image:
@@ -58,13 +73,33 @@ os-frontend-build:
     - docker tag frontend-ci:$OS_TAG $REGISTRY_URL/$OPENSHIFT_PROJECTNAME/frontend-ci:$OS_TAG
     - docker push $REGISTRY_URL/$OPENSHIFT_PROJECTNAME/frontend-ci:$OS_TAG
 
-openshift-deploy:
+os-modelscan-build:
   stage: os_build
-  needs: ['os-frontend-build', 'os-backend-build']
   rules:
-    - if: '$CI_COMMIT_BRANCH == "main"'
-      when: always
+    - when: manual
       allow_failure: true
+  image:
+    name: docker:latest
+  variables:
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_TLS_CERTDIR: ''
+  services:
+    - name: docker:stable-dind
+      command:
+        - '--tls=false'
+        - '--host=tcp://0.0.0.0:2375'
+  before_script:
+    - docker login -u $OPENSHIFT_USER -p $OPENSHIFT_TOKEN $REGISTRY_URL
+    - cd lib/modelscan_api
+  script:
+    - DOCKER_BUILDKIT=1 docker build -t modelscan-ci:$OS_TAG .
+    - docker tag modelscan-ci:$OS_TAG $REGISTRY_URL/$OPENSHIFT_PROJECTNAME/modelscan-ci:$OS_TAG
+    - docker push $REGISTRY_URL/$OPENSHIFT_PROJECTNAME/modelscan-ci:$OS_TAG
+
+openshift-deploy:
+  stage: os_build
+  needs: ['os-frontend-build', 'os-backend-build', 'os-modelscan-build']
+  rules:
     - when: manual
       allow_failure: true
   image:
@@ -99,6 +134,8 @@ openshift-deploy:
         frontendTag: $OS_TAG
         backendRepository: "$BACKEND_REPO"
         backendTag: $OS_TAG
+        modelscanRepository: "$BACKEND_REPO"
+        modelscanTag: $OS_TAG
       # Used for openshift
       route:
         enabled: true
@@ -123,9 +160,6 @@ openshift-deploy:
 aws-backend-build:
   stage: aws_build
   rules:
-    - if: '$CI_COMMIT_BRANCH == "main"'
-      when: always
-      allow_failure: true
     - when: manual
       allow_failure: true
   image:
@@ -155,9 +189,6 @@ aws-backend-build:
 aws-frontend-build:
   stage: aws_build
   rules:
-    - if: '$CI_COMMIT_BRANCH == "main"'
-      when: always
-      allow_failure: true
     - when: manual
       allow_failure: true
   image:
@@ -183,3 +214,32 @@ aws-frontend-build:
     - docker tag $DOCKER_REGISTRY/$APP_NAME_FRONTEND:$AWS_TAG $DOCKER_REGISTRY/$APP_NAME_FRONTEND:$OS_TAG
     - docker push $DOCKER_REGISTRY/$APP_NAME_FRONTEND:$AWS_TAG
     - docker push $DOCKER_REGISTRY/$APP_NAME_FRONTEND:$OS_TAG
+
+aws-modelscan-build:
+  stage: aws_build
+  rules:
+    - when: manual
+      allow_failure: true
+  image:
+    name: docker:latest
+  variables:
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_TLS_CERTDIR: ''
+  services:
+    - name: docker:stable-dind
+      command:
+        - '--tls=false'
+        - '--host=tcp://0.0.0.0:2375'
+  before_script:
+    - apk add --no-cache curl jq python3 py3-pip
+    - pip install awscli --break-system-packages
+    - aws ecr get-login-password | docker login --username AWS --password-stdin $DOCKER_REGISTRY
+    - aws --version
+    - docker info
+    - docker --version
+    - cd lib/modelscan_api
+  script:
+    - DOCKER_BUILDKIT=1 docker build -t $DOCKER_REGISTRY/$APP_NAME_MODELSCAN:$AWS_TAG .
+    - docker tag $DOCKER_REGISTRY/$APP_NAME_MODELSCAN:$AWS_TAG $DOCKER_REGISTRY/$APP_NAME_MODELSCAN:$OS_TAG
+    - docker push $DOCKER_REGISTRY/$APP_NAME_MODELSCAN:$AWS_TAG
+    - docker push $DOCKER_REGISTRY/$APP_NAME_MODELSCAN:$OS_TAG
diff --git a/README.md b/README.md
@@ -134,11 +134,12 @@ something more secure.
 We expect the administrator to provide their own forms of authentication. By default all users authenticate using as
 'user'.
 
-You can test out your new deployment using the example models which can be found in `__tests__`
-[`minimal_binary.zip`](__tests__/example_models/minimal_binary.zip) and
-[`minimal_code.zip`](__tests__/example_models/minimal_code.zip). There are also example forms in the `scripts` folder
-[`minimal_upload_schema_examples.json`](backend/src/scripts/example_schemas/minimal_upload_schema_examples.json) and
-[`minimal_deployment_schema_examples.json`](backend/src/scripts/example_schemas/minimal_deployment_schema_examples.json).
+You can test out your new deployment using the example models which can be found in `frontend/cypress/fixtures`
+[`minimal_binary.zip`](frontend/cypress/fixtures/minimal_binary.zip) and
+[`minimal_code.zip`](frontend/cypress/fixtures/minimal_code.zip). There are also example forms in the `scripts` folder
+[`minimal_model_schema.json`](backend/src/scripts/example_schemas/minimal_model_schema.json),
+[`minimal_data_card_schema.json`](backend/src/scripts/example_schemas/minimal_data_card_schema.json) and
+[`minimal_access_request_schema.json`](backend/src/scripts/example_schemas/minimal_access_request_schema.json).
 
 <br />
 

diff --git a/backend/Dockerfile b/backend/Dockerfile
@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1
-FROM python:3.13.0-bullseye AS sphinx-docs
+FROM python:3.13.1-bullseye AS sphinx-docs
 
 RUN apt update && apt install -y pandoc
 

diff --git a/backend/Dockerfile.dev b/backend/Dockerfile.dev
@@ -1,4 +1,4 @@
-FROM python:3.13.0-bullseye AS sphinx-docs
+FROM python:3.13.1-bullseye AS sphinx-docs
 
 RUN apt update && apt install -y pandoc
 

diff --git a/backend/config/default.cjs b/backend/config/default.cjs
@@ -222,6 +222,9 @@ module.exports = {
 
     fileScanners: {
       kinds: [],
+      retryDelayInMinutes: 60,
+      maxInitRetries: 5,
+      initRetryDelay: 5000,
     },
   },
 

diff --git a/backend/config/docker_compose.cjs b/backend/config/docker_compose.cjs
@@ -74,6 +74,9 @@ module.exports = {
   connectors: {
     fileScanners: {
       kinds: ['clamAV', 'modelScan'],
+      retryDelayInMinutes: 60,
+      maxInitRetries: 5,
+      initRetryDelay: 5000,
     },
   },
 }