Merge pull request #455 from raft-tech/devops/2457-ci-path-filtering

Devops/2457 ci path filtering

.circleci/README.md

@@ -14,13 +14,13 @@ This script will generate a complete config for building, testing, and deploying
 ### Directory structure
 
 #### build-and-test
-Contains workflows, jobs, and commands for building and testing the application.
+Contains workflows, jobs, and commands for building and testing the application. These are now triggered by GitHub Actions that serve as a filter so only the code that's changed is tested. See [build-all](../.github/workflows/build-all.yml), [build-backend](../.github/workflows/build-backend.yml), and [build-frontend](../.github/workflows/build-frontend.yml)
 
 #### infrastructure
-Contains workflows, jobs, and commands for setting up the infrastructure on Cloud gov.
+Contains workflows, jobs, and commands for setting up the infrastructure on Cloud gov. This is now triggered by GitHub Actions that serve as a filter so only runs when infrastructure code is changed. See [deploy-infrastructure](../.github/workflows/deploy-infrastructure.yml)
 
 #### deployment
-Contains workflows, jobs, and commands for deploying the application on Cloud gov.
+Contains workflows, jobs, and commands for deploying the application on Cloud gov. Note: merges to develop now automatically trigger a develop deploy using [deploy-develop-on-merge](../.github/workflows/deploy-develop-on-merge.yml) and deploys to dev environments happen when a label is created on the PR using [deploy-on-label](../.github/workflows/deploy-on-label.yml)
 
 #### owasp
 Contains workflows, jobs, and commands for running OWASP scans on the application in during the pipeline against Cloud.gov environments.

.circleci/base_config.yml

@@ -21,6 +21,18 @@ executors:
     resource_class: large
 
 parameters:
+  build_and_test_all:
+    type: boolean
+    default: false
+  build_and_test_backend:
+    type: boolean
+    default: false
+  build_and_test_frontend:
+    type: boolean
+    default: false
+  deploy_infrastructure:
+    type: boolean
+    default: false
   develop_branch_deploy:
     type: boolean
     default: false
@@ -36,3 +48,6 @@ parameters:
   target_env:
     type: string
     default: ''
+  util_make_erd:
+    type: boolean
+    default: false

.circleci/build-and-test/workflows.yml

@@ -1,19 +1,41 @@
 # workflows:
-  build-and-test:
-    unless:
+  build-and-test-all:
+    when:
       or:
-        - << pipeline.parameters.run_dev_deployment >>
-        - << pipeline.parameters.develop_branch_deploy >>
-        - << pipeline.parameters.run_owasp_scan >>
-        - << pipeline.parameters.run_nightly_owasp_scan >>
+        - << pipeline.parameters.build_and_test_all >>
+        - equal: [ main, << pipeline.git.branch >> ]
+        - equal: [ master, << pipeline.git.branch >> ]
+        - equal: [ 'hhs-dev-devops', << pipeline.git.branch >> ]
     jobs:
       - secrets-check
+      - test-backend:
+          requires:
+            - secrets-check      
       - test-frontend:
           requires:
             - secrets-check
+      - test-e2e:
+          requires:
+            - secrets-check
+
+  build-and-test-backend:
+    when: << pipeline.parameters.build_and_test_backend >>
+    jobs:
+      - secrets-check
       - test-backend:
           requires:
             - secrets-check
       - test-e2e:
           requires:
             - secrets-check
+
+  build-and-test-frontend:
+    when: << pipeline.parameters.build_and_test_frontend >>
+    jobs:
+      - secrets-check
+      - test-frontend:
+          requires:
+            - secrets-check
+      - test-e2e:
+          requires:
+            - secrets-check

.circleci/config.yml

@@ -13,6 +13,18 @@ orbs:
 
 # parameters from github actions
 parameters:
+  build_and_test_all:
+    type: boolean
+    default: false
+  build_and_test_backend:
+    type: boolean
+    default: false
+  build_and_test_frontend:
+    type: boolean
+    default: false
+  deploy_infrastructure:
+    type: boolean
+    default: false
   develop_branch_deploy:
     type: boolean
     default: false
@@ -28,6 +40,9 @@ parameters:
   target_env:
     type: string
     default: ''
+  util_make_erd:
+    type: boolean
+    default: false
 
 jobs:
   setup:

.circleci/infrastructure/workflows.yml

@@ -1,7 +1,10 @@
 #workflows:
-
   enable-versioning-for-s3-buckets:
-    unless: << pipeline.parameters.run_nightly_owasp_scan >>
+    when: 
+      or:
+        - << pipeline.parameters.deploy_infrastructure >>
+        - equal: [ main, << pipeline.git.branch >> ]
+        - equal: [ master, << pipeline.git.branch >> ]
     jobs:
       - enable-versioning:
           filters:

.circleci/util/workflows.yml

@@ -1,9 +1,8 @@
 # workflows:
   erd:
-    unless: << pipeline.parameters.run_nightly_owasp_scan >>
+    when:
+      or:
+        - equal: [ develop, << pipeline.git.branch >> ]
+        - equal: [ master, << pipeline.git.branch >> ]
     jobs:
-      - make_erd:
-          filters:
-            branches:
-              only:
-                develop
+      - make_erd

.github/workflows/build-all.yml

@@ -0,0 +1,67 @@
+###########################################################################
+# GitHub Action Workflow
+# On pull request or changes to scripts/commands or changes to the pipeline
+# code to any branch besides develop, staging and master triggers the full 
+# build and test pipeline.
+# 
+# NOTE: develop, staging(main) and master are skipped on the push because this
+#       would be redundant after running the full set of tests from the PR.
+#       See deploy-develop-on-merge.yml and make_erd for the workflow
+#       pipelines that run on merge to develop, staging, and master branches.
+#
+# Step 0: make PR from your branch into develop, or make changes on your 
+#         branch to non-documentation files in scripts or commands.sh and 
+#         push changes to your remote branch.
+#
+# Step 1: Makes a request to the V2 CircleCI API to initiate the project,
+#         which will filter based upon build_and_test_backend and 
+#         build_and_test_frontend to run the workflow/jobs listed here:
+#           build-and-test:[
+#             test-backend,
+#             test-frontend,
+#             test-e2e
+#           ]
+#
+#         Leverages the open source GitHub Action:
+#           https://github.com/promiseofcake/circleci-trigger-action
+###########################################################################
+name: Build and test All for PRs and when scripts/commands change
+on:
+  push:
+    branches_ignore:
+      - develop
+      - main
+      - master
+    paths:
+      - 'scripts/**'
+      - 'commands.sh'
+      - '.circleci/**'
+      - '.github/**'
+    paths_ignore:
+      - '**.md'
+  pull_request:
+    branches_ignore: #handled in circleci
+      - main
+      - master
+    types: [review_requested, ready_for_review]
+    paths_ignore:
+      - 'docs/**'
+      - '**.md'
+      - '**.txt'
+      - '.gitattributes'
+      - '.gitignore'
+      - 'LICENSE'
+jobs:
+  build_and_test_all:
+    runs-on: ubuntu-latest
+    name: Initiate deploy job in CircleCI
+    steps:
+      - uses: actions/checkout@v2
+      - name: Circle CI Deployment Trigger
+        id: curl-circle-ci
+        uses: promiseofcake/circleci-trigger-action@v1
+        with:
+          user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }}
+          project-slug: ${{ github.repository }}
+          branch: ${{ github.ref }}
+          payload: '{"build_and_test_all": true}'

.github/workflows/build-backend.yml

@@ -0,0 +1,44 @@
+###########################################################################
+# GitHub Action Workflow
+# On push to any branch, triggers the back end build and test pipeline
+# if the tdrs-backend has changed.
+#
+# Step 0: make changes on your branch to non-documentation files in
+#         tdrs-backend and push changes to your remote branch
+#
+# Step 1: Makes a request to the V2 CircleCI API to initiate the project,
+#         which will filter based upon build_and_test_backend
+#         to run the workflow/jobs listed here:
+#           build-and-test:[
+#             test-backend,
+#             test-e2e
+#           ]
+#
+#         Leverages the open source GitHub Action:
+#           https://github.com/promiseofcake/circleci-trigger-action
+###########################################################################
+name: Build Only Backend When tdrs-backend/ Files Change
+on:
+  push:
+    paths: 'tdrs-backend/**'
+    paths_ignore:
+      - '**.md'
+      - '**.txt'
+    branches-ignore:
+      - develop
+      - main
+      - master
+jobs:
+  build_and_test_backend:
+    runs-on: ubuntu-latest
+    name: Build and Test Backend
+    steps:
+    - uses: actions/checkout@v2
+    - name: Circle CI Deployment Trigger
+      id: curl-circle-ci
+      uses: promiseofcake/circleci-trigger-action@v1
+      with:
+        user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }}
+        project-slug: ${{ github.repository }}
+        branch: ${{ github.ref }}
+        payload: '{"build_and_test_backend": true}'

.github/workflows/build-frontend.yml

@@ -0,0 +1,44 @@
+###########################################################################
+# GitHub Action Workflow
+# On push to any branch, triggers the front end build and test pipeline
+# if the tdrs-frontend has changed.
+#
+# Step 0: make changes on your branch to non-documentation files in
+#         tdrs-frontend and push changes to your remote branch
+#
+# Step 1: Makes a request to the V2 CircleCI API to initiate the project,
+#         which will filter based upon build_and_test_frontend 
+#         to run the workflow/jobs listed here:
+#           build-and-test:[
+#             test-frontend,
+#             test-e2e
+#           ]
+#
+#         Leverages the open source GitHub Action:
+#           https://github.com/promiseofcake/circleci-trigger-action
+###########################################################################
+name: Build Only Frontend When tdrs-frontend Files Change
+on:
+  push:
+    paths: 'tdrs-frontend/**'
+    paths_ignore:
+      - '**.md'
+      - '**.txt'
+    branches-ignore:
+      - develop
+      - main
+      - master
+jobs:
+  build_and_test_frontend:
+    runs-on: ubuntu-latest
+    name: Build and Test Frontend
+    steps:
+    - uses: actions/checkout@v2
+    - name: Circle CI Deployment Trigger
+      id: curl-circle-ci
+      uses: promiseofcake/circleci-trigger-action@v1
+      with:
+        user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }}
+        project-slug: ${{ github.repository }}
+        branch: ${{ github.ref }}
+        payload: '{"build_and_test_frontend": true}'

.github/workflows/deploy-develop-on-merge.yml

@@ -43,6 +43,6 @@ jobs:
       uses: promiseofcake/circleci-trigger-action@v1
       with:
         user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }}
-        project-slug: raft-tech/TANF-app
+        project-slug: ${{ github.repository }}
         branch: ${{ github.ref }}
         payload: '{"develop_branch_deploy": true, "target_env": "develop"}'

.github/workflows/deploy-infrastructure.yml

@@ -0,0 +1,44 @@
+###########################################################################
+# GitHub Action Workflow
+# On push changing terraform files or infrastructure pipelines, triggers the 
+# terraform deploy pipeline for the appropriate cf space.
+#
+# Step 0: make changes to non-documentation files in terraform/ or
+#         .circleci/infrastructure/ and push/merge changes.
+#
+# Step 1: Makes a request to the V2 CircleCI API to initiate the project,
+#         which will filter based upon terraform: true flag 
+#         to run the workflow/jobs listed here:
+#           build-and-test:[
+#             enable-versioning-for-s3-buckets
+#           ]
+#
+#         Leverages the open source GitHub Action:
+#           https://github.com/promiseofcake/circleci-trigger-action
+###########################################################################
+name: Run Infrastructure Pipeline When Terraform or Infrastructure Files Change
+on:
+  push:
+    branches_ignore: #handled in CircleCI
+      - main
+      - master
+    paths:
+      - 'terraform/**'
+      - '.circleci/infrastructure/**'
+    paths_ignore:
+      - '**.md'
+      - '**.txt'
+jobs:
+  run_infrastructure_deployment:
+    runs-on: ubuntu-latest
+    name: Deploy Infrastructure
+    steps:
+      - uses: actions/checkout@v2
+      - name: Circle CI Deployment Trigger
+        id: curl-circle-ci
+        uses: promiseofcake/circleci-trigger-action@v1
+        with:
+          user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }}
+          project-slug: ${{ github.repository }}
+          branch: ${{ github.ref }}
+          payload: '{"deploy_infrastructure": true}'

.github/workflows/deploy-on-label.yml

@@ -64,6 +64,6 @@ jobs:
       uses: promiseofcake/circleci-trigger-action@v1
       with:
         user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }}
-        project-slug: raft-tech/TANF-app
+        project-slug: ${{ github.repository }}
         branch: ${{ github.head_ref }}
         payload: '{"run_dev_deployment": true, "target_env": "${{steps.extract-deploy-env.outputs.DEPLOY_ENV}}"}'

docs/Technical-Documentation/github-actions.md

@@ -0,0 +1,5 @@
+# How We Use GitHub Actions
+For now, the only use case we have for GitHub Actions is to help up trigger CircleCI builds the way we want to. This is actually the preferred method CircleCI advises for branch, path, pull-request, and labelled filtering and job triggering. See this [blog](https://circleci.com/blog/trigger-circleci-pipeline-github-action/) for details, though we use [promiseofcake/circleci-trigger-action@v](https://github.com/promiseofcake/circleci-trigger-action) plugin vs circleci/trigger_circleci_pipeline@v1.0
+
+## Path Filtering
+We use Actions to filter which workflows are getting run by CircleCI by sending different flags to CircleCI through the promiseofcake CircleCI API trigger. See the individual files in [.github](../../.github/) for detailed instructions for how to use each.