Multistage Docker + Test in Docker

This change updates force to use a multistage docker build and leverages the new docker build enginer, buildkit. Some of the benefits are better layer caching, meaning we should see decreases in the average image build time, we now run our tests in the same docker image that they are compiled in, and the production image is significantly smaller (down to 500MiB from 3GiB).
artsy · Sep 30, 2020 · d1c8fd0 · d1c8fd0
1 parent e6431c0
commit d1c8fd0
Show file tree

Hide file tree

Showing 15 changed files with 589 additions and 217 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -6,6 +6,7 @@ orbs:
   horizon: artsy/release@0.0.1
   node: artsy/node@1.0.0
   yarn: artsy/yarn@5.1.3
+  artsy-remote-docker: artsy/remote-docker@dev:d16c1de2b9f6072b68c6b2c8ef44cbe7
 
 jobs:
   run_deepcrawl_automator:
@@ -20,14 +21,6 @@ jobs:
       - run:
           name: Start automator crawl
           command: chmod +x automator.sh && ./automator.sh
-  acceptance_cypress:
-    docker:
-      - image: circleci/node:12-stretch-browsers
-    steps:
-      - yarn/setup
-      - run:
-          name: Cypress Tests
-          command: yarn test:smoke
 
   validate_production_schema:
     executor: node/build
@@ -37,29 +30,6 @@ jobs:
           name: Validate Production Schema
           command: node scripts/validateSchemas.js production
 
-  build:
-    executor: node/build
-    steps:
-      - yarn/setup
-      - run:
-          name: Build force assets
-          command: yarn assets
-      - store_artifacts:
-          path: ~/project/.artifacts
-      - run:
-          name: Duplicates Report
-          command: curl "https://artsy-dupe-report.now.sh/packages/dupe-report/now.js?owner=artsy&repo=force&buildNum=$CIRCLE_BUILD_NUM"
-
-  # FIXME: Reenable after https://github.com/artsy/force/pull/5673 is addressed
-  # danger:
-  #   executor: node/build
-  #   steps:
-  #     - yarn/setup
-  #     - run:
-  #         name: Danger
-  #         # Formatted this way to prevent GitHub's token detection. This is intended to be committed and public.
-  #         command: DANGER_GITHUB_API_TOKEN="3f715685d9d032e17""48c3368dc8f22c672849136" yarn danger ci
-
   create_or_update_review_app:
     executor: hokusai/deploy
     steps:
@@ -76,15 +46,55 @@ jobs:
           name: "Create or update review app"
           command: |
             review_app_name=$(echo $CIRCLE_BRANCH | sed 's/review-app-//')
-
             kubectl config use-context staging
-
             if $(kubectl get namespace | grep -qi $review_app_name); then
               ./scripts/update_review_app.sh $review_app_name
             else
               ./scripts/build_review_app.sh $review_app_name
             fi
 
+  mocha:
+    executor: hokusai/deploy
+    steps:
+      - hokusai/setup-docker
+      - run: hokusai registry pull --tag "${CIRCLE_SHA1}-builder"
+      - run: docker run --rm -e NODE_ENV=test --entrypoint /bin/bash "hokusai_force" /usr/local/bin/yarn test:mocha
+
+  jest-v1:
+    executor: hokusai/deploy
+    steps:
+      - hokusai/setup-docker
+      - run: hokusai registry pull --tag "${CIRCLE_SHA1}-builder"
+      - run: docker run --rm -e NODE_ENV=test --entrypoint /bin/bash "hokusai_force" /usr/local/bin/yarn test:jest:v1
+
+  jest-v2:
+    executor: hokusai/deploy
+    steps:
+      - hokusai/setup-docker
+      - run: hokusai registry pull --tag "${CIRCLE_SHA1}-builder"
+      - run: docker run --rm -e NODE_ENV=test --entrypoint /bin/bash "hokusai_force" /usr/local/bin/yarn test:jest:v2
+
+  type-check:
+    executor: hokusai/deploy
+    steps:
+      - hokusai/setup-docker
+      - run: hokusai registry pull --tag "${CIRCLE_SHA1}-builder"
+      - run: docker run --rm -e NODE_ENV=test --entrypoint /bin/bash "hokusai_force" /usr/local/bin/yarn type-check
+
+  acceptance:
+    executor: hokusai/deploy
+    steps:
+      - hokusai/setup-docker
+      - run: hokusai registry pull --tag "${CIRCLE_SHA1}-electron-runner"
+      - run: docker run --rm --entrypoint /app/scripts/xvfb-run.sh "hokusai_force" /usr/local/bin/yarn test:acceptance
+
+  acceptance-cypress:
+    executor: hokusai/deploy
+    steps:
+      - hokusai/setup-docker
+      - run: hokusai registry pull --tag "${CIRCLE_SHA1}-electron-runner"
+      - run: docker run --rm --entrypoint /app/scripts/xvfb-run.sh "hokusai_force" /usr/local/bin/yarn test:smoke
+
 not_master_or_staging_or_release: &not_master_or_staging_or_release
   filters:
     branches:
@@ -115,87 +125,127 @@ only_release: &only_release
 workflows:
   default:
     jobs:
-      - horizon/block:
-          <<: *only_release
-          context: horizon
-          project_id: 11
+      # Main build
+      - artsy-remote-docker/buildkit-build:
+          <<: *not_staging_or_release
+          context: hokusai
+          name: builder-build
+          pre-steps:
+            - run:
+                command: echo 'export BUILD_TARGET="builder";' >> $BASH_ENV
 
-      # Pre-staging
-      - yarn/jest:
+      - artsy-remote-docker/buildkit-push:
           <<: *not_staging_or_release
-          args: --runInBand
+          context: hokusai
+          name: builder-push
+          requires:
+            - builder-build
+          pre-steps:
+            - run:
+                command: echo 'export BUILD_TARGET="builder";' >> $BASH_ENV
 
-      - yarn/run:
+      # Electron Runner build
+      - artsy-remote-docker/buildkit-build:
           <<: *not_staging_or_release
-          name: test:mocha
-          script: "test:mocha"
-
-      # TODO: Disabled due to memory issues. Can we use our workflows above and
-      # upload coverage that way (similar to reaction). We save a lot of time
-      # by running outside of an unnecessary docker context.
-
-      # - hokusai/test:
-      #     name: test
-      #     <<: *not_staging_or_release
-      #     post-steps:
-      #       - run: mkdir -p ./coverage ./.nyc_output ./reports
-      #       - run:
-      #           name: Copy jest coverage artifacts
-      #           command: docker cp hokusai_force_1:/app/coverage ./
-      #           when: always
-      #       - codecov/upload:
-      #           file: ./coverage/lcov.info
-      #       - run:
-      #           name: Copy mocha coverage artifacts
-      #           command: docker cp hokusai_force_1:/app/.nyc_output ./
-      #           when: always
-      #       - codecov/upload:
-      #           file: ./.nyc_output/lcov.info
-      #       - run:
-      #           name: Copy coverage reports
-      #           command: docker cp hokusai_force_1:/app/reports ./
-      #           when: always
-      #       - store_test_results:
-      #           path: ./reports
-
-      - yarn/update-cache:
+          context: hokusai
+          name: electron-runner-build
+          requires:
+            - builder-build
+          pre-steps:
+            - run:
+                command: echo 'export BUILD_TARGET="electron-runner";' >> $BASH_ENV
+
+      - artsy-remote-docker/buildkit-push:
           <<: *not_staging_or_release
-      - yarn/type-check:
+          context: hokusai
+          name: electron-runner-push
+          requires:
+            - electron-runner-build
+          pre-steps:
+            - run:
+                command: echo 'export BUILD_TARGET="electron-runner";' >> $BASH_ENV
+
+      # Test steps
+      - mocha:
           <<: *not_staging_or_release
-      - acceptance_cypress:
-          <<: *not_master_or_staging_or_release
-      - build:
+          context: hokusai
+          requires:
+            - builder-push
+
+      - jest-v1:
+          <<: *not_staging_or_release
+          context: hokusai
+          requires:
+            - builder-push
+
+      - jest-v2:
           <<: *not_staging_or_release
-      # - danger:
-      #     <<: *not_staging_or_release
+          context: hokusai
+          requires:
+            - builder-push
+
+      - type-check:
+          <<: *not_staging_or_release
+          context: hokusai
+          requires:
+            - builder-push
+
+      - acceptance:
+          <<: *not_staging_or_release
+          context: hokusai
+          requires:
+            - electron-runner-push
+
+      - acceptance-cypress:
+          <<: *not_staging_or_release
+          context: hokusai
+          requires:
+            - electron-runner-push
 
       # Staging
-      - hokusai/push:
-          name: push-staging-image
+      - artsy-remote-docker/buildkit-build:
           <<: *only_master
+          context: hokusai
+          name: production-build
           requires:
-            - yarn/type-check
-            - yarn/jest
-            - test:mocha
-            - build
+            - mocha
+            - jest-v1
+            - jest-v2
+            - type-check
+            - acceptance
+            - acceptance-cypress
+
+      - artsy-remote-docker/buildkit-push:
+          <<: *not_staging_or_release
+          context: hokusai
+          name: production-push
+          requires:
+            - production-build
 
       - hokusai/deploy-staging:
           <<: *only_master
           name: deploy-staging
           project-name: force
           requires:
-            - push-staging-image
+            - production-push
 
       # Release
+      - horizon/block:
+          <<: *only_release
+          context: horizon
+          project_id: 11
+
       - validate_production_schema:
           <<: *only_release
+
       - hokusai/deploy-production:
           <<: *only_release
           name: deploy-production
           requires:
             - horizon/block
             - validate_production_schema
 
+      # Other
       - run_deepcrawl_automator:
           context: deepcrawl-automator
           requires:

diff --git a/.dockerignore b/.dockerignore
@@ -1,23 +1,36 @@
-.git
-.gitignore
-.dockerignore
-Dockerfile
-hokusai
+# Prefer excluding all assets by default. It prevents large artifacts from
+# accidentally becomming part of the Docker context.
+*
 
-README.md
-LICENSE
-VERSION
-docs
+# CI
+!scripts/
 
-.DS_Store
-.AppleDouble
-.LSOverride
+# Source
+!data/
+!patches/
+!src/
+!webpack/
 
-.env
-.env.*
+# Testing
+!__mocks__/
+!cypress/
 !.env.oss
-log
-tmp
-node_modules
-.vscode
-redis.conf
+!.env.test
+!cypress.json
+!jest.config.v1.js
+!jest.config.v2.js
+!test.config.js
+!test.mocha.js
+
+# Build Configuration
+!.eslintrc.js
+!.nvmrc
+!.prettierignore
+!apollo.config.js
+!babel.config.js
+!coffeelint.json
+!dangerfile.ts
+!package.json
+!relay.config.js
+!tsconfig.json
+!yarn.lock