Multistage Docker + Test in Docker

This change updates force to use a multistage docker build and leverages the new docker build enginer, buildkit. Some of the benefits are better layer caching, meaning we should see decreases in the average image build time, we now run our tests in the same docker image that they are compiled in, and the production image is significantly smaller (down to 500MiB from 3GiB).
artsy · Sep 28, 2020 · 07dfd5a · 07dfd5a
1 parent e6431c0
commit 07dfd5a
Show file tree

Hide file tree

Showing 13 changed files with 315 additions and 165 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -6,6 +6,7 @@ orbs:
   horizon: artsy/release@0.0.1
   node: artsy/node@1.0.0
   yarn: artsy/yarn@5.1.3
+  artsy-remote-docker: artsy/remote-docker@dev:612b35f7058aed43d189438a46cdeee8
 
 jobs:
   run_deepcrawl_automator:
@@ -20,14 +21,6 @@ jobs:
       - run:
           name: Start automator crawl
           command: chmod +x automator.sh && ./automator.sh
-  acceptance_cypress:
-    docker:
-      - image: circleci/node:12-stretch-browsers
-    steps:
-      - yarn/setup
-      - run:
-          name: Cypress Tests
-          command: yarn test:smoke
 
   validate_production_schema:
     executor: node/build
@@ -37,29 +30,6 @@ jobs:
           name: Validate Production Schema
           command: node scripts/validateSchemas.js production
 
-  build:
-    executor: node/build
-    steps:
-      - yarn/setup
-      - run:
-          name: Build force assets
-          command: yarn assets
-      - store_artifacts:
-          path: ~/project/.artifacts
-      - run:
-          name: Duplicates Report
-          command: curl "https://artsy-dupe-report.now.sh/packages/dupe-report/now.js?owner=artsy&repo=force&buildNum=$CIRCLE_BUILD_NUM"
-
-  # FIXME: Reenable after https://github.com/artsy/force/pull/5673 is addressed
-  # danger:
-  #   executor: node/build
-  #   steps:
-  #     - yarn/setup
-  #     - run:
-  #         name: Danger
-  #         # Formatted this way to prevent GitHub's token detection. This is intended to be committed and public.
-  #         command: DANGER_GITHUB_API_TOKEN="3f715685d9d032e17""48c3368dc8f22c672849136" yarn danger ci
-
   create_or_update_review_app:
     executor: hokusai/deploy
     steps:
@@ -76,15 +46,50 @@ jobs:
           name: "Create or update review app"
           command: |
             review_app_name=$(echo $CIRCLE_BRANCH | sed 's/review-app-//')
-
             kubectl config use-context staging
-
             if $(kubectl get namespace | grep -qi $review_app_name); then
               ./scripts/update_review_app.sh $review_app_name
             else
               ./scripts/build_review_app.sh $review_app_name
             fi
 
+  mocha:
+    executor: hokusai/deploy
+    steps:
+      - hokusai/setup-docker
+      - run: hokusai registry pull --tag "$CIRCLE_SHA1"
+      - run: docker run --rm -e NODE_ENV=test --entrypoint /bin/bash "hokusai_force" /usr/local/bin/yarn test:mocha
+
+  jest:
+    executor: hokusai/deploy
+    steps:
+      - hokusai/setup-docker
+      - run: hokusai registry pull --tag "$CIRCLE_SHA1"
+      - run: docker run --rm -e NODE_ENV=test --entrypoint /bin/bash "hokusai_force" /usr/local/bin/yarn test:jest
+
+  type-check:
+    executor: hokusai/deploy
+    steps:
+      - hokusai/setup-docker
+      - run: hokusai registry pull --tag "$CIRCLE_SHA1"
+      - run: docker run --rm -e NODE_ENV=test --entrypoint /bin/bash "hokusai_force" /usr/local/bin/yarn type-check
+
+  acceptance:
+    executor: hokusai/deploy
+    steps:
+      - hokusai/setup-docker
+      - run: hokusai registry pull --tag "$CIRCLE_SHA1"
+      - run: docker build -t electron-runner -f electron.Dockerfile .
+      - run: docker run --rm --entrypoint /app/scripts/xvfb-run.sh electron-runner /usr/local/bin/yarn test:acceptance
+
+  acceptance-cypress:
+    executor: hokusai/deploy
+    steps:
+      - hokusai/setup-docker
+      - run: hokusai registry pull --tag "$CIRCLE_SHA1"
+      - run: docker build -t electron-runner -f electron.Dockerfile .
+      - run: docker run --rm --entrypoint /app/scripts/xvfb-run.sh electron-runner /usr/local/bin/yarn test:smoke
+
 not_master_or_staging_or_release: &not_master_or_staging_or_release
   filters:
     branches:
@@ -115,69 +120,59 @@ only_release: &only_release
 workflows:
   default:
     jobs:
-      - horizon/block:
-          <<: *only_release
-          context: horizon
-          project_id: 11
+      # Main build
+      - artsy-remote-docker/build:
+          <<: *not_staging_or_release
+          context: hokusai
+          name: build
+          pre-steps:
+            - run:
+                command: echo 'export BUILD_TARGET="builder"; export DOCKER_BUILDKIT=1; export BUILDKIT_PROGRESS=plain; export COMPOSE_DOCKER_CLI_BUILD=1;' >> $BASH_ENV
 
-      # Pre-staging
-      - yarn/jest:
+      - mocha:
           <<: *not_staging_or_release
-          args: --runInBand
+          context: hokusai
+          requires:
+            - build
 
-      - yarn/run:
+      - jest:
           <<: *not_staging_or_release
-          name: test:mocha
-          script: "test:mocha"
-
-      # TODO: Disabled due to memory issues. Can we use our workflows above and
-      # upload coverage that way (similar to reaction). We save a lot of time
-      # by running outside of an unnecessary docker context.
-
-      # - hokusai/test:
-      #     name: test
-      #     <<: *not_staging_or_release
-      #     post-steps:
-      #       - run: mkdir -p ./coverage ./.nyc_output ./reports
-      #       - run:
-      #           name: Copy jest coverage artifacts
-      #           command: docker cp hokusai_force_1:/app/coverage ./
-      #           when: always
-      #       - codecov/upload:
-      #           file: ./coverage/lcov.info
-      #       - run:
-      #           name: Copy mocha coverage artifacts
-      #           command: docker cp hokusai_force_1:/app/.nyc_output ./
-      #           when: always
-      #       - codecov/upload:
-      #           file: ./.nyc_output/lcov.info
-      #       - run:
-      #           name: Copy coverage reports
-      #           command: docker cp hokusai_force_1:/app/reports ./
-      #           when: always
-      #       - store_test_results:
-      #           path: ./reports
-
-      - yarn/update-cache:
+          context: hokusai
+          requires:
+            - build
+
+      - type-check:
           <<: *not_staging_or_release
-      - yarn/type-check:
+          context: hokusai
+          requires:
+            - build
+
+      - acceptance:
           <<: *not_staging_or_release
-      - acceptance_cypress:
-          <<: *not_master_or_staging_or_release
-      - build:
+          context: hokusai
+          requires:
+            - build
+
+      - acceptance-cypress:
           <<: *not_staging_or_release
-      # - danger:
-      #     <<: *not_staging_or_release
+          context: hokusai
+          requires:
+            - build
 
       # Staging
-      - hokusai/push:
-          name: push-staging-image
+      - artsy-remote-docker/build:
           <<: *only_master
+          context: hokusai
+          name: push-staging-image
           requires:
-            - yarn/type-check
-            - yarn/jest
-            - test:mocha
-            - build
+            - mocha
+            - jest
+            - type-check
+            - acceptance
+            - acceptance-cypress
+          pre-steps:
+            - run:
+                command: echo 'export BUILD_TARGET="production"; export DOCKER_BUILDKIT=1; export BUILDKIT_PROGRESS=plain; export COMPOSE_DOCKER_CLI_BUILD=1;' >> $BASH_ENV
 
       - hokusai/deploy-staging:
           <<: *only_master
@@ -187,15 +182,22 @@ workflows:
             - push-staging-image
 
       # Release
+      - horizon/block:
+          <<: *only_release
+          context: horizon
+          project_id: 11
+
       - validate_production_schema:
           <<: *only_release
+
       - hokusai/deploy-production:
           <<: *only_release
           name: deploy-production
           requires:
             - horizon/block
             - validate_production_schema
 
+      # Other
       - run_deepcrawl_automator:
           context: deepcrawl-automator
           requires:

diff --git a/.dockerignore b/.dockerignore
@@ -1,23 +1,36 @@
-.git
-.gitignore
-.dockerignore
-Dockerfile
-hokusai
+# Prefer excluding all assets by default. It prevents large artifacts from
+# accidentally becomming part of the Docker context.
+*
 
-README.md
-LICENSE
-VERSION
-docs
+# CI
+!scripts/
 
-.DS_Store
-.AppleDouble
-.LSOverride
+# Source
+!data/
+!patches/
+!src/
+!webpack/
 
-.env
-.env.*
+# Testing
+!__mocks__/
+!cypress/
 !.env.oss
-log
-tmp
-node_modules
-.vscode
-redis.conf
+!.env.test
+
+# Build Configuration
+!.eslintrc.js
+!.nvmrc
+!.prettierignore
+!apollo.config.js
+!babel.config.js
+!coffeelint.json
+!cypress.json
+!dangerfile.ts
+!jest.config.js
+!package.json
+!relay.config.js
+!renovate.json
+!test.config.js
+!test.mocha.js
+!tsconfig.json
+!yarn.lock