Experimental detox based tests (#145)

* Experimenting with detox e2e tests

* WIP

* Working TS tests

* Upgrades detox to 7.3.7

* Updates detox config and readme

* Tentative ios e2e build step

* fixes typo

* Install required tools

* Updated Detox config to use iPhone 6, iOS 11.2

see available sims: https://circle-macos-docs.s3.amazonaws.com/image-manifest/build-419/index.html

* Sets test timeout to 5mins

* verbose pod install

* Upgrades detox to 7.4.0

* Upgrades to detox@7.4.3

* Prestart ios simulator, collect logs

* Install ios-sim

* Tentative storing simulator logs

* Upgrades detox to 8.0.0

* Run Detox on release build

* Removes detox types (conflicts with jest)

* Adds articacts recording

* Register Jest adapter as Jasmine reporter

* Tentative fix for native-base Utils issue

* Splits detox build from running tests

* Install detox and sim utils before running tests

* Tentative fix for https://circleci.com/gh/teamdigitale/italia-app/3843

* Attempts to avoid flacky e2e tests by waiting for login button to appear

* Removes match on idp grid

* Refactored tests

* Disables synchronization, increase log verbosity

* Disable video recording of tests, requires hardware acceleration

* Skip storing (non existing) simulator logs

* A few fixes and make tests continue previous one

* Removes unused var

* Tentatively made app start faster

.circleci/config.yml

@@ -1,12 +1,51 @@
+#
+# Defaults
+#
+
+# shared
 defaults: &defaults
   working_directory: ~/italia-app
 
+# nodejs builds
 defaults_js: &defaults_js
   <<: *defaults
   parallelism: 2
   docker:
     - image: circleci/node:8
 
+# android builds
+defaults_android: &defaults_android
+  <<: *defaults
+  docker:
+    - image: circleci/android:api-27-node8-alpha
+  environment:
+    - TERM: "dumb"
+    - ANDROID_SDK_BUILD_TOOLS_REVISION: "23.0.1"
+    - ANDROID_SDK_BUILD_API_LEVEL="23": "23"
+
+# macos builds
+defaults_macos: &defaults_macos
+  <<: *defaults
+  macos:
+    xcode: "9.3.1"
+  working_directory: /Users/distiller/italia-app
+  environment:
+    # Fastlane requires locale set to UTF-8
+    # see https://docs.fastlane.tools/getting-started/ios/setup/#set-up-environment-variables
+    LC_ALL: en_US.UTF-8
+    LANG: en_US.UTF-8
+    # Set Fastlane output dir
+    FL_OUTPUT_DIR: output
+    # Make Ruby bundler a little faster
+    BUNDLE_JOBS: 3
+    BUNDLE_RETRY: 3
+    BUNDLE_PATH: vendor/bundle
+  shell: /bin/bash --login -o pipefail
+
+#
+# Build pipeline
+#
+
 version: 2
 jobs:
   # Runs JavaScript tests on Node 8
@@ -75,15 +114,7 @@ jobs:
 
   # Run test on android
   test-android:
-    <<: *defaults
-
-    docker:
-      - image: circleci/android:api-27-node8-alpha
-
-    environment:
-      - TERM: "dumb"
-      - ANDROID_SDK_BUILD_TOOLS_REVISION: "23.0.1"
-      - ANDROID_SDK_BUILD_API_LEVEL="23": "23"
+    <<: *defaults_android
 
     steps:
       # Restore workflow workspace
@@ -187,26 +218,45 @@ jobs:
 
   # Test ios build
   test-ios:
-    <<: *defaults
+    <<: *defaults_macos
 
-    macos:
-      xcode: "9.3.1"
+    steps:
+      # Restore workflow workspace
+      - attach_workspace:
+          at: /Users/distiller
 
-    working_directory: /Users/distiller/italia-app
+      # Restore bundle cache
+      - restore_cache:
+          keys:
+            - ruby-{{ checksum "Gemfile.lock" }}
+            - ruby-
 
-    environment:
-      # Fastlane requires locale set to UTF-8
-      # see https://docs.fastlane.tools/getting-started/ios/setup/#set-up-environment-variables
-      LC_ALL: en_US.UTF-8
-      LANG: en_US.UTF-8
-      # Set Fastlane output dir
-      FL_OUTPUT_DIR: output
-      # Make Ruby bundler a little faster
-      BUNDLE_JOBS: 3
-      BUNDLE_RETRY: 3
-      BUNDLE_PATH: vendor/bundle
-
-    shell: /bin/bash --login -o pipefail
+      # Set Ruby Version for chruby
+      - run: echo "ruby-2.4" > .ruby-version
+
+      # Install bundle dependencies
+      - run:
+          name: Bundle Install
+          command: bundle check || bundle install
+
+      # Store bundle cache
+      - save_cache:
+          key: ruby-{{ checksum "Gemfile.lock" }}
+          paths:
+            - vendor/bundle
+
+      # Fetch CocoaPods specs
+      - run: curl https://cocoapods-specs.circleci.com/fetch-cocoapods-repo-from-s3.sh | bash -s cf
+
+      # Re-run postinstall scripts (setup of react-native libs)
+      - run: npm run postinstall
+
+      # Move to the ios project directory and run the test_build lane
+      - run: cd ios && bundle exec fastlane test_build
+
+  # Prepare build for end-to-end iOS tests
+  test-ios-e2e-build:
+    <<: *defaults_macos
 
     steps:
       # Restore workflow workspace
@@ -222,6 +272,11 @@ jobs:
       # Set Ruby Version for chruby
       - run: echo "ruby-2.4" > .ruby-version
 
+      # TODO: possibly remove this after the following issue has been fixed:
+      # https://github.com/GeekyAnts/NativeBase/issues/1957
+      - run: '[ -d node_modules/native-base/dist/src/Utils ] && mv node_modules/native-base/dist/src/Utils node_modules/native-base/dist/src/utils'
+      - run: '[ -d node_modules/native-base/src/Utils ] && mv node_modules/native-base/src/Utils node_modules/native-base/src/utils'
+
       # Install bundle dependencies
       - run:
           name: Bundle Install
@@ -239,8 +294,58 @@ jobs:
       # Re-run postinstall scripts (setup of react-native libs)
       - run: npm run postinstall
 
-      # Move to the ios project directory and run the test_build lane
-      - run: cd ios && bundle exec fastlane test_build
+      # Install pods
+      - run: cd ios && pod install --verbose
+
+      # Install Detox and ios-sim utils
+      # see: https://github.com/wix/detox/
+      - run: npm install -g detox
+
+      # cleanup metro and haste cache
+      - run: rm -rf /tmp/metro-bundler-cache-*
+      - run: rm -rf /tmp/haste-map-react-native-packager-*
+
+      # Run detox e2e tests
+      - run: detox build -c ios.sim.release
+
+      # Save workspace for next jobs in the workflow
+      - persist_to_workspace:
+          root: /Users/distiller
+          paths:
+            - italia-app
+
+  # Run end-to-end iOS tests
+  test-ios-e2e-run:
+    <<: *defaults_macos
+
+    steps:
+      # Restore workflow workspace
+      - attach_workspace:
+          at: /Users/distiller
+
+      # Install Detox and ios-sim utils
+      # see: https://github.com/wix/detox/
+      - run: npm install -g detox ios-sim
+
+      # Install required tools
+      # see: https://github.com/wix/detox/blob/master/docs/Guide.RunningOnCI.md
+      - run: brew tap wix/brew
+      - run: brew install applesimutils
+
+      # Enumerate device types available in simulator
+      - run: ios-sim showdevicetypes
+
+      # Start emulator in background
+      - run:
+          command: ios-sim start --verbose --log /tmp/detox.last_launch_app_log --devicetypeid "iPhone-6, 10.3"
+          background: true
+
+      # Run detox e2e tests
+      - run: detox test --loglevel verbose -c ios.sim.release --cleanup --artifacts-location /tmp/detox_artifacts/ --record-logs all --take-screenshots all --debug-synchronization 1000
+
+      # store detox artifacts
+      - store_artifacts:
+          path: /tmp/detox_artifacts
 
 workflows:
   version: 2
@@ -266,10 +371,17 @@ workflows:
           requires:
             - hold-native-tests
 
-      # Native iOS tests
-      - test-ios:
+      # Native iOS e2e tests
+      - hold-ios-e2e-tests:
+          type: approval
           requires:
-            - hold-native-tests
+            - test-js-node-8
+      - test-ios-e2e-build:
+          requires:
+            - hold-ios-e2e-tests
+      - test-ios-e2e-run:
+          requires:
+            - test-ios-e2e-build
 
       # Build Android alpha release only on master branch
       - alpha-release-android:

README.md

@@ -451,6 +451,33 @@ Esempio di utilizzo:
 
 Per cambiare il tema del wrapper, dell'icona o del testo modificare il file `ts/theme/components/TextWithIcon.ts`.
 
+### Test end to end con Detox (sperimentale)
+
+Per i test di integrazione sui simulatori usiamo
+[Detox](https://github.com/wix/detox).
+
+I test end to end si trovano in [ts/__e2e__/](ts/__e2e__/).
+
+Per compilare l'app in preparazione al test:
+
+```
+$ detox build
+```
+
+(opzionale) Lanciare il simulatore iOS (con [ios-sim](https://www.npmjs.com/package/ios-sim) per comodità):
+
+```
+$ ios-sim start --devicetypeid "iPhone-6, 10.2"
+```
+
+Nel caso non si lanci il simulatore, Detox ne lancerà uno in background.
+
+Lancio dei test:
+
+```
+$ detox test
+```
 
 ## License
-[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgh.neting.cc%2Fteamdigitale%2Fitalia-app.svg?type=large)](https://app.fossa.io/projects/git%2Bgh.neting.cc%2Fteamdigitale%2Fitalia-app?ref=badge_large)
+
+[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgh.neting.cc%2Fteamdigitale%2Fitalia-app.svg?type=large)](https://app.fossa.io/projects/git%2Bgh.neting.cc%2Fteamdigitale%2Fitalia-app?ref=badge_large)

jest-e2e.json

@@ -0,0 +1,18 @@
+{
+  "modulePaths": [
+    "<rootDir>"
+  ],
+  "moduleFileExtensions": [
+    "ts",
+    "tsx",
+    "js"
+  ],
+  "transform": {
+    "^.+\\.(ts|tsx)$": "<rootDir>/node_modules/ts-jest/preprocessor.js"
+  },
+  "testMatch": [
+    "**/__e2e__/*.e2e.ts?(x)"
+  ],
+  "forceExit": true,
+  "verbose": true
+}

package.json

@@ -62,6 +62,7 @@
     "@types/color": "^3.0.0",
     "@types/jest": "^23.1.1",
     "@types/lodash": "^4.14.108",
+    "@types/node": "^10.1.0",
     "@types/react": "16.3.16",
     "@types/react-native": "0.55.17",
     "@types/react-native-fs": "^2.8.1",
@@ -82,6 +83,7 @@
     "babel-preset-react-native": "4.0.0",
     "danger": "^3.7.19",
     "danger-plugin-digitalcitizenship": "^0.3.1",
+    "detox": "^8.0.0",
     "italia-tslint-rules": "^0.1.1",
     "italia-utils": "^3.11.1",
     "jest": "23.0.1",
@@ -115,7 +117,7 @@
       "^.+\\.(ts|tsx)$": "<rootDir>/node_modules/ts-jest/preprocessor.js"
     },
     "testMatch": [
-      "**/__tests__/*.ts?(x)"
+      "**/__tests__/*.(test|spec).ts?(x)"
     ],
     "transformIgnorePatterns": [
       "node_modules/(?!react-native|react-navigation|react-native-device-info|native-base|native-base-shoutem-theme|@shoutem/animation|@shoutem/ui)"
@@ -128,5 +130,24 @@
       "./assets/fonts/io-icon-font",
       "./assets/fonts/RobotoMono"
     ]
+  },
+  "detox": {
+    "test-runner": "jest",
+    "specs": "e2e",
+    "runner-config": "jest-e2e.json",
+    "configurations": {
+      "ios.sim.debug": {
+        "binaryPath": "ios/build/Build/Products/Debug-iphonesimulator/ItaliaApp.app",
+        "build": "xcodebuild -workspace ios/ItaliaApp.xcworkspace -scheme ItaliaApp -configuration Debug -sdk iphonesimulator -derivedDataPath ios/build",
+        "type": "ios.simulator",
+        "name": "iPhone 6, iOS 10.3"
+      },
+      "ios.sim.release": {
+        "binaryPath": "ios/build/Build/Products/Release-iphonesimulator/ItaliaApp.app",
+        "build": "xcodebuild -workspace ios/ItaliaApp.xcworkspace -scheme ItaliaApp -configuration Release -sdk iphonesimulator -derivedDataPath ios/build",
+        "type": "ios.simulator",
+        "name": "iPhone 6, iOS 10.3"
+      }
+    }
   }
 }