Convert examples tests to run through prysk (vercel#4000)

The advantage of running examples tests through prysk are: - isolated directory by default. don't need to setup a nested git repo - can run all tests in a single command (`.cram_env/bin/prysk example_tests/1) - matching the pattern with existing integration tests - set the pattern to add more sophisticated real-world integration tests against examples
chris-olszewski · Feb 28, 2023 · 55d7f5e · 55d7f5e
1 parent 40bc5ea
commit 55d7f5e
Show file tree

Hide file tree

Showing 14 changed files with 110 additions and 233 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -152,9 +152,7 @@ jobs:
         with:
           PATTERNS: |
             examples/**
-            cli/**/*.go
-            cli/go.mod
-            cli/go.sum
+            examples_tests/**
             scripts/run-example.sh
             !**.md
             !**.mdx

diff --git a/.gitignore b/.gitignore
@@ -47,3 +47,9 @@ sweep.timestamp
 crates/turborepo-ffi/bindings.h
 crates/turborepo-ffi/ffi/proto/*
 cli/internal/ffi/libturborepo_ffi.a
+
+# Prysk test error files
+*.t.err
+*.t.rej
+*.t.orig
+.cram_env
diff --git a/cli/.gitignore b/cli/.gitignore
@@ -15,7 +15,6 @@
 /scripts/turbo-*
 /.cram_env
 testbed
-integration_tests/**/*.t.err
 
 # Windows lib files
 turbo.h

diff --git a/examples_tests/npm-non-monorepo.t b/examples_tests/npm-non-monorepo.t
@@ -0,0 +1,5 @@
+  $ . ${TESTDIR}/setup.sh non-monorepo npm
+# run twice and make sure it works
+  $ npx turbo build lint > /dev/null
+  $ npx turbo build lint > /dev/null
+  $ git diff
diff --git a/examples_tests/npm-with-npm.t b/examples_tests/npm-with-npm.t
@@ -0,0 +1,5 @@
+  $ . ${TESTDIR}/setup.sh with-npm npm
+# run twice and make sure it works
+  $ npm run build lint > /dev/null
+  $ npm run build lint > /dev/null  
+  $ git diff
diff --git a/examples_tests/npm-with-yarn.t b/examples_tests/npm-with-yarn.t
@@ -0,0 +1,5 @@
+  $ . ${TESTDIR}/setup.sh with-yarn npm
+# run twice and make sure it works
+  $ npm run build lint > /dev/null
+  $ npm run build lint > /dev/null  
+  $ git diff
diff --git a/examples_tests/pnpm-basic.t b/examples_tests/pnpm-basic.t
@@ -0,0 +1,5 @@
+  $ . ${TESTDIR}/setup.sh basic pnpm
+# run twice and make sure it works
+  $ pnpm run build lint > /dev/null
+  $ pnpm run build lint > /dev/null
+  $ git diff
diff --git a/examples_tests/pnpm-kitchen-sink.t b/examples_tests/pnpm-kitchen-sink.t
@@ -0,0 +1,5 @@
+  $ . ${TESTDIR}/setup.sh kitchen-sink pnpm
+# run twice and make sure it works
+  $ pnpm run build lint > /dev/null
+  $ pnpm run build lint > /dev/null
+  $ git diff
diff --git a/examples_tests/pnpm-with-svelte.t b/examples_tests/pnpm-with-svelte.t
@@ -0,0 +1,5 @@
+  $ . ${TESTDIR}/setup.sh with-svelte pnpm
+# run twice and make sure it works
+  $ pnpm run build lint > /dev/null
+  $ pnpm run build lint > /dev/null
+  $ git diff
diff --git a/examples_tests/setup.sh b/examples_tests/setup.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+set -e
+
+exampleName=$1
+pkgManager=$2
+
+# Copy the example dir over to the test dir that prysk puts you in
+SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
+EXAMPLE_DIR="../examples/$exampleName"
+TARGET_DIR="$(pwd)"
+cp -a "${SCRIPT_DIR}/$EXAMPLE_DIR/." "${TARGET_DIR}/"
+
+# cleanup lockfiles so we can install from scratch
+[ ! -f yarn.lock ] || mv yarn.lock yarn.lock.bak
+[ ! -f pnpm-lock.yaml ] || mv pnpm-lock.yaml pnpm-lock.yaml.bak
+[ ! -f package-lock.json ] || mv package-lock.json package-lock.json.bak
+
+# $TESTDIR is set by prysk to be the directory the test script is in
+# (not this setup.sh script, but it happens to be the same.
+SOURCE_TURBO_DIR="$TESTDIR/../cli"
+TURBO_VERSION_FILE="${SOURCE_TURBO_DIR}/../version.txt"
+# Change package.json in the example directory to point to @canary if our branch is currently at that version
+TURBO_TAG=$(cat "$TURBO_VERSION_FILE" | sed -n '2 p')
+if [ "$TURBO_TAG" == "canary" ]; then
+  cat package.json | jq '.devDependencies.turbo = "canary"' | sponge package.json
+fi
+
+function set_package_manager() {
+  cat package.json | jq ".packageManager=\"$1\"" | sponge package.json
+}
+
+# Set the packageManger version
+NPM_PACKAGE_MANAGER_VALUE="npm@8.1.2"
+PNPM_PACKAGE_MANAGER_VALUE="pnpm@6.26.1"
+YARN_PACKAGE_MANAGER_VALUE="yarn@1.22.17"
+if [ "$pkgManager" == "npm" ]; then
+  set_package_manager "$NPM_PACKAGE_MANAGER_VALUE"
+  npm install > /dev/null
+elif [ "$pkgManager" == "pnpm" ]; then
+  set_package_manager "$PNPM_PACKAGE_MANAGER_VALUE"
+  pnpm install > /dev/null
+elif [ "$pkgManager" == "yarn" ]; then
+  set_package_manager "$YARN_PACKAGE_MANAGER_VALUE"
+  yarn install > /dev/null
+fi
+
+# Delete .git directory if it's there, we'll set up a new git repo
+[ ! -d .git ] || rm -rf .git
+"${SCRIPT_DIR}/../cli/integration_tests/setup_git.sh" "${TARGET_DIR}"
diff --git a/examples_tests/yarn-non-monorepo.t b/examples_tests/yarn-non-monorepo.t
@@ -0,0 +1,5 @@
+  $ . ${TESTDIR}/setup.sh non-monorepo yarn
+# run twice and make sure it works
+  $ npx turbo build lint > /dev/null
+  $ npx turbo build lint > /dev/null
+  $ git diff
diff --git a/examples_tests/yarn-with-npm.t b/examples_tests/yarn-with-npm.t
@@ -0,0 +1,5 @@
+  $ . ${TESTDIR}/setup.sh with-npm yarn
+# run twice and make sure it works
+  $ yarn build lint > /dev/null
+  $ yarn build lint > /dev/null
+  $ git diff
diff --git a/examples_tests/yarn-with-yarn.t b/examples_tests/yarn-with-yarn.t
@@ -0,0 +1,5 @@
+  $ . ${TESTDIR}/setup.sh with-yarn yarn
+# run twice and make sure it works
+  $ yarn turbo build lint > /dev/null
+  $ yarn turbo build lint > /dev/null
+  $ git diff
diff --git a/scripts/run-example.sh b/scripts/run-example.sh
@@ -2,238 +2,18 @@
 
 set -e
 
-echo "=> Running examples..."
-# echo "=> Building turbo from source..."
-# cd cli && CGO_ENABLED=0 go build ./cmd/turbo/... && cd ..;
-export TURBO_BINARY_PATH=$(pwd)/target/debug/turbo
-export TURBO_VERSION=$(head -n 1 $(pwd)/version.txt)
-export TURBO_TAG=$(cat $(pwd)/version.txt | sed -n '2 p')
+python3 -m venv .cram_env
+.cram_env/bin/pip install prysk
+
 export folder=$1
 export pkgManager=$2
-echo "=> Binary path: TURBO_BINARY_PATH=$TURBO_BINARY_PATH"
-echo "=> Local Turbo Version: TURBO_VERSION=$TURBO_VERSION"
-echo "=> Moving our own eslint settings out of the way..."
-echo "=> Actually running examples for real..."
-
-if [ -f ".eslintrc.js" ]; then
-  mv .eslintrc.js .eslintrc.js.bak
-fi
-
-function cleanup {
-  rm -rf node_modules
-  rm -rf apps/*/node_modules
-  rm -rf apps/*/.next
-  rm -rf apps/*/.turbo
-  rm -rf packages/*/node_modules
-  rm -rf packages/*/.next
-  rm -rf packages/*/.turbo
-  rm -rf *.log
-}
-
-function setup_git {
-  echo "=> Setting up git..."
-  rm -rf .git
-  mkdir .git
-  touch .git/config
-  echo "[user]" >> .git/config
-  echo "  name = GitHub Actions" >> .git/config
-  echo "  email = actions@users.noreply.github.com" >> .git/config
-  echo "" >> .git/config
-  echo "[init]" >> .git/config
-  echo "  defaultBranch = main" >> .git/config
-  git init . -q
-  git add .
-  git commit -m "Initial commit"
-}
-
-function run_npm {
-  cat package.json | jq '.packageManager = "npm@8.1.2"' | sponge package.json
-  if [ "$TURBO_TAG" == "canary" ]; then
-    cat package.json | jq '.devDependencies.turbo = "canary"' | sponge package.json
-  fi
-
-  echo "======================================================="
-  echo "=> $folder: move lockfile"
-  echo "======================================================="
-  [ ! -f yarn.lock ] || mv yarn.lock yarn.lock.bak
-  [ ! -f pnpm-lock.yaml ] || mv pnpm-lock.yaml pnpm-lock.yaml.bak
-  [ ! -f package-lock.json ] || mv package-lock.json package-lock.json.bak
-
-  echo "======================================================="
-  echo "=> $folder: npm install"
-  echo "======================================================="
-  npm install --force
-
-  echo "======================================================="
-  echo "=> $folder: npm build lint"
-  echo "======================================================="
-  if [ $folder == "non-monorepo" ]; then
-    npx turbo build lint
-  else
-    npm run build lint
-  fi
-
-  echo "======================================================="
-  echo "=> $folder: npm build lint again"
-  echo "======================================================="
-  if [ $folder == "non-monorepo" ]; then
-    npx turbo build lint
-  else
-    npm run build lint
-  fi
-
-  # restore lockfile
-  rm -rf package-lock.json
-  [ ! -f yarn.lock.bak ] || mv yarn.lock.bak yarn.lock
-  [ ! -f pnpm-lock.yaml.bak ] || mv pnpm-lock.yaml.bak pnpm-lock.yaml
-  [ ! -f package-lock.json.bak ] || mv package-lock.json.bak package-lock.json
-
-  echo "======================================================="
-  echo "=> $folder: npm SUCCESSFUL"
-  echo "======================================================="
-}
-
-function run_pnpm {
-  cat package.json | jq '.packageManager = "pnpm@6.26.1"' | sponge package.json
-  if [ "$TURBO_TAG" == "canary" ]; then
-    cat package.json | jq '.devDependencies.turbo = "canary"' | sponge package.json
-  fi
-
-  echo "======================================================="
-  echo "=> $folder: move lockfile"
-  echo "======================================================="
-  [ ! -f yarn.lock ] || mv yarn.lock yarn.lock.bak
-  [ ! -f pnpm-lock.yaml ] || mv pnpm-lock.yaml pnpm-lock.yaml.bak
-  [ ! -f package-lock.json ] || mv package-lock.json package-lock.json.bak
-
-  echo "======================================================="
-  echo "=> $folder: pnpm install"
-  echo "======================================================="
-  pnpm install
-
-  echo "======================================================="
-  echo "=> $folder: pnpm build lint"
-  echo "======================================================="
-  if [ $folder == "non-monorepo" ]; then
-    pnpm turbo build lint
-  else
-    pnpm run build lint
-  fi
-
-  echo "======================================================="
-  echo "=> $folder: pnpm build lint again"
-  echo "======================================================="
-  if [ $folder == "non-monorepo" ]; then
-    pnpm turbo build lint
-  else
-    pnpm run build lint
-  fi
-
-  # restore lockfile
-  rm -rf pnpm-lock.yaml
-  [ ! -f yarn.lock.bak ] || mv yarn.lock.bak yarn.lock
-  [ ! -f pnpm-lock.yaml.bak ] || mv pnpm-lock.yaml.bak pnpm-lock.yaml
-  [ ! -f package-lock.json.bak ] || mv package-lock.json.bak package-lock.json
-
-  echo "======================================================="
-  echo "=> $folder: pnpm SUCCESSFUL"
-  echo "======================================================="
-}
 
-function run_yarn {
-  cat package.json | jq '.packageManager = "yarn@1.22.17"' | sponge package.json
-  if [ "$TURBO_TAG" == "canary" ]; then
-    cat package.json | jq '.devDependencies.turbo = "canary"' | sponge package.json
-  fi
+TEST_FILE="examples_tests/$2-$1.t"
 
-  echo "======================================================="
-  echo "=> $folder: move lockfile"
-  echo "======================================================="
-  [ ! -f yarn.lock ] || mv yarn.lock yarn.lock.bak
-  [ ! -f pnpm-lock.yaml ] || mv pnpm-lock.yaml pnpm-lock.yaml.bak
-  [ ! -f package-lock.json ] || mv package-lock.json package-lock.json.bak
-
-
-  echo "======================================================="
-  echo "=> $folder: yarn install"
-  echo "======================================================="
-  yarn install
-
-  echo "======================================================="
-  echo "=> $folder: yarn build lint"
-  echo "======================================================="
-  if [ $folder == "non-monorepo" ]; then
-    yarn turbo build lint
-  else
-    yarn build lint
-  fi
-
-  echo "======================================================="
-  echo "=> $folder: yarn build again"
-  echo "======================================================="
-  if [ $folder == "non-monorepo" ]; then
-
-    yarn turbo build lint
-  else
-    yarn build lint
-  fi
-
-    # restore lockfile
-  rm -rf yarn.lock
-  [ ! -f yarn.lock.bak ] || mv yarn.lock.bak yarn.lock
-  [ ! -f pnpm-lock.yaml.bak ] || mv pnpm-lock.yaml.bak pnpm-lock.yaml
-  [ ! -f package-lock.json.bak ] || mv package-lock.json.bak package-lock.json
-
-  echo "======================================================="
-  echo "=> $folder: yarn SUCCESSFUL"
-  echo "======================================================="
-}
-
-
-hasRun=0
-if [ -f "examples/$folder/package.json" ]; then
-  cd "examples/$folder"
-  echo "======================================================="
-  echo "=> checking $folder "
-  echo "======================================================="
-  cleanup
-  setup_git
-  # save the default packageManager
-  packageManager=$(cat 'package.json' | jq '.packageManager')
-  if [ "$pkgManager" == "npm" ]; then
-    run_npm
-  elif [ "$pkgManager" == "pnpm" ]; then
-    run_pnpm
-  elif [ "$pkgManager" == "yarn" ]; then
-    run_yarn
-  else
-    echo "Unknown package manager ${folder}"
-    exit 2
-  fi
-  hasRun=1
-
-  # reset to the default packageManager
-  cat package.json | jq ".packageManager = ${packageManager}" | sponge package.json
-  if [ "$TURBO_TAG" == "canary" ]; then
-    cat package.json | jq '.devDependencies.turbo = "latest"' | sponge package.json
-  fi
-
-  cleanup
-
-  cd ../..
-fi
-
-if [ -f ".eslintrc.js.bak" ]; then
-  mv .eslintrc.js.bak .eslintrc.js
-fi
-
-if [[ ! -z $(git status -s) ]];then
-  echo "Detected changes"
-  git diff
+if [ -f "$TEST_FILE" ]; then
+  echo "Running $TEST_FILE"
+  .cram_env/bin/prysk --shell="$(which bash)" "$TEST_FILE"
+else
+  echo "Could not find $TEST_FILE"
   exit 1
 fi
-
-if [ $hasRun -eq 0 ]; then
-  echo "Did not run any examples"
-  exit 2
-fi