Add end to end tests with cypress

.github/workflows/e2e.yaml

@@ -0,0 +1,43 @@
+name: Build + e2e Test
+
+on:
+  workflow_dispatch:
+
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  e2e:
+    name: Test default experiment end-to-end
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Build Empirica
+        run: docker build -t empirica-tmp -f ./build/Dockerfile --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg BUILD_SHA=$(git rev-list -1 HEAD) --build-arg BUILD_NUM=$GITHUB_RUN_NUMBER --build-arg VERSION=v0.0.0 .
+
+      - name: Copy binaries from image
+        run: docker run --rm empirica-tmp tar -cC /out . | tar -xC $(pwd)/build/public
+
+      - name: Build default experiment
+        run: docker build -f test/Dockerfile --force-rm --platform linux/x86_64 -t empirica-test .
+
+      - name: Run cypress tests
+        uses: cypress-io/github-action@v4.0.0
+        with:
+          # in theory the container should shut down automatically
+          start: docker run -p 3000:3000 empirica-test
+          working-directory: test
+          browser: chrome
+          record: true
+          wait-on: 'http://localhost:3000'
+          wait-on-timeout: 120
+        env:
+            CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+            GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+

test/.gitignore

@@ -0,0 +1,5 @@
+/videos
+/cypress/
+/node_modules
+/screenshots
+/downloads

test/Dockerfile

@@ -0,0 +1,35 @@
+
+# Set up empirica on a container
+#FROM ubuntu:jammy as empirica_base
+# for some reason, volta had trouble installing its own node version?
+FROM node:16.15.1-bullseye as empirica_base  
+
+RUN apt-get update && apt-get install -y ca-certificates curl rsync
+# note: this file should be run from the main empirica directory,
+# with the -f flag, as is pattern for vscode: https://stackoverflow.com/a/67944726
+ADD build/public/empirica-linux-amd64 /usr/local/bin/empirica
+RUN chmod u+x /usr/local/bin/empirica
+RUN empirica setup
+#
+#RUN curl https://get.empirica.dev | sh
+# RUN apt-get remove --yes ca-certificates curl && \
+#     apt-get clean autoclean && \
+#     apt-get autoremove --yes && \
+#     rm -rf /var/lib/{apt,dpkg,cache,log}/
+
+# set up and build the template experiment
+FROM empirica_base as builder
+WORKDIR /app
+RUN empirica create test_experiment
+WORKDIR /app/test_experiment
+COPY test/empirica.toml .empirica/empirica.toml
+RUN empirica bundle
+
+# serve the template experiment
+FROM empirica_base as server
+COPY --from=builder /app/test_experiment/test_experiment.tar.zst /app/test_experiment.tar.zst
+# For some reason, the config is not picked up if it's not first setup during
+# the build, so we run server for a few seconds to let the settings settle.
+RUN timeout --preserve-status 5s empirica serve /app/test_experiment.tar.zst
+EXPOSE 3000
+ENTRYPOINT ["empirica", "serve", "/app/test_experiment.tar.zst"]

test/README.md

@@ -0,0 +1,20 @@
+
+# Test Setup
+To build empirica from source, create the test 
+
+Run this to build the empirica code
+```
+docker build -t empirica-tmp -f ./build/Dockerfile --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg BUILD_SHA=$(git rev-list -1 HEAD) --build-arg BUILD_NUM=$GITHUB_RUN_NUMBER --build-arg VERSION=v0.0.0 .
+
+docker run --rm empirica-tmp tar -cC /out . | tar -xC $(pwd)/build/public
+```
+
+
+from the project root:
+``` 
+docker build -f test/Dockerfile --force-rm --platform linux/x86_64 -t empirica-test .```
+
+then run:
+```
+docker run -p 3000:3000 empirica-test
+```

test/cypress.config.js

@@ -0,0 +1,13 @@
+module.exports = {
+  projectId: "vbn4os",
+  e2e: {
+    // We've imported your old cypress plugins here.
+    // You may want to clean this up later by importing these.
+    setupNodeEvents(on, config) {
+      return require('../test/plugins/index.js')(on, config)
+    },
+    baseUrl: 'http://localhost:3000',
+    specPattern: '../test/e2e/**/*.{js,jsx,ts,tsx}',
+    supportFile: '../test/support/e2e.{js,jsx,ts,tsx}'
+  },
+}

test/e2e/Batch_Canceled.js

@@ -0,0 +1,102 @@
+// Batch_Canceled.js
+
+describe("Batch canceled", () => {
+    const condition = "cypress1";
+
+    beforeEach(() => {
+
+        cy.empiricaLoginAdmin();
+        cy.empiricaClearBatches();
+        cy.empiricaCreateBatch("Solo");
+
+        //Start batch
+        cy.get('tr', { log: false })
+          .last({ log: false })
+          .contains("Start", { log: false })
+          .click( {log: "Start Button"})
+
+        //Check started
+        cy.waitUntil(() => 
+            cy.get('tr').last().then( $tr => $tr.find('button:contains("Stop")').length == 1)
+        )
+
+    });
+
+    it("from introsteps", () => {
+        const playerKey = 'test_'+Math.floor(Math.random() * 1e13)
+        cy.visit(`http://localhost:3000/?playerKey=${playerKey}`);
+
+        // Consent
+        cy.log("Consent");
+        cy.contains("consent", { timeout: 5000 });
+        cy.get("button").contains("I AGREE").click();
+
+        // Login
+        cy.log("Add Username");
+        cy.contains("Enter your", { timeout: 5000 });
+        cy.get("input").click().type(playerKey);
+        cy.get("button").contains("Enter").click();
+
+        //Instructions
+        cy.log("Intro Page")
+        cy.contains("Instruction", { timeout: 5000 });
+        // Do not advance here
+
+        // Cancel Batch
+        cy.empiricaLoginAdmin();
+        cy.empiricaClearBatches()
+
+        // Check redirect to correct screen
+        cy.visit(`http://localhost:3000/?playerKey=${playerKey}`);
+        cy.wait(3000)
+        cy.contains("Instruction", { timeout: 5000 }).should("not.exist");
+
+        // TODO: Check that sorry page displays here
+    });
+
+
+    it("from game", () => {
+        const playerKey = 'test_'+Math.floor(Math.random() * 1e13)
+        cy.visit(`http://localhost:3000/?playerKey=${playerKey}`);
+
+        // Consent
+        cy.log("Consent");
+        cy.contains("consent", { timeout: 5000 });
+        cy.get("button").contains("I AGREE").click();
+
+        // Login
+        cy.log("Add Username");
+        cy.contains("Enter your", { timeout: 5000 });
+        cy.get("input").click().type(playerKey);
+        cy.get("button").contains("Enter").click();
+
+        //Instructions
+        cy.log("Intro Page")
+        cy.contains("Instruction", { timeout: 5000 });
+        cy.get("button").contains("Next").click();
+
+        // Jelly Beans
+        cy.contains("Jelly Beans", { timeout: 10000 });
+        cy.get("button").contains("Submit").click();
+        cy.contains("Result", { timeout: 5000 })
+        cy.get("button").contains("Submit").click();
+
+        // Minesweeper
+        cy.contains("Minesweeper")
+        // Do not advance here
+
+        // Cancel Batch
+        cy.empiricaLoginAdmin();
+        cy.empiricaClearBatches()
+
+        // Check redirect to correct screen
+        cy.visit(`http://localhost:3000/?playerKey=${playerKey}`);
+        cy.wait(3000)
+        cy.contains("Minesweeper", { timeout: 5000 }).should("not.exist");
+
+        // TODO: Check that sorry page displays here
+    });
+
+
+
+});

test/e2e/Games_Filled.js

@@ -0,0 +1,75 @@
+// Game_Filled.js
+
+describe("All games fill up with extra player in intro steps", () => {
+  const playerKey = "test_" + Math.floor(Math.random() * 1e13);
+
+  before(() => {
+    cy.empiricaLoginAdmin();
+        cy.empiricaClearBatches();
+        cy.empiricaCreateBatch("Solo");
+
+        //Start batch
+        cy.get('tr', { log: false })
+          .last({ log: false })
+          .contains("Start", { log: false })
+          .click( {log: "Start Button"})
+
+        //Check started
+        cy.waitUntil(() => 
+            cy.get('tr').last().then( $tr => $tr.find('button:contains("Stop")').length == 1)
+        )
+
+  });
+
+  it("redirects to sorry on game full", () => {
+    //Non-completing player
+    cy.visit(`http://localhost:3000/?playerKey=${playerKey}`);
+
+    // Consent
+    cy.log("Consent");
+    cy.contains("consent", { timeout: 5000 });
+    cy.get("button").contains("I AGREE").click();
+
+    // Login
+    cy.log("Add Username");
+    cy.contains("Enter your", { timeout: 5000 });
+    cy.get("input").click().type(playerKey);
+    cy.get("button").contains("Enter").click();
+
+    //Instructions
+    cy.log("Intro Page")
+    cy.contains("Instruction", { timeout: 5000 });
+    // Do not advance beyond instructions
+
+
+    //Completing player
+    cy.visit(`http://localhost:3000/?playerKey=${playerKey + "_complete"}`);
+
+    // Consent
+    cy.log("Consent");
+    cy.contains("consent", { timeout: 5000 });
+    cy.get("button").contains("I AGREE").click();
+
+    // Login
+    cy.log("Add Username");
+    cy.contains("Enter your", { timeout: 5000 });
+    cy.get("input").click().type(playerKey);
+    cy.get("button").contains("Enter").click();
+
+    //Instructions
+    cy.log("Intro Page")
+    cy.contains("Instruction", { timeout: 5000 });
+    cy.get("button").contains("Next").click();
+
+    // Jelly Beans
+    cy.contains("Jelly Beans", { timeout: 5000 });
+    // stay in game
+
+    // Back to non-completing player
+    cy.visit(`http://localhost:3000/?playerKey=${playerKey + "_no_complete"}`);
+    cy.wait(3000);
+    cy.contains("Instruction").should("not.exist");
+
+    // Todo: check for gamefull page here.
+  });
+});

test/e2e/No_Experiments_Available.js

@@ -0,0 +1,31 @@
+// No_Experiments_Available.js
+// This tests that the user is not presented with a game option 
+// if there are no games available.
+
+describe("no_experiments_available", () => {
+    const playerKey = 'test_'+Math.floor(Math.random() * 1e13)
+
+    before(() => {
+        cy.empiricaLoginAdmin();
+        cy.empiricaCreateBatch("Solo");
+        cy.empiricaClearBatches();  // clearing just started batch
+
+
+    });
+
+    it("has no games available", () => {
+
+        cy.waitUntil(
+            () => cy.get('body', { log: false }).then( $body => $body.find('Running').length < 1),
+            {description: "-assert that no games are running"}
+        )
+
+        cy.visit(`http://localhost:3000/?playerKey=${playerKey}`);
+
+        cy.contains("No experiments available")
+        cy.wait(10000)
+        cy.contains("consent").should("not.exist");
+        cy.contains("I AGREE").should("not.exist");
+    })
+
+});