fix: Snapshot makes shot even when its failed on specific functionali…

…ty (#177)

* fix: Snapshot makes shots even if selector in function not found

* fix: Comments update

* fix: Update height on firefox for tests

* fix: E2E Cleaning state remove and setup pipeline to save records for debug

* fix: Update docker composer with mapping volume

* fix: update reference for firefox

* fix: Snapshot added E2E tests

.gitignore

@@ -20,16 +20,27 @@ coverage/*
 
 # e2eTest folders 
 e2eTests/generic/updateBaseline/baseline/*
+e2eTests/generic/gridError/baseline/*
+e2eTests/generic/nonExistScript/baseline/*
 e2eTests/chrome/snap/latest/*
 e2eTests/firefox/snap/latest/*
 
 e2eTests/generic/compare/generatedDiffs/*
 e2eTests/generic/compare/reports/*
-
+e2eTests/generic/onBeforeSuiteScript/latest/*
+e2eTests/generic/gridError/latest/*
+e2eTests/generic/nonExistScript/latest/*
 
 e2eTests/chrome/removeElements/latest/*
 e2eTests/firefox/removeElements/latest/*
-
+e2eTests/chrome/removeElements/generatedDiffs/*
+e2eTests/firefox/removeElements/generatedDiffs/*
+e2eTests/chrome/removeElements/reports/*
+e2eTests/firefox/removeElements/reports/*
 
 e2eTests/chrome/hideElements/latest/*
 e2eTests/firefox/hideElements/latest/*
+e2eTests/chrome/hideElements/generatedDiffs/*
+e2eTests/firefox/hideElements/generatedDiffs/*
+e2eTests/chrome/hideElements/reports/*
+e2eTests/firefox/hideElements/reports/*

e2eTests/chrome/hideElements/hideElements.test.js

@@ -1,27 +1,8 @@
 /* globals expect */
 
 import { execSync } from 'child_process';
-import path from 'path';
-import fs from 'fs';
-import config from './hideElementsConfig';
-
-function cleanState(dir) {
-  if (fs.existsSync(dir)) {
-    const files = fs.readdirSync(dir);
-    files.forEach(file => fs.unlinkSync(`${dir}/${file}`));
-    fs.rmdirSync(dir);
-  }
-}
 
 describe('e2e Tests hide elements', () => {
-  let latestMockImagesPath;
-
-  beforeEach(() => {
-    latestMockImagesPath = path.resolve(config.latest);
-
-    cleanState(latestMockImagesPath);
-  });
-
   it('hides elements using zero opacity', () => {
     let exitCode = 0;
 

e2eTests/chrome/removeElements/removeElements.test.js

@@ -1,27 +1,8 @@
 /* globals expect */
 
 import { execSync } from 'child_process';
-import path from 'path';
-import fs from 'fs';
-import config from './removeElementsConfig';
-
-function cleanState(dir) {
-  if (fs.existsSync(dir)) {
-    const files = fs.readdirSync(dir);
-    files.forEach(file => fs.unlinkSync(`${dir}/${file}`));
-    fs.rmdirSync(dir);
-  }
-}
 
 describe('e2e Tests remove elements', () => {
-  let latestMockImagesPath;
-
-  beforeEach(() => {
-    latestMockImagesPath = path.resolve(config.latest);
-
-    cleanState(latestMockImagesPath);
-  });
-
   it('remove elements from webpage and compares', () => {
     let exitCode = 0;
 

e2eTests/chrome/snap/snapLocal.test.js

@@ -10,12 +10,6 @@ describe('e2e Tests taking snaps locally', () => {
 
   beforeEach(() => {
     dirPath = path.resolve(config.latest);
-
-    if (fs.existsSync(dirPath)) {
-      const files = fs.readdirSync(dirPath);
-      files.forEach(file => fs.unlinkSync(`${dirPath}/${file}`));
-      fs.rmdirSync(dirPath);
-    }
   });
 
   it('should successfully take a snapshot', async () => {

e2eTests/docker-compose.yml

@@ -2,13 +2,13 @@ version: '3'
 
 services:
   hub:
-    image: selenium/hub:3.141
+    image: selenium/hub
     ports:
       - "4444:4444"
     networks:
       - selenium-grid
   firefox:
-    image: selenium/node-firefox:3.141
+    image: selenium/node-firefox
     environment:
       - NODE_MAX_INSTANCES=2
       - NODE_MAX_SESSION=2
@@ -21,7 +21,7 @@ services:
     depends_on:
       - hub
   chrome:
-    image: selenium/node-chrome:3.141
+    image: selenium/node-chrome
     environment:
       - NODE_MAX_INSTANCES=2
       - NODE_MAX_SESSION=2
@@ -48,6 +48,8 @@ services:
       - chrome
     networks:
       - selenium-grid
+    volumes:
+      - ./:/home/node/app/e2eTests/
 
   ayespy_report:
     image: nginx

e2eTests/firefox/hideElements/hideElements.test.js

@@ -1,27 +1,8 @@
 /* globals expect */
 
 import { execSync } from 'child_process';
-import path from 'path';
-import fs from 'fs';
-import config from './hideElementsConfig';
-
-function cleanState(dir) {
-  if (fs.existsSync(dir)) {
-    const files = fs.readdirSync(dir);
-    files.forEach(file => fs.unlinkSync(`${dir}/${file}`));
-    fs.rmdirSync(dir);
-  }
-}
 
 describe('e2e Tests hide elements', () => {
-  let latestMockImagesPath;
-
-  beforeEach(() => {
-    latestMockImagesPath = path.resolve(config.latest);
-
-    cleanState(latestMockImagesPath);
-  });
-
   it('hides elements using zero opacity', () => {
     let exitCode = 0;
 

e2eTests/firefox/removeElements/removeElements.test.js

@@ -1,27 +1,8 @@
 /* globals expect */
 
 import { execSync } from 'child_process';
-import path from 'path';
-import fs from 'fs';
-import config from './removeElementsConfig';
-
-function cleanState(dir) {
-  if (fs.existsSync(dir)) {
-    const files = fs.readdirSync(dir);
-    files.forEach(file => fs.unlinkSync(`${dir}/${file}`));
-    fs.rmdirSync(dir);
-  }
-}
 
 describe('e2e Tests remove elements', () => {
-  let latestMockImagesPath;
-
-  beforeEach(() => {
-    latestMockImagesPath = path.resolve(config.latest);
-
-    cleanState(latestMockImagesPath);
-  });
-
   it('remove elements from webpage and compares', () => {
     let exitCode = 0;
 

e2eTests/firefox/snap/snapLocal.test.js

@@ -10,12 +10,6 @@ describe('e2e Tests taking snaps locally', () => {
 
   beforeEach(() => {
     dirPath = path.resolve(config.latest);
-
-    if (fs.existsSync(dirPath)) {
-      const files = fs.readdirSync(dirPath);
-      files.forEach(file => fs.unlinkSync(`${dirPath}/${file}`));
-      fs.rmdirSync(dirPath);
-    }
   });
 
   it('should successfully take a snapshot', async () => {

e2eTests/generic/gridError/gridErrorSuiteScript.test.js

@@ -0,0 +1,36 @@
+/* globals expect */
+
+import path from 'path';
+import fs from 'fs';
+import { execSync } from 'child_process';
+import config from './gridErrorSuiteScriptConfig';
+
+describe('e2e Tests running gridErrorSuiteScript', () => {
+  let dirPath;
+
+  beforeEach(() => {
+    dirPath = path.resolve(config.latest);
+
+    if (fs.existsSync(dirPath)) {
+      const files = fs.readdirSync(dirPath);
+      files.forEach(file => fs.unlinkSync(`${dirPath}/${file}`));
+      fs.rmdirSync(dirPath);
+    }
+  });
+
+  it('throw exception and no snapshot is taken', async () => {
+    let exitCode = 0;
+    let stdout;
+    try {
+      stdout = execSync(
+        'node ./lib/bin/run.js snap --browser chrome --config e2eTests/generic/gridError/gridErrorSuiteScriptConfig.json'
+      ).toString();
+      //pipe stdout to Jest console
+      console.log(stdout);
+    } catch (error) {
+      exitCode = error.status;
+    }
+
+    expect(exitCode).toEqual(1);
+  });
+});

e2eTests/generic/gridError/gridErrorSuiteScriptConfig.json

@@ -0,0 +1,15 @@
+{
+  "gridUrl": "",
+  "baseline": "./e2eTests/generic/gridError/baseline",
+  "latest": "./e2eTests/generic/gridError/latest",
+  "generatedDiffs": "./e2eTests/generic/gridError/generatedDiffs",
+  "report": "./e2eTests/generic/gridError/reports",
+  "remoteBucketName": "aye-spy",
+  "remoteRegion": "eu-west-1",
+  "scenarios": [
+    {
+      "url": "http://ayespy_report:4000",
+      "label": "image",
+      "viewports": [{"height": 2400, "width": 1024, "label": "large"}]
+    }]
+}

e2eTests/generic/nonExistScript/nonExistScriptSuiteScript.test.js

@@ -0,0 +1,36 @@
+/* globals expect */
+
+import path from 'path';
+import fs from 'fs';
+import { execSync } from 'child_process';
+import config from './nonExistScriptSuiteScriptConfig';
+
+describe('e2e Tests running nonExistScriptSuiteScript', () => {
+  let dirPath;
+
+  beforeEach(() => {
+    dirPath = path.resolve(config.latest);
+
+    if (fs.existsSync(dirPath)) {
+      const files = fs.readdirSync(dirPath);
+      files.forEach(file => fs.unlinkSync(`${dirPath}/${file}`));
+      fs.rmdirSync(dirPath);
+    }
+  });
+
+  it('should successfully run the snap with with non exist script', async () => {
+    let exitCode = 0;
+    let stdout;
+    try {
+      stdout = execSync(
+        'node ./lib/bin/run.js snap --browser chrome --config e2eTests/generic/nonExistScript/nonExistScriptSuiteScriptConfig.json'
+      ).toString();
+      //pipe stdout to Jest console
+      console.log(stdout);
+    } catch (error) {
+      exitCode = error.status;
+    }
+
+    expect(exitCode).toEqual(0);
+  });
+});

e2eTests/generic/nonExistScript/nonExistScriptSuiteScriptConfig.json

@@ -0,0 +1,16 @@
+{
+  "gridUrl": "http://hub:4444/wd/hub",
+  "baseline": "./e2eTests/generic/nonExistScript/baseline",
+  "latest": "./e2eTests/generic/nonExistScript/latest",
+  "generatedDiffs": "./e2eTests/generic/nonExistScript/generatedDiffs",
+  "report": "./e2eTests/generic/nonExistScript/reports",
+  "remoteBucketName": "aye-spy",
+  "remoteRegion": "eu-west-1",
+  "scenarios": [
+    {
+      "url": "http://ayespy_report:4000",
+      "label": "image",
+      "onReadyScript": "./nonExistScript.js",
+      "viewports": [{"height": 2400, "width": 1024, "label": "large"}]
+    }]
+}

src/scenarioValidator.test.js

@@ -0,0 +1,59 @@
+/* globals expect */
+import scenarioValidator from './scenarioValidator';
+
+describe('Validate Scenario', () => {
+  it('Valid Scenario', () => {
+    const scenario = {
+      url: 'http://lol.co.uk/',
+      label: 'scenario-valid-viewport',
+      viewports: [{ width: 1024, height: 800, label: 'large' }]
+    };
+
+    expect(() => scenarioValidator(scenario)).not.toThrow();
+  });
+  it('Scenario missing height', () => {
+    const scenario = {
+      url: 'http://lol.co.uk/',
+      label: 'scenario-height-missing',
+      viewports: [{ width: 1024, label: 'large' }]
+    };
+    expect(() => scenarioValidator(scenario)).toThrow(
+      'scenario-height-missing has no height set'
+    );
+  });
+
+  it('Scenario missing width', () => {
+    const scenario = {
+      url: 'http://lol.co.uk/',
+      label: 'scenario-width-missing',
+      viewports: [{ height: 1024, label: 'large' }]
+    };
+
+    expect(() => scenarioValidator(scenario)).toThrow(
+      'scenario-width-missing has no width set'
+    );
+  });
+
+  it('Scenario missing label', () => {
+    const scenario = {
+      url: 'http://lol.co.uk/',
+      label: 'scenario-label-missing',
+      viewports: [{ height: 1024, width: 800 }]
+    };
+
+    expect(() => scenarioValidator(scenario)).toThrow(
+      'scenario-label-missing has no label set'
+    );
+  });
+
+  it('Scenario missing viewport', () => {
+    const scenario = {
+      url: 'http://lol.co.uk/',
+      label: 'scenario-viewport-missing'
+    };
+
+    expect(() => scenarioValidator(scenario)).toThrow(
+      'scenario-viewport-missing has no viewports array defined'
+    );
+  });
+});