Newfeature/layout comparator with treshold

CHANGELOG.md

@@ -10,6 +10,7 @@ All notable changes to AET will be documented in this file.
 ## Unreleased
 **List of changes that are finished but not yet released in any final version.**
 
+- [PR-293](https://github.com/Cognifide/aet/pull/293) Added error treshold in pixels and percentages for screen comparator
 - [PR-289](https://github.com/Cognifide/aet/pull/289) User now stays on the same tab while navigating between URLs
 - [PR-271](https://github.com/Cognifide/aet/pull/271) Added possibility to override name parameter from the aet client
 - [PR-268](https://github.com/Cognifide/aet/pull/268) Bobcat upgrade to version 1.4.0

api/communication-api/src/main/java/com/cognifide/aet/communication/api/metadata/ComparatorStepResult.java

@@ -51,6 +51,7 @@ public enum Status {
     FAILED,
     WARNING,
     REBASED,
-    PROCESSING_ERROR
+    PROCESSING_ERROR,
+    CONDITIONALLY_PASSED,
   }
 }

core/jobs/src/main/java/com/cognifide/aet/job/common/comparators/layout/LayoutComparator.java

@@ -16,6 +16,8 @@
 package com.cognifide.aet.job.common.comparators.layout;
 
 import com.cognifide.aet.communication.api.metadata.ComparatorStepResult;
+import com.cognifide.aet.communication.api.metadata.ComparatorStepResult.Status;
+import com.cognifide.aet.job.api.ParametersValidator;
 import com.cognifide.aet.job.api.comparator.ComparatorJob;
 import com.cognifide.aet.job.api.comparator.ComparatorProperties;
 import com.cognifide.aet.job.api.exceptions.ParametersException;
@@ -41,10 +43,18 @@ public class LayoutComparator implements ComparatorJob {
 
   public static final String CONTENT_TYPE = "image/png";
 
+  public static final String PERCENTAGE_THRESHOLD_PARAM = "percentageThreshold";
+
+  public static final String PIXEL_THRESHOLD_PARAM = "pixelThreshold";
+
   private final ComparatorProperties properties;
 
   private final ArtifactsDAO artifactsDAO;
 
+  private Integer pixelThreshold;
+
+  private Double percentageThreshold;
+
   LayoutComparator(ComparatorProperties comparatorProperties, ArtifactsDAO artifactsDAO) {
     this.properties = comparatorProperties;
     this.artifactsDAO = artifactsDAO;
@@ -67,6 +77,7 @@ public ComparatorStepResult compare() throws ProcessingException {
         BufferedImage collectedImg = ImageIO.read(collectedArtifact);
         imageComparisonResult = ImageComparison.compare(patternImg, collectedImg);
         stepResult = saveArtifacts(imageComparisonResult);
+
       } catch (IOException e) {
         throw new ProcessingException("Error while obtaining artifacts!", e);
       }
@@ -84,7 +95,7 @@ private boolean areInputsIdentical(ArtifactsDAO artifactsDAO, ComparatorProperti
   private ComparatorStepResult saveArtifacts(ImageComparisonResult imageComparisonResult)
       throws ProcessingException {
     final ComparatorStepResult result;
-    if (imageComparisonResult.isMatch()) {
+    if (isMaskWithoutDifference(imageComparisonResult)) {
       result = getPassedStepResult();
     } else {
       InputStream mask = null;
@@ -93,19 +104,13 @@ private ComparatorStepResult saveArtifacts(ImageComparisonResult imageComparison
         mask = new ByteArrayInputStream(baos.toByteArray());
         String maskArtifactId = artifactsDAO.saveArtifact(properties, mask, CONTENT_TYPE);
 
-        result = new ComparatorStepResult(maskArtifactId, ComparatorStepResult.Status.FAILED, true);
-
-        result.addData("heightDifference",
-            Integer.toString(imageComparisonResult.getHeightDifference()));
-        result.addData("widthDifference",
-            Integer.toString(imageComparisonResult.getWidthDifference()));
-        result.addData("pixelDifference",
-            Integer.toString(imageComparisonResult.getPixelDifferenceCount()));
-        result.addData("patternTimestamp", Long.toString(
-            artifactsDAO.getArtifactUploadDate(properties, properties.getPatternId()).getTime()));
-        result.addData("collectTimestamp", Long.toString(
-            artifactsDAO.getArtifactUploadDate(properties, properties.getCollectedId())
-                .getTime()));
+        if (hasMaskThresholdWithAcceptableDifference(imageComparisonResult)) {
+          result = new ComparatorStepResult(maskArtifactId, Status.CONDITIONALLY_PASSED, true);
+        } else {
+          result = new ComparatorStepResult(maskArtifactId, Status.FAILED, true);
+        }
+        addPixelDifferenceDataToResult(result, imageComparisonResult);
+        addTimestampToResult(result);
       } catch (Exception e) {
         throw new ProcessingException(e.getMessage(), e);
       } finally {
@@ -116,18 +121,77 @@ private ComparatorStepResult saveArtifacts(ImageComparisonResult imageComparison
     return result;
   }
 
-  private ComparatorStepResult getPassedStepResult() {
-    ComparatorStepResult result = new ComparatorStepResult(null, ComparatorStepResult.Status.PASSED,
-        false);
+  boolean hasMaskThresholdWithAcceptableDifference(ImageComparisonResult mask) {
+    if (pixelThreshold != null && percentageThreshold != null) {
+      return isAcceptablePixelChange(mask) && this.isAcceptablePercentageChange(mask);
+    } else if (pixelThreshold != null) {
+      return isAcceptablePixelChange(mask);
+    } else if (percentageThreshold != null) {
+      return isAcceptablePercentageChange(mask);
+    }
+    return false;
+  }
+
+
+  public void setPixelThreshold(Integer pixelThreshold) {
+    this.pixelThreshold = pixelThreshold;
+  }
+
+  public void setPercentageThreshold(Double percentageThreshold) {
+    this.percentageThreshold = percentageThreshold;
+  }
+
+  private boolean isAcceptablePixelChange(ImageComparisonResult mask) {
+    return mask.getPixelDifferenceCount() <= this.pixelThreshold;
+  }
+
+  private boolean isAcceptablePercentageChange(ImageComparisonResult mask) {
+    return mask.getPercentagePixelDifference() <= this.percentageThreshold;
+  }
+
+  private boolean isMaskWithoutDifference(ImageComparisonResult mask) {
+    return mask.getHeightDifference() == 0 && mask.getWidthDifference() == 0
+        && mask.getPixelDifferenceCount() == 0;
+  }
+
+  private void addPixelDifferenceDataToResult(ComparatorStepResult result,
+      ImageComparisonResult imageComparisonResult) {
+    result.addData("heightDifference",
+        Integer.toString(imageComparisonResult.getHeightDifference()));
+    result.addData("widthDifference",
+        Integer.toString(imageComparisonResult.getWidthDifference()));
+    result.addData("pixelDifference",
+        Integer.toString(imageComparisonResult.getPixelDifferenceCount()));
+    result.addData("percentagePixelDifference",
+        Double.toString(imageComparisonResult.getPercentagePixelDifference()));
+  }
+
+  private void addTimestampToResult(ComparatorStepResult result) {
     result.addData("patternTimestamp", Long.toString(
         artifactsDAO.getArtifactUploadDate(properties, properties.getPatternId()).getTime()));
     result.addData("collectTimestamp", Long.toString(System.currentTimeMillis()));
+  }
+
+  private ComparatorStepResult getPassedStepResult() {
+    ComparatorStepResult result = new ComparatorStepResult(null, ComparatorStepResult.Status.PASSED,
+        false);
+    addTimestampToResult(result);
     return result;
   }
 
   @Override
   public void setParameters(Map<String, String> params) throws ParametersException {
-    // no parameters needed
+    if (params.containsKey(PERCENTAGE_THRESHOLD_PARAM)) {
+      setPercentageThreshold(Double.valueOf(params.get(PERCENTAGE_THRESHOLD_PARAM)));
+      ParametersValidator
+          .checkRange(percentageThreshold.intValue(), 0, 100,
+              "Percentage threshold should be a decimal value between 0 and 100");
+    }
+    if (params.containsKey(PIXEL_THRESHOLD_PARAM)) {
+      setPixelThreshold(Integer.valueOf(params.get(PIXEL_THRESHOLD_PARAM)));
+      ParametersValidator.checkParameter(pixelThreshold >= 0,
+          "Pixel threshold should be greater or equal to 0");
+    }
   }
 
 }

core/jobs/src/main/java/com/cognifide/aet/job/common/comparators/layout/utils/ImageComparisonResult.java

@@ -19,12 +19,16 @@
 
 public class ImageComparisonResult {
 
+  private int pixelCount;
+
   private int pixelDifferenceCount;
 
   private int heightDifference;
 
   private int widthDifference;
 
+  private double percentagePixelDifference;
+
   private BufferedImage resultImage;
 
   public ImageComparisonResult() {
@@ -37,11 +41,8 @@ public ImageComparisonResult(final int pixelDifferenceCount, final int widthDiff
     this.heightDifference = heightDifference;
     this.widthDifference = widthDifference;
     this.resultImage = resultImage;
-  }
-
-  public boolean isMatch() {
-    return this.pixelDifferenceCount == 0 && this.heightDifference == 0
-        && this.widthDifference == 0;
+    this.pixelCount = resultImage.getHeight() * resultImage.getWidth();
+    this.percentagePixelDifference = 100 * this.pixelDifferenceCount / (double) this.pixelCount;
   }
 
   public int getPixelDifferenceCount() {
@@ -59,4 +60,9 @@ public int getWidthDifference() {
   public BufferedImage getResultImage() {
     return resultImage;
   }
+
+  public double getPercentagePixelDifference() {
+    return percentagePixelDifference;
+  }
+
 }

core/jobs/src/test/java/com/cognifide/aet/job/common/comparators/layout/LayoutComparatorTest.java

@@ -0,0 +1,143 @@
+/**
+ * AET
+ *
+ * Copyright (C) 2013 Cognifide Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.cognifide.aet.job.common.comparators.layout;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.when;
+
+import com.cognifide.aet.job.api.comparator.ComparatorProperties;
+import com.cognifide.aet.job.common.comparators.layout.utils.ImageComparisonResult;
+import com.cognifide.aet.vs.ArtifactsDAO;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class LayoutComparatorTest {
+
+  @Mock
+  private ComparatorProperties comparatorProperties;
+
+  @Mock
+  private ArtifactsDAO artifactsDAO;
+
+  @Mock
+  private ImageComparisonResult imageComparisonResult;
+
+  private LayoutComparator layoutComparator;
+
+  @Before
+  public void setUp() {
+    //given
+    this.layoutComparator = new LayoutComparator(this.comparatorProperties, this.artifactsDAO);
+  }
+
+  @Test
+  public void hasMaskThresholdWithAcceptableDifference_withoutThreshold_expectFalse() {
+    //when
+    when(imageComparisonResult.getPercentagePixelDifference()).thenReturn(12.567);
+    when(imageComparisonResult.getPixelDifferenceCount()).thenReturn(300);
+
+    //then
+    assertThat(
+        this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult),
+        is(false));
+  }
+
+  @Test
+  public void hasMaskThresholdWithAcceptableDifference_withThreshold_expectFalse() {
+    //when
+    when(imageComparisonResult.getPercentagePixelDifference()).thenReturn(12.567);
+    when(imageComparisonResult.getPixelDifferenceCount()).thenReturn(300);
+
+    this.layoutComparator.setPixelThreshold(299);
+    this.layoutComparator.setPercentageThreshold(null);
+
+    //then
+    assertThat(this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult),
+        is(false));
+
+    //when
+    this.layoutComparator.setPixelThreshold(null);
+    this.layoutComparator.setPercentageThreshold(12.566);
+
+    //then
+    assertThat(this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult),
+        is(false));
+  }
+
+  @Test
+  public void hasMaskThresholdWithAcceptableDifference_withThreshold_expectTrue() {
+    //when
+    when(imageComparisonResult.getPercentagePixelDifference()).thenReturn(12.567);
+    when(imageComparisonResult.getPixelDifferenceCount()).thenReturn(300);
+
+    this.layoutComparator.setPixelThreshold(300);
+    this.layoutComparator.setPercentageThreshold(null);
+
+    //then
+    assertThat(this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult),
+        is(true));
+
+    //when
+    this.layoutComparator.setPixelThreshold(null);
+    this.layoutComparator.setPercentageThreshold(12.567);
+
+    //then
+    assertThat(this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult),
+        is(true));
+  }
+
+  @Test
+  public void hasMaskThresholdWithAcceptableDifference_withBothThreshold_expectFalse() {
+    //when
+    when(imageComparisonResult.getPercentagePixelDifference()).thenReturn(12.567);
+    when(imageComparisonResult.getPixelDifferenceCount()).thenReturn(300);
+
+    this.layoutComparator.setPixelThreshold(299);
+    this.layoutComparator.setPercentageThreshold(30.0);
+
+    //then
+    assertThat(this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult),
+        is(false));
+
+    //when
+    this.layoutComparator.setPixelThreshold(301);
+    this.layoutComparator.setPercentageThreshold(12.566);
+
+    //then
+    assertThat(this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult),
+        is(false));
+  }
+
+  @Test
+  public void hasMaskThresholdWithAcceptableDifference_withBothThreshold_expectTrue() {
+    //when
+    when(imageComparisonResult.getPercentagePixelDifference()).thenReturn(12.567);
+    when(imageComparisonResult.getPixelDifferenceCount()).thenReturn(300);
+
+    this.layoutComparator.setPixelThreshold(300);
+    this.layoutComparator.setPercentageThreshold(12.567);
+
+    //then
+    assertThat(this.layoutComparator.hasMaskThresholdWithAcceptableDifference(imageComparisonResult),
+        is(true));
+  }
+
+}

core/jobs/src/test/java/com/cognifide/aet/job/common/comparators/layout/utils/ImageComparisonTest.java

@@ -45,7 +45,7 @@ public void test_sameScreenshot_expectNoDifferencesInResultAndTransparentMask()
       // when
       ImageComparisonResult imageComparisonResult = ImageComparison.compare(pattern, sample);
       // then
-      assertThat(imageComparisonResult.isMatch(), is(true));
+      assertThat(imageComparisonResult.getPercentagePixelDifference(), is(0.0));
       assertThat(imageComparisonResult.getHeightDifference(), is(0));
       assertThat(imageComparisonResult.getWidthDifference(), is(0));
       assertThat(imageComparisonResult.getPixelDifferenceCount(), is(0));
@@ -76,7 +76,7 @@ public void testCompare_different() throws Exception {
       // when
       ImageComparisonResult imageComparisonResult = ImageComparison.compare(pattern, sample);
       // then
-      assertThat(imageComparisonResult.isMatch(), is(false));
+      assertThat(imageComparisonResult.getPercentagePixelDifference(), is(0.7435369480668039));
       assertThat(imageComparisonResult.getHeightDifference(), is(0));
       assertThat(imageComparisonResult.getWidthDifference(), is(0));
       assertThat(imageComparisonResult.getPixelDifferenceCount(), is(15600));
@@ -110,7 +110,7 @@ public void compare_differentSizeScreenshots_expectSizeDifferenceMarkedWithYello
       // when
       ImageComparisonResult imageComparisonResult = ImageComparison.compare(pattern, sample);
       // then
-      assertThat(imageComparisonResult.isMatch(), is(false));
+      assertThat(imageComparisonResult.getPercentagePixelDifference(), is(59.99583333333333));
       assertThat(imageComparisonResult.getHeightDifference(), is(100));
       assertThat(imageComparisonResult.getWidthDifference(), is(20));
       assertThat(imageComparisonResult.getPixelDifferenceCount(), is(14399));

documentation/src/main/wiki/LayoutComparator.md

@@ -10,7 +10,12 @@ Resource name: screen
 
 ##### Parameters
 
-No parameters
+| Parameter | Value | Description | Mandatory |
+| --------- | ----- | ----------- | --------- |
+| `pixelThreshold` | int (equal or greater than 0) | The value to set the error threshold in pixels e.g if difference between photos is smaller or equal to `pixelThreshold`, the test will pass. In case of difference is bigger than `pixelThreshold`, the test will fail. | no |
+| `percentageThreshold` | double (between 0 and 100) | It works as `pixelThreshold` but values are in percentages | no |
+
+When you provide `pixelThreshold` and `percentageThreshold` test will pass only if pixel difference is smaller or equal than `pixelThreshold` and percentage difference is smaller or equal than `percentageThreshold`.
 
 ##### Example Usage