Faster, collision-free coverage instrumentation

fuzz/pom.xml

@@ -50,6 +50,11 @@
             <artifactId>picocli</artifactId>
             <version>4.0.4</version>
         </dependency>
+        <dependency>
+            <groupId>org.eclipse.collections</groupId>
+            <artifactId>eclipse-collections</artifactId>
+            <version>10.4.0</version>
+        </dependency>
     </dependencies>
 
 

fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/afl/PerfFuzzGuidance.java

@@ -49,6 +49,7 @@
 import edu.berkeley.cs.jqf.instrument.tracing.events.ReadEvent;
 import edu.berkeley.cs.jqf.instrument.tracing.events.ReturnEvent;
 import edu.berkeley.cs.jqf.instrument.tracing.events.TraceEvent;
+import org.eclipse.collections.api.list.primitive.IntList;
 
 /**
  * A front-end that uses AFL for increasing performance counters
@@ -303,15 +304,13 @@ protected int getAyclicExecutionContextForEvent(TraceEvent e) {
      *                     one positive integer for each memory access
      * @return     the redundancy score
      */
-    public static double computeRedundancyScore(Collection<Integer> accessCounts) {
+    public static double computeRedundancyScore(IntList accessCounts) {
         double numCounts = accessCounts.size();
         if (numCounts == 0) {
             return 0.0;
         }
         double sumCounts = 0.0;
-        for (int count : accessCounts) {
-            sumCounts += count;
-        }
+        sumCounts = accessCounts.sum();
         double averageCounts = sumCounts / numCounts;
         double score = (averageCounts - 1)*(numCounts - 1)/sumCounts;
 

fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ExecutionIndexingGuidance.java

@@ -54,6 +54,9 @@
 import edu.berkeley.cs.jqf.instrument.tracing.SingleSnoop;
 import edu.berkeley.cs.jqf.instrument.tracing.events.CallEvent;
 import edu.berkeley.cs.jqf.instrument.tracing.events.TraceEvent;
+import org.eclipse.collections.api.iterator.IntIterator;
+import org.eclipse.collections.api.set.primitive.IntSet;
+import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet;
 
 /**
  * A guidance that represents inputs as maps from
@@ -260,7 +263,9 @@ public void handleResult(Result result, Throwable error) throws GuidanceExceptio
                         savedInputs.set(otherIdx, currentInput);
 
                         // Second, update responsibilities
-                        for (Object b : otherInput.responsibilities) {
+                        IntIterator otherResponsibilitiesIter = otherInput.responsibilities.intIterator();
+                        while(otherResponsibilitiesIter.hasNext()){
+                            int b = otherResponsibilitiesIter.next();
                             // Subsume responsibility
                             // infoLog("-- Stealing responsibility for %s from old input %d", b, otherIdx);
                             // We are now responsible
@@ -272,7 +277,7 @@ public void handleResult(Result result, Throwable error) throws GuidanceExceptio
                         // Third, store basic book-keeping data
                         currentInput.id = otherIdx;
                         currentInput.saveFile = otherInput.saveFile;
-                        currentInput.coverage = new Coverage(runCoverage);
+                        currentInput.coverage = runCoverage.copy();
                         currentInput.nonZeroCoverage = runCoverage.getNonZeroCount();
                         currentInput.offspring = 0;
                         savedInputs.get(currentParentInputIdx).offspring += 1;
@@ -296,7 +301,7 @@ public void handleResult(Result result, Throwable error) throws GuidanceExceptio
 
     /** Saves an interesting input to the queue. */
     @Override
-    protected void saveCurrentInput(Set<Object> responsibilities, String why) throws IOException {
+    protected void saveCurrentInput(IntHashSet responsibilities, String why) throws IOException {
         // First, do same as Zest
         super.saveCurrentInput(responsibilities, why);
 

fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ZestGuidance.java

@@ -40,11 +40,12 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintWriter;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.time.Duration;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Date;
 import java.util.Deque;
 import java.util.HashMap;
@@ -62,8 +63,15 @@
 import edu.berkeley.cs.jqf.fuzz.guidance.Result;
 import edu.berkeley.cs.jqf.fuzz.guidance.TimeoutException;
 import edu.berkeley.cs.jqf.fuzz.util.Coverage;
+import edu.berkeley.cs.jqf.fuzz.util.CoverageFactory;
+import edu.berkeley.cs.jqf.fuzz.util.ICoverage;
 import edu.berkeley.cs.jqf.fuzz.util.IOUtils;
+import edu.berkeley.cs.jqf.instrument.tracing.FastCoverageSnoop;
 import edu.berkeley.cs.jqf.instrument.tracing.events.TraceEvent;
+import janala.instrument.FastCoverageListener;
+import org.eclipse.collections.api.iterator.IntIterator;
+import org.eclipse.collections.api.list.primitive.IntList;
+import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet;
 
 import static java.lang.Math.ceil;
 import static java.lang.Math.log;
@@ -143,13 +151,13 @@ public class ZestGuidance implements Guidance {
     protected int numSavedInputs = 0;
 
     /** Coverage statistics for a single run. */
-    protected Coverage runCoverage = new Coverage();
+    protected ICoverage runCoverage = CoverageFactory.newInstance();
 
     /** Cumulative coverage statistics. */
-    protected Coverage totalCoverage = new Coverage();
+    protected ICoverage totalCoverage = CoverageFactory.newInstance();
 
     /** Cumulative coverage for valid inputs. */
-    protected Coverage validCoverage = new Coverage();
+    protected ICoverage validCoverage = CoverageFactory.newInstance();
 
     /** The maximum number of keys covered by any single input found so far. */
     protected int maxCoverage = 0;
@@ -158,7 +166,7 @@ public class ZestGuidance implements Guidance {
     protected Map<Object, Input> responsibleInputs = new HashMap<>(totalCoverage.size());
 
     /** The set of unique failures found so far. */
-    protected Set<List<StackTraceElement>> uniqueFailures = new HashSet<>();
+    protected Set<String> uniqueFailures = new HashSet<>();
 
     /** save crash to specific location (should be used with EXIT_ON_CRASH) **/
     protected final String EXACT_CRASH_PATH = System.getProperty("jqf.ei.EXACT_CRASH_PATH");
@@ -278,6 +286,10 @@ public ZestGuidance(String testName, Duration duration, Long trials, File output
         this.validityFuzzing = !Boolean.getBoolean("jqf.ei.DISABLE_VALIDITY_FUZZING");
         prepareOutputDirectory();
 
+        if(this.runCoverage instanceof FastCoverageListener){
+            FastCoverageSnoop.setFastCoverageListener((FastCoverageListener) this.runCoverage);
+        }
+
         // Try to parse the single-run timeout
         String timeout = System.getProperty("jqf.ei.TIMEOUT");
         if (timeout != null && !timeout.isEmpty()) {
@@ -729,7 +741,7 @@ public void handleResult(Result result, Throwable error) throws GuidanceExceptio
                 // Newly covered branches are always included.
                 // Existing branches *may* be included, depending on the heuristics used.
                 // A valid input will steal responsibility from invalid inputs
-                Set<Object> responsibilities = computeResponsibilities(valid);
+                IntHashSet responsibilities = computeResponsibilities(valid);
 
                 // Determine if this input should be saved
                 List<String> savingCriteriaSatisfied = checkSavingCriteriaSatisfied(result);
@@ -775,7 +787,7 @@ public void handleResult(Result result, Throwable error) throws GuidanceExceptio
                 }
 
                 // Attempt to add this to the set of unique failures
-                if (uniqueFailures.add(Arrays.asList(rootCause.getStackTrace()))) {
+                if (uniqueFailures.add(failureDigest(rootCause.getStackTrace()))) {
 
                     // Trim input (remove unused keys)
                     currentInput.gc();
@@ -862,18 +874,18 @@ protected List<String> checkSavingCriteriaSatisfied(Result result) {
 
 
     // Compute a set of branches for which the current input may assume responsibility
-    protected Set<Object> computeResponsibilities(boolean valid) {
-        Set<Object> result = new HashSet<>();
+    protected IntHashSet computeResponsibilities(boolean valid) {
+        IntHashSet result = new IntHashSet();
 
         // This input is responsible for all new coverage
-        Collection<?> newCoverage = runCoverage.computeNewCoverage(totalCoverage);
+        IntList newCoverage = runCoverage.computeNewCoverage(totalCoverage);
         if (newCoverage.size() > 0) {
             result.addAll(newCoverage);
         }
 
         // If valid, this input is responsible for all new valid coverage
         if (valid) {
-            Collection<?> newValidCoverage = runCoverage.computeNewCoverage(validCoverage);
+            IntList newValidCoverage = runCoverage.computeNewCoverage(validCoverage);
             if (newValidCoverage.size() > 0) {
                 result.addAll(newValidCoverage);
             }
@@ -883,12 +895,13 @@ protected Set<Object> computeResponsibilities(boolean valid) {
         if (STEAL_RESPONSIBILITY) {
             int currentNonZeroCoverage = runCoverage.getNonZeroCount();
             int currentInputSize = currentInput.size();
-            Set<?> covered = new HashSet<>(runCoverage.getCovered());
+            IntHashSet covered = new IntHashSet();
+            covered.addAll(runCoverage.getCovered());
 
             // Search for a candidate to steal responsibility from
             candidate_search:
             for (Input candidate : savedInputs) {
-                Set<?> responsibilities = candidate.responsibilities;
+                IntHashSet responsibilities = candidate.responsibilities;
 
                 // Candidates with no responsibility are not interesting
                 if (responsibilities.isEmpty()) {
@@ -903,7 +916,9 @@ protected Set<Object> computeResponsibilities(boolean valid) {
                                 currentInputSize < candidate.size())) {
 
                     // Check if we can steal all responsibilities from candidate
-                    for (Object b : responsibilities) {
+                    IntIterator iter = responsibilities.intIterator();
+                    while(iter.hasNext()){
+                        int b = iter.next();
                         if (covered.contains(b) == false) {
                             // Cannot steal if this input does not cover something
                             // that the candidate is responsible for
@@ -932,7 +947,7 @@ protected void writeCurrentInputToFile(File saveFile) throws IOException {
     }
 
     /* Saves an interesting input to the queue. */
-    protected void saveCurrentInput(Set<Object> responsibilities, String why) throws IOException {
+    protected void saveCurrentInput(IntHashSet responsibilities, String why) throws IOException {
 
         // First, save to disk (note: we issue IDs to everyone, but only write to disk  if valid)
         int newInputIdx = numSavedInputs++;
@@ -953,14 +968,16 @@ protected void saveCurrentInput(Set<Object> responsibilities, String why) throws
         // Third, store basic book-keeping data
         currentInput.id = newInputIdx;
         currentInput.saveFile = saveFile;
-        currentInput.coverage = new Coverage(runCoverage);
+        currentInput.coverage = runCoverage.copy();
         currentInput.nonZeroCoverage = runCoverage.getNonZeroCount();
         currentInput.offspring = 0;
         savedInputs.get(currentParentInputIdx).offspring += 1;
 
         // Fourth, assume responsibility for branches
         currentInput.responsibilities = responsibilities;
-        for (Object b : responsibilities) {
+        IntIterator iter = responsibilities.intIterator();
+        while(iter.hasNext()){
+            int b = iter.next();
             // If there is an old input that is responsible,
             // subsume it
             Input oldResponsible = responsibleInputs.get(b);
@@ -989,12 +1006,15 @@ public Consumer<TraceEvent> generateCallBack(Thread thread) {
     /**
      * Handles a trace event generated during test execution.
      *
+     * Not used by FastNonCollidingCoverage, which does not allocate an
+     * instance of TraceEvent at each branch probe execution.
+     *
      * @param e the trace event to be handled
      */
     protected void handleEvent(TraceEvent e) {
         conditionallySynchronize(multiThreaded, () -> {
             // Collect totalCoverage
-            runCoverage.handleEvent(e);
+            ((Coverage) runCoverage).handleEvent(e);
             // Check for possible timeouts every so often
             if (this.singleRunTimeoutMillis > 0 &&
                     this.runStart != null && (++this.branchCount) % 10_000 == 0) {
@@ -1010,7 +1030,7 @@ protected void handleEvent(TraceEvent e) {
      * Returns a reference to the coverage statistics.
      * @return a reference to the coverage statistics
      */
-    public Coverage getTotalCoverage() {
+    public ICoverage getTotalCoverage() {
         return totalCoverage;
     }
 
@@ -1029,6 +1049,25 @@ protected void conditionallySynchronize(boolean cond, Runnable task) {
         }
     }
 
+    private static MessageDigest sha1;
+
+    private static String failureDigest(StackTraceElement[] stackTrace) {
+        if (sha1 == null) {
+            try {
+                sha1 = MessageDigest.getInstance("SHA-1");
+            } catch (NoSuchAlgorithmException e) {
+                throw new GuidanceException(e);
+            }
+        }
+        byte[] bytes = sha1.digest(Arrays.deepToString(stackTrace).getBytes());
+        StringBuilder sb = new StringBuilder();
+        for (int i = 0; i < bytes.length; i++) {
+            sb.append(Integer.toString((bytes[i] & 0xff) + 0x100, 16)
+                    .substring(1));
+        }
+        return sb.toString();
+    }
+
     /**
      * A candidate or saved test input that maps objects of type K to bytes.
      */
@@ -1061,7 +1100,7 @@ public static abstract class Input<K> implements Iterable<Integer> {
          *
          * <p>This field is null for inputs that are not saved.</p>
          */
-        Coverage coverage = null;
+        ICoverage coverage = null;
 
         /**
          * The number of non-zero elements in `coverage`.
@@ -1093,7 +1132,7 @@ public static abstract class Input<K> implements Iterable<Integer> {
          * in at least some responsibility set. Hence, this list
          * needs to be kept in-sync with {@link #responsibleInputs}.</p>
          */
-        Set<Object> responsibilities = null;
+        IntHashSet responsibilities = null;
 
         /**
          * Create an empty input.

fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/repro/ReproServerGuidance.java

@@ -105,7 +105,7 @@ public void handleResult(Result result, Throwable error){
 
         // Print coverage
         try (PrintWriter out = new PrintWriter(coverageFile)) {
-            coverage.getCovered().stream().sorted().forEach((i) ->
+            coverage.getCovered().toSortedList().forEach((i) ->
                 out.println(String.format("%05d", i))
             );
             out.println(result.toString()); // EOF marker for tools to realize that repro is done

fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Counter.java

@@ -28,6 +28,9 @@
  */
 package edu.berkeley.cs.jqf.fuzz.util;
 
+import org.eclipse.collections.api.list.primitive.IntList;
+import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -183,8 +186,8 @@ public boolean hasNonZeros(){
      *
      * @return a set of indices at which the count is non-zero
      */
-    public Collection<Integer> getNonZeroIndices() {
-        List<Integer> indices = new ArrayList<>(size /2);
+    public IntList getNonZeroIndices() {
+        IntArrayList indices = new IntArrayList(size /2);
         for (int i = 0; i < counts.length; i++) {
             int count = counts[i];
             if (count != 0) {
@@ -199,8 +202,8 @@ public Collection<Integer> getNonZeroIndices() {
      *
      * @return a set of non-zero count values in this counter.
      */
-    public Collection<Integer> getNonZeroValues() {
-        List<Integer> values = new ArrayList<>(size /2);
+    public IntList getNonZeroValues() {
+        IntArrayList values = new IntArrayList(size /2);
         for (int i = 0; i < counts.length; i++) {
             int count = counts[i];
             if (count != 0) {