From 75dcc31576b6d7bd5d765277e7f8753a8189c795 Mon Sep 17 00:00:00 2001 From: Jonathan Bell Date: Tue, 28 Dec 2021 15:19:16 -0500 Subject: [PATCH 1/4] Faster, collision-free coverage instrumentation. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit introduces new coverage collection instrumentation that avoids the Janala trace framework, and instead directly adds coverage probe calls without requiring additional object allocations, and without instrumenting any instructions that don’t have branch probes . This coverage implementation is also collision-free, and can be enabled if desired using the flag -DuseFastNonCollidingCoverageInstrumentation=true --- fuzz/pom.xml | 5 + .../cs/jqf/fuzz/afl/PerfFuzzGuidance.java | 7 +- .../fuzz/ei/ExecutionIndexingGuidance.java | 11 +- .../berkeley/cs/jqf/fuzz/ei/ZestGuidance.java | 56 +++-- .../jqf/fuzz/repro/ReproServerGuidance.java | 2 +- .../berkeley/cs/jqf/fuzz/util/Counter.java | 11 +- .../berkeley/cs/jqf/fuzz/util/Coverage.java | 47 ++-- .../cs/jqf/fuzz/util/CoverageFactory.java | 33 +++ .../fuzz/util/FastNonCollidingCounter.java | 164 ++++++++++++ .../fuzz/util/FastNonCollidingCoverage.java | 234 ++++++++++++++++++ .../berkeley/cs/jqf/fuzz/util/ICoverage.java | 60 +++++ .../cs/jqf/fuzz/util/MapOfCounters.java | 7 +- .../jqf/fuzz/util/NonZeroCachingCounter.java | 24 +- .../cs/jqf/fuzz/afl/RedundancyTest.java | 11 +- .../cs/jqf/fuzz/util/CountersTest.java | 33 ++- .../fuzz/util/NonZeroCachingCountersTest.java | 33 ++- .../instrument/tracing/FastCoverageSnoop.java | 45 ++++ .../main/java/janala/instrument/Config.java | 17 +- .../instrument/FastCoverageListener.java | 5 + .../instrument/FastCoverageMethodAdapter.java | 215 ++++++++++++++++ .../GlobalStateForInstrumentation.java | 8 + .../SnoopInstructionClassAdapter.java | 10 +- .../SnoopInstructionTransformer.java | 2 +- 23 files changed, 953 insertions(+), 87 deletions(-) create mode 100644 fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/CoverageFactory.java create mode 100644 fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/FastNonCollidingCounter.java create mode 100644 fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/FastNonCollidingCoverage.java create mode 100644 fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/ICoverage.java create mode 100644 instrument/src/main/java/edu/berkeley/cs/jqf/instrument/tracing/FastCoverageSnoop.java create mode 100644 instrument/src/main/java/janala/instrument/FastCoverageListener.java create mode 100644 instrument/src/main/java/janala/instrument/FastCoverageMethodAdapter.java diff --git a/fuzz/pom.xml b/fuzz/pom.xml index 6350dfc4c..e1bef3db5 100644 --- a/fuzz/pom.xml +++ b/fuzz/pom.xml @@ -50,6 +50,11 @@ picocli 4.0.4 + + org.eclipse.collections + eclipse-collections + 10.4.0 + diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/afl/PerfFuzzGuidance.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/afl/PerfFuzzGuidance.java index 024c85fe3..52e5e001f 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/afl/PerfFuzzGuidance.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/afl/PerfFuzzGuidance.java @@ -49,6 +49,7 @@ import edu.berkeley.cs.jqf.instrument.tracing.events.ReadEvent; import edu.berkeley.cs.jqf.instrument.tracing.events.ReturnEvent; import edu.berkeley.cs.jqf.instrument.tracing.events.TraceEvent; +import org.eclipse.collections.api.list.primitive.IntList; /** * A front-end that uses AFL for increasing performance counters @@ -303,15 +304,13 @@ protected int getAyclicExecutionContextForEvent(TraceEvent e) { * one positive integer for each memory access * @return the redundancy score */ - public static double computeRedundancyScore(Collection accessCounts) { + public static double computeRedundancyScore(IntList accessCounts) { double numCounts = accessCounts.size(); if (numCounts == 0) { return 0.0; } double sumCounts = 0.0; - for (int count : accessCounts) { - sumCounts += count; - } + sumCounts = accessCounts.sum(); double averageCounts = sumCounts / numCounts; double score = (averageCounts - 1)*(numCounts - 1)/sumCounts; diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ExecutionIndexingGuidance.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ExecutionIndexingGuidance.java index 90e4dab8f..fe8815867 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ExecutionIndexingGuidance.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ExecutionIndexingGuidance.java @@ -54,6 +54,9 @@ import edu.berkeley.cs.jqf.instrument.tracing.SingleSnoop; import edu.berkeley.cs.jqf.instrument.tracing.events.CallEvent; import edu.berkeley.cs.jqf.instrument.tracing.events.TraceEvent; +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; /** * A guidance that represents inputs as maps from @@ -260,7 +263,9 @@ public void handleResult(Result result, Throwable error) throws GuidanceExceptio savedInputs.set(otherIdx, currentInput); // Second, update responsibilities - for (Object b : otherInput.responsibilities) { + IntIterator otherResponsibilitiesIter = otherInput.responsibilities.intIterator(); + while(otherResponsibilitiesIter.hasNext()){ + int b = otherResponsibilitiesIter.next(); // Subsume responsibility // infoLog("-- Stealing responsibility for %s from old input %d", b, otherIdx); // We are now responsible @@ -272,7 +277,7 @@ public void handleResult(Result result, Throwable error) throws GuidanceExceptio // Third, store basic book-keeping data currentInput.id = otherIdx; currentInput.saveFile = otherInput.saveFile; - currentInput.coverage = new Coverage(runCoverage); + currentInput.coverage = runCoverage.copy(); currentInput.nonZeroCoverage = runCoverage.getNonZeroCount(); currentInput.offspring = 0; savedInputs.get(currentParentInputIdx).offspring += 1; @@ -296,7 +301,7 @@ public void handleResult(Result result, Throwable error) throws GuidanceExceptio /** Saves an interesting input to the queue. */ @Override - protected void saveCurrentInput(Set responsibilities, String why) throws IOException { + protected void saveCurrentInput(IntHashSet responsibilities, String why) throws IOException { // First, do same as Zest super.saveCurrentInput(responsibilities, why); diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ZestGuidance.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ZestGuidance.java index 9a575fde6..4ddf193d1 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ZestGuidance.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ZestGuidance.java @@ -44,7 +44,6 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Date; import java.util.Deque; import java.util.HashMap; @@ -62,8 +61,15 @@ import edu.berkeley.cs.jqf.fuzz.guidance.Result; import edu.berkeley.cs.jqf.fuzz.guidance.TimeoutException; import edu.berkeley.cs.jqf.fuzz.util.Coverage; +import edu.berkeley.cs.jqf.fuzz.util.CoverageFactory; +import edu.berkeley.cs.jqf.fuzz.util.ICoverage; import edu.berkeley.cs.jqf.fuzz.util.IOUtils; +import edu.berkeley.cs.jqf.instrument.tracing.FastCoverageSnoop; import edu.berkeley.cs.jqf.instrument.tracing.events.TraceEvent; +import janala.instrument.FastCoverageListener; +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.api.list.primitive.IntList; +import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; import static java.lang.Math.ceil; import static java.lang.Math.log; @@ -143,13 +149,13 @@ public class ZestGuidance implements Guidance { protected int numSavedInputs = 0; /** Coverage statistics for a single run. */ - protected Coverage runCoverage = new Coverage(); + protected ICoverage runCoverage = CoverageFactory.newInstance(); /** Cumulative coverage statistics. */ - protected Coverage totalCoverage = new Coverage(); + protected ICoverage totalCoverage = CoverageFactory.newInstance(); /** Cumulative coverage for valid inputs. */ - protected Coverage validCoverage = new Coverage(); + protected ICoverage validCoverage = CoverageFactory.newInstance(); /** The maximum number of keys covered by any single input found so far. */ protected int maxCoverage = 0; @@ -278,6 +284,10 @@ public ZestGuidance(String testName, Duration duration, Long trials, File output this.validityFuzzing = !Boolean.getBoolean("jqf.ei.DISABLE_VALIDITY_FUZZING"); prepareOutputDirectory(); + if(this.runCoverage instanceof FastCoverageListener){ + FastCoverageSnoop.setFastCoverageListener((FastCoverageListener) this.runCoverage); + } + // Try to parse the single-run timeout String timeout = System.getProperty("jqf.ei.TIMEOUT"); if (timeout != null && !timeout.isEmpty()) { @@ -729,7 +739,7 @@ public void handleResult(Result result, Throwable error) throws GuidanceExceptio // Newly covered branches are always included. // Existing branches *may* be included, depending on the heuristics used. // A valid input will steal responsibility from invalid inputs - Set responsibilities = computeResponsibilities(valid); + IntHashSet responsibilities = computeResponsibilities(valid); // Determine if this input should be saved List savingCriteriaSatisfied = checkSavingCriteriaSatisfied(result); @@ -862,18 +872,18 @@ protected List checkSavingCriteriaSatisfied(Result result) { // Compute a set of branches for which the current input may assume responsibility - protected Set computeResponsibilities(boolean valid) { - Set result = new HashSet<>(); + protected IntHashSet computeResponsibilities(boolean valid) { + IntHashSet result = new IntHashSet(); // This input is responsible for all new coverage - Collection newCoverage = runCoverage.computeNewCoverage(totalCoverage); + IntList newCoverage = runCoverage.computeNewCoverage(totalCoverage); if (newCoverage.size() > 0) { result.addAll(newCoverage); } // If valid, this input is responsible for all new valid coverage if (valid) { - Collection newValidCoverage = runCoverage.computeNewCoverage(validCoverage); + IntList newValidCoverage = runCoverage.computeNewCoverage(validCoverage); if (newValidCoverage.size() > 0) { result.addAll(newValidCoverage); } @@ -883,12 +893,13 @@ protected Set computeResponsibilities(boolean valid) { if (STEAL_RESPONSIBILITY) { int currentNonZeroCoverage = runCoverage.getNonZeroCount(); int currentInputSize = currentInput.size(); - Set covered = new HashSet<>(runCoverage.getCovered()); + IntHashSet covered = new IntHashSet(); + covered.addAll(runCoverage.getCovered()); // Search for a candidate to steal responsibility from candidate_search: for (Input candidate : savedInputs) { - Set responsibilities = candidate.responsibilities; + IntHashSet responsibilities = candidate.responsibilities; // Candidates with no responsibility are not interesting if (responsibilities.isEmpty()) { @@ -903,7 +914,9 @@ protected Set computeResponsibilities(boolean valid) { currentInputSize < candidate.size())) { // Check if we can steal all responsibilities from candidate - for (Object b : responsibilities) { + IntIterator iter = responsibilities.intIterator(); + while(iter.hasNext()){ + int b = iter.next(); if (covered.contains(b) == false) { // Cannot steal if this input does not cover something // that the candidate is responsible for @@ -932,7 +945,7 @@ protected void writeCurrentInputToFile(File saveFile) throws IOException { } /* Saves an interesting input to the queue. */ - protected void saveCurrentInput(Set responsibilities, String why) throws IOException { + protected void saveCurrentInput(IntHashSet responsibilities, String why) throws IOException { // First, save to disk (note: we issue IDs to everyone, but only write to disk if valid) int newInputIdx = numSavedInputs++; @@ -953,14 +966,16 @@ protected void saveCurrentInput(Set responsibilities, String why) throws // Third, store basic book-keeping data currentInput.id = newInputIdx; currentInput.saveFile = saveFile; - currentInput.coverage = new Coverage(runCoverage); + currentInput.coverage = runCoverage.copy(); currentInput.nonZeroCoverage = runCoverage.getNonZeroCount(); currentInput.offspring = 0; savedInputs.get(currentParentInputIdx).offspring += 1; // Fourth, assume responsibility for branches currentInput.responsibilities = responsibilities; - for (Object b : responsibilities) { + IntIterator iter = responsibilities.intIterator(); + while(iter.hasNext()){ + int b = iter.next(); // If there is an old input that is responsible, // subsume it Input oldResponsible = responsibleInputs.get(b); @@ -989,12 +1004,15 @@ public Consumer generateCallBack(Thread thread) { /** * Handles a trace event generated during test execution. * + * Not used by FastNonCollidingCoverage, which does not allocate an + * instance of TraceEvent at each branch probe execution. + * * @param e the trace event to be handled */ protected void handleEvent(TraceEvent e) { conditionallySynchronize(multiThreaded, () -> { // Collect totalCoverage - runCoverage.handleEvent(e); + ((Coverage) runCoverage).handleEvent(e); // Check for possible timeouts every so often if (this.singleRunTimeoutMillis > 0 && this.runStart != null && (++this.branchCount) % 10_000 == 0) { @@ -1010,7 +1028,7 @@ protected void handleEvent(TraceEvent e) { * Returns a reference to the coverage statistics. * @return a reference to the coverage statistics */ - public Coverage getTotalCoverage() { + public ICoverage getTotalCoverage() { return totalCoverage; } @@ -1061,7 +1079,7 @@ public static abstract class Input implements Iterable { * *

This field is null for inputs that are not saved.

*/ - Coverage coverage = null; + ICoverage coverage = null; /** * The number of non-zero elements in `coverage`. @@ -1093,7 +1111,7 @@ public static abstract class Input implements Iterable { * in at least some responsibility set. Hence, this list * needs to be kept in-sync with {@link #responsibleInputs}.

*/ - Set responsibilities = null; + IntHashSet responsibilities = null; /** * Create an empty input. diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/repro/ReproServerGuidance.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/repro/ReproServerGuidance.java index 4df622c10..56b2bb2aa 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/repro/ReproServerGuidance.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/repro/ReproServerGuidance.java @@ -105,7 +105,7 @@ public void handleResult(Result result, Throwable error){ // Print coverage try (PrintWriter out = new PrintWriter(coverageFile)) { - coverage.getCovered().stream().sorted().forEach((i) -> + coverage.getCovered().toSortedList().forEach((i) -> out.println(String.format("%05d", i)) ); out.println(result.toString()); // EOF marker for tools to realize that repro is done diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Counter.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Counter.java index 7ba54cfc6..f29ebb661 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Counter.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Counter.java @@ -28,6 +28,9 @@ */ package edu.berkeley.cs.jqf.fuzz.util; +import org.eclipse.collections.api.list.primitive.IntList; +import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -183,8 +186,8 @@ public boolean hasNonZeros(){ * * @return a set of indices at which the count is non-zero */ - public Collection getNonZeroIndices() { - List indices = new ArrayList<>(size /2); + public IntList getNonZeroIndices() { + IntArrayList indices = new IntArrayList(size /2); for (int i = 0; i < counts.length; i++) { int count = counts[i]; if (count != 0) { @@ -199,8 +202,8 @@ public Collection getNonZeroIndices() { * * @return a set of non-zero count values in this counter. */ - public Collection getNonZeroValues() { - List values = new ArrayList<>(size /2); + public IntList getNonZeroValues() { + IntArrayList values = new IntArrayList(size /2); for (int i = 0; i < counts.length; i++) { int count = counts[i]; if (count != 0) { diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Coverage.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Coverage.java index 578a52acc..e4db4aabe 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Coverage.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Coverage.java @@ -37,13 +37,16 @@ import edu.berkeley.cs.jqf.instrument.tracing.events.CallEvent; import edu.berkeley.cs.jqf.instrument.tracing.events.TraceEvent; import edu.berkeley.cs.jqf.instrument.tracing.events.TraceEventVisitor; +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.api.list.primitive.IntList; +import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList; /** * Utility class to collect branch and function coverage * * @author Rohan Padhye */ -public class Coverage implements TraceEventVisitor { +public class Coverage implements TraceEventVisitor, ICoverage { /** The size of the coverage map. */ private final int COVERAGE_MAP_SIZE = (1 << 16) - 1; // Minus one to reduce collisions @@ -57,14 +60,15 @@ public Coverage() { } /** - * Creates a copy of an existing coverage map. + * Creates a copy of an this coverage map. * - * @param that the coverage map to copy */ - public Coverage(Coverage that) { + public Coverage copy() { + Coverage ret = new Coverage(); for (int idx = 0; idx < COVERAGE_MAP_SIZE; idx++) { - this.counter.setAtIndex(idx, that.counter.getAtIndex(idx)); + ret.counter.setAtIndex(idx, this.counter.getAtIndex(idx)); } + return ret; } /** @@ -72,6 +76,7 @@ public Coverage(Coverage that) { * * @return the size of the coverage map */ + @Override public int size() { return COVERAGE_MAP_SIZE; } @@ -103,6 +108,7 @@ public void visitCallEvent(CallEvent e) { * * @return the number of edges with non-zero counts */ + @Override public int getNonZeroCount() { return counter.getNonZeroSize(); } @@ -112,7 +118,8 @@ public int getNonZeroCount() { * * @return a collection of keys that are covered */ - public Collection getCovered() { + @Override + public IntList getCovered() { return counter.getNonZeroIndices(); } @@ -122,20 +129,25 @@ public Collection getCovered() { * @param baseline the baseline coverage * @return the set of edges that do not exist in {@code baseline} */ - public Collection computeNewCoverage(Coverage baseline) { - Collection newCoverage = new ArrayList<>(); - for (int idx : this.counter.getNonZeroIndices()) { - if (baseline.counter.getAtIndex(idx) == 0) { + @Override + public IntList computeNewCoverage(ICoverage baseline) { + IntArrayList newCoverage = new IntArrayList(); + + IntList baseNonZero = this.counter.getNonZeroIndices(); + IntIterator iter = baseNonZero.intIterator(); + while (iter.hasNext()) { + int idx = iter.next(); + if (baseline.getCounter().getAtIndex(idx) == 0) { newCoverage.add(idx); } } return newCoverage; - } /** * Clears the coverage map. */ + @Override public void clear() { this.counter.clear(); } @@ -181,12 +193,13 @@ private static int hob(int num) { * @return true iff that is not a subset * of this, causing this to change. */ - public boolean updateBits(Coverage that) { + @Override + public boolean updateBits(ICoverage that) { boolean changed = false; - if (that.counter.hasNonZeros()) { + if (that.getCounter().hasNonZeros()) { for (int idx = 0; idx < COVERAGE_MAP_SIZE; idx++) { int before = this.counter.getAtIndex(idx); - int after = before | hob(that.counter.getAtIndex(idx)); + int after = before | hob(that.getCounter().getAtIndex(idx)); if (after != before) { this.counter.setAtIndex(idx, after); changed = true; @@ -207,6 +220,7 @@ public int hashCode() { * * @return a hash of non-zero entries */ + @Override public int nonZeroHashCode() { return counter.getNonZeroIndices().hashCode(); } @@ -229,4 +243,9 @@ public String toString() { } return sb.toString(); } + + @Override + public Counter getCounter() { + return counter; + } } diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/CoverageFactory.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/CoverageFactory.java new file mode 100644 index 000000000..216a4729b --- /dev/null +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/CoverageFactory.java @@ -0,0 +1,33 @@ +package edu.berkeley.cs.jqf.fuzz.util; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +public class CoverageFactory { + + public static final String propFile = System.getProperty("janala.conf", "janala.conf"); + + private static boolean FAST_NON_COLLIDING_COVERAGE_ENABLED; + static + { + Properties properties = new Properties(); + try (InputStream propStream = new FileInputStream(propFile)) { + properties.load(propStream); + } catch (IOException e) { + // Swallow exception and continue with defaults + // System.err.println("Warning: No janala.conf file found"); + } + properties.putAll(System.getProperties()); + FAST_NON_COLLIDING_COVERAGE_ENABLED = Boolean.parseBoolean(properties.getProperty("useFastNonCollidingCoverageInstrumentation", "false")); + } + + public static ICoverage newInstance() { + if (FAST_NON_COLLIDING_COVERAGE_ENABLED) { + return new FastNonCollidingCoverage(); + } else { + return new Coverage(); + } + } +} diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/FastNonCollidingCounter.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/FastNonCollidingCounter.java new file mode 100644 index 000000000..5a5b65e56 --- /dev/null +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/FastNonCollidingCounter.java @@ -0,0 +1,164 @@ +package edu.berkeley.cs.jqf.fuzz.util; + +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.api.list.primitive.IntList; +import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList; +import org.eclipse.collections.impl.map.mutable.primitive.IntIntHashMap; + +/** + * An implementation of {@link Counter} that uses an IntIntHashMap to store values + * + * "Fast" in that it is faster than using something that involves other HashMaps, + * and boxing to-and-from primitive values. There is surely a way to make it *faster* + * than this too, without avoiding the collisions, but given the performance improvement + * compared to the original (and collision-prone) coverage, I made the opinionated decision + * to implement it this way to avoid other concerns from high collisions rates on some particular + * target applications. Further experimentation with fast (non-colliding) coverage implementations + * might help to determine which approach is preferable. + * + * @author Jonathan Bell + */ +public class FastNonCollidingCounter extends Counter { + /** The counter map as a map of integers. */ + IntIntHashMap counts; + + /* List of indices in the map that are non-zero */ + protected IntArrayList nonZeroKeys; + + /** + * Creates a new counter + */ + public FastNonCollidingCounter(int size) { + super(1); + this.counts = new IntIntHashMap(size); + this.nonZeroKeys = new IntArrayList(size / 2); + } + + + /** + * Returns the size of this counter. + * + * @return the size of this counter + */ + public synchronized int size() { + return this.counts.size(); + } + + /** + * Clears the counter by setting all values to zero. + */ + public synchronized void clear() { + this.counts.clear(); + this.nonZeroKeys.clear(); + } + + /** + * Increments the count at the given key. + * + * + * @param key the key whose count to increment + * @return the new value after incrementing the count + */ + public synchronized int increment(int key) { + int newVal = this.counts.addToValue(key, 1); + if (newVal == 1) { + this.nonZeroKeys.add(key); + } + return newVal; + } + + /** + * + * Increments the count at the given key by a given delta. + * + * @param key the key whose count to increment + * @param delta the amount to increment by + * @return the new value after incrementing the count + */ + public synchronized int increment(int key, int delta) { + int newVal = this.counts.addToValue(key, delta); + if (newVal == delta) { + nonZeroKeys.add(key); + } + return newVal; + } + + @Override + protected int incrementAtIndex(int index, int delta) { + throw new UnsupportedOperationException("This coverage is already non-colliding, please just use get"); + } + + @Override + public void setAtIndex(int idx, int value) { + throw new UnsupportedOperationException("This coverage is already non-colliding, please just use setAtIndex"); + } + + @Override + public int getAtIndex(int idx) { + throw new UnsupportedOperationException("This coverage is already non-colliding, please just use set"); + } + + /** + * Returns the number of indices with non-zero counts. + * + * @return the number of indices with non-zero counts + */ + public synchronized int getNonZeroSize() { + return nonZeroKeys.size(); + } + + + /** + * Returns a set of keys at which the count is non-zero. + * + * @return a set of keys at which the count is non-zero + */ + public synchronized IntList getNonZeroKeys() { + return this.nonZeroKeys; + } + + public IntList getNonZeroIndices(){ + return this.getNonZeroKeys(); + } + + /** + * Returns a set of non-zero count values in this counter. + * + * @return a set of non-zero count values in this counter. + */ + public synchronized IntList getNonZeroValues() { + IntArrayList values = new IntArrayList(this.counts.size() / 2); + IntIterator iter = this.counts.values().intIterator(); + while (iter.hasNext()) { + int val = iter.next(); + if (val != 0) { + values.add(val); + } + } + return values; + } + + /** + * Retreives a value for a given key. + * + *

The key is first hashed to retreive a value from + * the counter, and hence the result is modulo collisions.

+ * + * @param key the key to query + * @return the count for the index corresponding to this key + */ + public synchronized int get(int key) { + return this.counts.get(key); + } + + public synchronized void copyFrom(FastNonCollidingCounter counter) { + this.counts = new IntIntHashMap(counter.counts); + this.nonZeroKeys = new IntArrayList(counter.nonZeroKeys.size()); + this.nonZeroKeys.addAll(counter.nonZeroKeys); + } + + @Override + public boolean hasNonZeros() { + return !this.nonZeroKeys.isEmpty(); + } +} diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/FastNonCollidingCoverage.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/FastNonCollidingCoverage.java new file mode 100644 index 000000000..9908716f8 --- /dev/null +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/FastNonCollidingCoverage.java @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2017-2018 The Regents of the University of California + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package edu.berkeley.cs.jqf.fuzz.util; + +import edu.berkeley.cs.jqf.instrument.tracing.events.BranchEvent; +import edu.berkeley.cs.jqf.instrument.tracing.events.CallEvent; +import edu.berkeley.cs.jqf.instrument.tracing.events.TraceEvent; +import janala.instrument.FastCoverageListener; +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.api.list.primitive.IntList; +import org.eclipse.collections.api.tuple.primitive.IntIntPair; +import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; + +/** + * Utility class to collect branch and function coverage + * + * @author Jonathan Bell + */ +public class FastNonCollidingCoverage implements FastCoverageListener, ICoverage { + + /** The starting size of the coverage map. */ + private final int COVERAGE_MAP_SIZE = (1 << 8); + + private final FastNonCollidingCounter counter = new FastNonCollidingCounter(COVERAGE_MAP_SIZE); + + /** Creates a new coverage map. */ + public FastNonCollidingCoverage() { + + } + + /** + * Creates a copy of an existing coverage map. + * + */ + public FastNonCollidingCoverage copy() { + FastNonCollidingCoverage ret = new FastNonCollidingCoverage(); + ret.counter.copyFrom(this.counter); + return ret; + } + + /** + * Returns the size of the coverage map. + * + * @return the size of the coverage map + */ + public int size() { + return COVERAGE_MAP_SIZE; + } + + /** + * Returns the number of edges covered. + * + * @return the number of edges with non-zero counts + */ + public int getNonZeroCount() { + return counter.getNonZeroSize(); + } + + /** + * Returns a collection of branches that are covered. + * + * @return a collection of keys that are covered + */ + public IntList getCovered() { + return counter.getNonZeroIndices(); + } + + /** + * Returns a set of edges in this coverage that don't exist in baseline + * + * @param baseline the baseline coverage + * @return the set of edges that do not exist in {@code baseline} + */ + public IntList computeNewCoverage(ICoverage baseline) { + IntArrayList newCoverage = new IntArrayList(); + + IntList baseNonZero = this.counter.getNonZeroKeys(); + IntIterator iter = baseNonZero.intIterator(); + while (iter.hasNext()) { + int idx = iter.next(); + if (baseline.getCounter().get(idx) == 0) { + newCoverage.add(idx); + } + } + return newCoverage; + } + + /** + * Clears the coverage map. + */ + public void clear() { + this.counter.clear(); + } + + private static int[] HOB_CACHE = new int[1024]; + + /* Computes the highest order bit */ + private static int computeHob(int num) + { + if (num == 0) + return 0; + + int ret = 1; + + while ((num >>= 1) != 0) + ret <<= 1; + + return ret; + } + + /** Populates the HOB cache. */ + static { + for (int i = 0; i < HOB_CACHE.length; i++) { + HOB_CACHE[i] = computeHob(i); + } + } + + /** Returns the highest order bit (perhaps using the cache) */ + private static int hob(int num) { + if (num < HOB_CACHE.length) { + return HOB_CACHE[num]; + } else { + return computeHob(num); + } + } + + + /** + * Updates this coverage with bits from the parameter. + * + * @param that the run coverage whose bits to OR + * + * @return true iff that is not a subset + * of this, causing this to change. + */ + public boolean updateBits(ICoverage that) { + boolean changed = false; + synchronized (this.counter){ + synchronized (that.getCounter()){ + FastNonCollidingCounter thatCounter = (FastNonCollidingCounter) that.getCounter(); + Iterator thatIter = thatCounter.counts.keyValuesView().iterator(); + + while(thatIter.hasNext()){ + IntIntPair coverageEntry = thatIter.next(); + int before = this.counter.counts.get(coverageEntry.getOne()); + int after = before | hob(coverageEntry.getTwo()); + if(after != before){ + this.counter.counts.put(coverageEntry.getOne(), after); + changed = true; + } + if(before == 0){ + this.counter.nonZeroKeys.add(coverageEntry.getOne()); + } + } + } + } + return changed; + } + + /** Returns a hash code of the edge counts in the coverage map. */ + @Override + public int hashCode() { + return counter.counts.hashCode(); + } + + /** + * Returns a hash code of the list of edges that have been covered at least once. + * + * @return a hash of non-zero entries + */ + public int nonZeroHashCode() { + return counter.getNonZeroIndices().hashCode(); + } + + @Override + public Counter getCounter() { + return this.counter; + } + + /** + * @return a string representing the counter + */ + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("Coverage counts: \n"); + for (int i = 0; i < counter.counts.size(); i++) { + if (counter.counts.get(i) == 0) { + continue; + } + sb.append(i); + sb.append("->"); + sb.append(counter.counts.get(i)); + sb.append('\n'); + } + return sb.toString(); + } + + @Override + public void logCoverage(int iid, int arm) { + counter.increment(iid + arm); + } +} diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/ICoverage.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/ICoverage.java new file mode 100644 index 000000000..f98da6541 --- /dev/null +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/ICoverage.java @@ -0,0 +1,60 @@ +package edu.berkeley.cs.jqf.fuzz.util; + +import org.eclipse.collections.api.list.primitive.IntList; + +public interface ICoverage { + /** + * Returns the size of the coverage map. + * + * @return the size of the coverage map + */ + int size(); + + /** + * Returns the number of edges covered. + * + * @return the number of edges with non-zero counts + */ + int getNonZeroCount(); + + /** + * Returns a collection of branches that are covered. + * + * @return a collection of keys that are covered + */ + IntList getCovered(); + + /** + * Returns a set of edges in this coverage that don't exist in baseline + * + * @param baseline the baseline coverage + * @return the set of edges that do not exist in {@code baseline} + */ + IntList computeNewCoverage(ICoverage baseline); + + /** + * Clears the coverage map. + */ + void clear(); + + /** + * Updates this coverage with bits from the parameter. + * + * @param that the run coverage whose bits to OR + * + * @return true iff that is not a subset + * of this, causing this to change. + */ + boolean updateBits(ICoverage that); + + /** + * Returns a hash code of the list of edges that have been covered at least once. + * + * @return a hash of non-zero entries + */ + int nonZeroHashCode(); + + Counter getCounter(); + + ICoverage copy(); +} diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/MapOfCounters.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/MapOfCounters.java index 4ebabf3d7..f7cae29e1 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/MapOfCounters.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/MapOfCounters.java @@ -28,6 +28,9 @@ */ package edu.berkeley.cs.jqf.fuzz.util; +import org.eclipse.collections.api.list.primitive.IntList; +import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList; + import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -76,11 +79,11 @@ public void increment(int k1, int k2) { counters[idx].increment(k2); } - public Collection nonZeroCountsAtIndex(int idx) { + public IntList nonZeroCountsAtIndex(int idx) { if (counters[idx] != null) { return counters[idx].getNonZeroValues(); } else { - return Collections.emptyList(); + return new IntArrayList(0); } } diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/NonZeroCachingCounter.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/NonZeroCachingCounter.java index 20f558de6..83c9058aa 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/NonZeroCachingCounter.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/NonZeroCachingCounter.java @@ -28,6 +28,10 @@ */ package edu.berkeley.cs.jqf.fuzz.util; +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.api.list.primitive.IntList; +import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList; + import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -44,21 +48,23 @@ public class NonZeroCachingCounter extends Counter { private int nonZeroCount; - private Collection nonZeroIndices; + private IntArrayList nonZeroIndices; public NonZeroCachingCounter(int size) { super(size); this.nonZeroCount = 0; - this.nonZeroIndices = new ArrayList<>(); + this.nonZeroIndices = new IntArrayList(); } @Override public void clear() { - for (int idx : nonZeroIndices) { + IntIterator iter = nonZeroIndices.intIterator(); + while(iter.hasNext()){ + int idx = iter.next(); counts[idx] = 0; } this.nonZeroCount = 0; - this.nonZeroIndices.clear(); + this.nonZeroIndices = new IntArrayList(); } @@ -84,14 +90,16 @@ public boolean hasNonZeros(){ } @Override - public Collection getNonZeroIndices() { + public IntList getNonZeroIndices() { return nonZeroIndices; } @Override - public Collection getNonZeroValues() { - List values = new ArrayList<>(size /2); - for (int idx : nonZeroIndices) { + public IntList getNonZeroValues() { + IntArrayList values = new IntArrayList(this.size / 2); + IntIterator iter = nonZeroIndices.intIterator(); + while(iter.hasNext()){ + int idx = iter.next(); int count = counts[idx]; assert (count != 0); values.add(count); diff --git a/fuzz/src/test/java/edu/berkeley/cs/jqf/fuzz/afl/RedundancyTest.java b/fuzz/src/test/java/edu/berkeley/cs/jqf/fuzz/afl/RedundancyTest.java index 2a6f44068..d566b08f8 100644 --- a/fuzz/src/test/java/edu/berkeley/cs/jqf/fuzz/afl/RedundancyTest.java +++ b/fuzz/src/test/java/edu/berkeley/cs/jqf/fuzz/afl/RedundancyTest.java @@ -37,6 +37,7 @@ import com.pholser.junit.quickcheck.generator.InRange; import com.pholser.junit.quickcheck.generator.Size; import com.pholser.junit.quickcheck.runner.JUnitQuickcheck; +import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList; import org.junit.Assert; import org.junit.runner.RunWith; @@ -68,14 +69,18 @@ public void testRedundancyScore(@Size(min=1, max = 5) List<@InRange(minInt=1, ma Assert.assertTrue(root*root == squareSum); - List redundantCounts = new ArrayList<>(root); + IntArrayList redundantCounts = new IntArrayList(root); for (int i = 0; i < root; i++) { redundantCounts.add(root); } - Assert.assertTrue(sum(redundantCounts) == squareSum); + Assert.assertTrue(redundantCounts.sum() == squareSum); + IntArrayList countsIntArrayList = new IntArrayList(counts.size()); + for(int i : counts){ + countsIntArrayList.add(i); + } // Compute redundancy score for some memory accesses - double score = PerfFuzzGuidance.computeRedundancyScore(counts); + double score = PerfFuzzGuidance.computeRedundancyScore(countsIntArrayList); // Ensure that scores are in [0, 1) Assert.assertTrue(score >= 0 && score < 1); diff --git a/fuzz/src/test/java/edu/berkeley/cs/jqf/fuzz/util/CountersTest.java b/fuzz/src/test/java/edu/berkeley/cs/jqf/fuzz/util/CountersTest.java index fb8fca4b7..c729616ff 100644 --- a/fuzz/src/test/java/edu/berkeley/cs/jqf/fuzz/util/CountersTest.java +++ b/fuzz/src/test/java/edu/berkeley/cs/jqf/fuzz/util/CountersTest.java @@ -28,11 +28,14 @@ */ package edu.berkeley.cs.jqf.fuzz.util; +import java.util.ArrayList; import java.util.Collection; import com.pholser.junit.quickcheck.Property; import com.pholser.junit.quickcheck.generator.InRange; import com.pholser.junit.quickcheck.runner.JUnitQuickcheck; +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.api.list.primitive.IntList; import org.junit.Test; import org.junit.runner.RunWith; @@ -103,14 +106,11 @@ public void nonZeroValuesIsAccurate(int[] keys, int delta) { counter.increment(key, delta); } - Collection nonZeroValues = counter.getNonZeroValues(); + IntList nonZeroValues = counter.getNonZeroValues(); assertThat(nonZeroValues.size(), lessThanOrEqualTo(keys.length)); - int sum = 0; - for (int v : nonZeroValues) { - sum += v; - } + int sum = (int) nonZeroValues.sum(); assertEquals(keys.length * delta, sum); @@ -123,8 +123,8 @@ public void nonZeroIndicesIsAccurate(int[] keys, int delta) { counter.increment(key, delta); } - Collection nonZeroIndices = counter.getNonZeroIndices(); - Collection nonZeroValues = counter.getNonZeroValues(); + IntList nonZeroIndices = counter.getNonZeroIndices(); + IntList nonZeroValues = counter.getNonZeroValues(); assertEquals(nonZeroValues.size(), nonZeroIndices.size()); } @@ -138,11 +138,20 @@ public void nonZeroSizeIsAccurate(int[] keys, int delta) { } int nonZeroSize = counter.getNonZeroSize(); - Collection nonZeroValues = counter.getNonZeroValues(); + IntList nonZeroValues = counter.getNonZeroValues(); assertEquals(nonZeroValues.size(), nonZeroSize); } + static ArrayList toCollection(IntList primIntList){ + ArrayList ret = new ArrayList<>(primIntList.size()); + IntIterator iter = primIntList.intIterator(); + while(iter.hasNext()){ + ret.add(iter.next()); + } + return ret; + } + @Property public void setAtIndexWorks(@InRange(minInt=0, maxInt=COUNTER_SIZE-1) int index, int value) { Counter counter = new Counter(COUNTER_SIZE); @@ -151,8 +160,8 @@ public void setAtIndexWorks(@InRange(minInt=0, maxInt=COUNTER_SIZE-1) int index, int nonZeroSize = counter.getNonZeroSize(); - Collection nonZeroIndices = counter.getNonZeroIndices(); - Collection nonZeroValues = counter.getNonZeroValues(); + ArrayList nonZeroIndices = toCollection(counter.getNonZeroIndices()); + ArrayList nonZeroValues = toCollection(counter.getNonZeroValues()); if (value == 0) { assertThat(nonZeroSize, is(0)); assertThat(nonZeroIndices, iterableWithSize(0)); @@ -188,8 +197,8 @@ public void clearsToZero(int[] keys) { } assertThat(counter.getNonZeroSize(), is(0)); - assertThat(counter.getNonZeroIndices(), iterableWithSize(0)); - assertThat(counter.getNonZeroValues(), iterableWithSize(0)); + assertThat(toCollection(counter.getNonZeroIndices()), iterableWithSize(0)); + assertThat(toCollection(counter.getNonZeroValues()), iterableWithSize(0)); } @Test diff --git a/fuzz/src/test/java/edu/berkeley/cs/jqf/fuzz/util/NonZeroCachingCountersTest.java b/fuzz/src/test/java/edu/berkeley/cs/jqf/fuzz/util/NonZeroCachingCountersTest.java index 7493c5577..8e60df9e8 100644 --- a/fuzz/src/test/java/edu/berkeley/cs/jqf/fuzz/util/NonZeroCachingCountersTest.java +++ b/fuzz/src/test/java/edu/berkeley/cs/jqf/fuzz/util/NonZeroCachingCountersTest.java @@ -28,12 +28,17 @@ */ package edu.berkeley.cs.jqf.fuzz.util; +import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import com.pholser.junit.quickcheck.Property; import com.pholser.junit.quickcheck.generator.InRange; import com.pholser.junit.quickcheck.runner.JUnitQuickcheck; +import org.eclipse.collections.api.iterator.IntIterator; +import org.eclipse.collections.api.list.primitive.IntList; +import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList; +import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; import org.junit.runner.RunWith; import static org.hamcrest.Matchers.*; @@ -104,9 +109,9 @@ public void nonZeroValuesIsAccurate(int[] keys, int delta) { counter2.increment(key, delta); } - Collection nonZeroValues1 = counter1.getNonZeroValues(); - Collection nonZeroValues2 = counter2.getNonZeroValues(); - assertEquals(new HashSet<>(nonZeroValues1), new HashSet<>(nonZeroValues2)); + IntList nonZeroValues1 = counter1.getNonZeroValues(); + IntList nonZeroValues2 = counter2.getNonZeroValues(); + assertEquals(IntHashSet.newSet(nonZeroValues1), IntHashSet.newSet(nonZeroValues2)); } @@ -119,9 +124,9 @@ public void nonZeroIndicesIsAccurate(int[] keys, int delta) { counter2.increment(key, delta); } - Collection nonZeroIndices1 = counter1.getNonZeroIndices(); - Collection nonZeroIndices2 = counter2.getNonZeroIndices(); - assertEquals(new HashSet<>(nonZeroIndices1), new HashSet<>(nonZeroIndices2)); + IntList nonZeroIndices1 = counter1.getNonZeroIndices(); + IntList nonZeroIndices2 = counter2.getNonZeroIndices(); + assertEquals(IntHashSet.newSet(nonZeroIndices1), IntHashSet.newSet(nonZeroIndices2)); } @@ -141,6 +146,14 @@ public void nonZeroSizeIsAccurate(int[] keys, int delta) { } + static ArrayList toJavaArrayList(IntList primitiveList){ + ArrayList ret = new ArrayList<>(primitiveList.size()); + IntIterator iter = primitiveList.intIterator(); + while(iter.hasNext()){ + ret.add(iter.next()); + } + return ret; + } @Property public void setAtIndexWorks(@InRange(minInt=0, maxInt=COUNTER_SIZE-1) int index, int value) { Counter counter = new NonZeroCachingCounter(COUNTER_SIZE); @@ -149,8 +162,8 @@ public void setAtIndexWorks(@InRange(minInt=0, maxInt=COUNTER_SIZE-1) int index, int nonZeroSize = counter.getNonZeroSize(); - Collection nonZeroIndices = counter.getNonZeroIndices(); - Collection nonZeroValues = counter.getNonZeroValues(); + Collection nonZeroIndices = toJavaArrayList(counter.getNonZeroIndices()); + Collection nonZeroValues = toJavaArrayList(counter.getNonZeroValues()); if (value == 0) { assertThat(nonZeroSize, is(0)); assertThat(nonZeroIndices, iterableWithSize(0)); @@ -186,7 +199,7 @@ public void clearsToZero(int[] keys) { } assertThat(counter.getNonZeroSize(), is(0)); - assertThat(counter.getNonZeroIndices(), iterableWithSize(0)); - assertThat(counter.getNonZeroValues(), iterableWithSize(0)); + assertThat(toJavaArrayList(counter.getNonZeroIndices()), iterableWithSize(0)); + assertThat(toJavaArrayList(counter.getNonZeroValues()), iterableWithSize(0)); } } diff --git a/instrument/src/main/java/edu/berkeley/cs/jqf/instrument/tracing/FastCoverageSnoop.java b/instrument/src/main/java/edu/berkeley/cs/jqf/instrument/tracing/FastCoverageSnoop.java new file mode 100644 index 000000000..013d01faa --- /dev/null +++ b/instrument/src/main/java/edu/berkeley/cs/jqf/instrument/tracing/FastCoverageSnoop.java @@ -0,0 +1,45 @@ +package edu.berkeley.cs.jqf.instrument.tracing; + +import janala.instrument.FastCoverageListener; + +public class FastCoverageSnoop { + static FastCoverageListener coverageListener = new FastCoverageListener() { + @Override + public void logCoverage(int iid, int arm) { + + } + }; + + @SuppressWarnings("unused") //Invoked by instrumentation + public static void LOGJUMP(int iid, int branch) { + coverageListener.logCoverage(iid, branch); + } + + @SuppressWarnings("unused") //Invoked by instrumentation + public static void LOGLOOKUPSWITCH(int value, int iid, int dflt, int[] cases) { + // Compute arm index or else default + int arm = cases.length; + for (int i = 0; i < cases.length; i++) { + if (value == cases[i]) { + arm = i; + break; + } + } + arm++; + coverageListener.logCoverage(iid, arm); + } + + @SuppressWarnings("unused") //Invoked by instrumentation + public static void LOGTABLESWITCH(int value, int iid, int min, int max, int dflt) { + int arm = 1 + max - min; + if (value >= min && value <= max) { + arm = value - min; + } + arm++; + coverageListener.logCoverage(iid, arm); + } + + public static void setFastCoverageListener(FastCoverageListener runCoverage) { + coverageListener = runCoverage; + } +} diff --git a/instrument/src/main/java/janala/instrument/Config.java b/instrument/src/main/java/janala/instrument/Config.java index 811008657..1cbb99f05 100644 --- a/instrument/src/main/java/janala/instrument/Config.java +++ b/instrument/src/main/java/janala/instrument/Config.java @@ -18,6 +18,7 @@ class Config { public final boolean instrumentHeapLoad; public final boolean instrumentAlloc; public final String instrumentationCacheDir; + public final boolean useFastCoverageInstrumentation; private Config() { // Read properties from the conf file @@ -34,14 +35,24 @@ private Config() { verbose = Boolean.parseBoolean(properties.getProperty("janala.verbose", "false")); - analysisClass = - properties.getProperty("janala.snoopClass", "edu.berkeley.cs.jqf.instrument.tracing.SingleSnoop") - .replace('.', '/'); + useFastCoverageInstrumentation = Boolean.parseBoolean(properties.getProperty("useFastNonCollidingCoverageInstrumentation", "false")); + if(useFastCoverageInstrumentation){ + analysisClass = "edu/berkeley/cs/jqf/instrument/tracing/FastCoverageSnoop"; + } else { + analysisClass = + properties.getProperty("janala.snoopClass", "edu.berkeley.cs.jqf.instrument.tracing.SingleSnoop") + .replace('.', '/'); + } instrumentHeapLoad = Boolean.parseBoolean(properties.getProperty("janala.instrumentHeapLoad", "false")); instrumentAlloc = Boolean.parseBoolean(properties.getProperty("janala.instrumentAlloc", "false")); + if((instrumentAlloc || instrumentHeapLoad) && useFastCoverageInstrumentation){ + throw new UnsupportedOperationException("It is currently not possible to use allocation or heap load tracking in conjunction with fast coverage"); + } + + String excludeInstStr = properties.getProperty("janala.excludes", null); if (excludeInstStr != null) { excludeInst = excludeInstStr.replace('.', '/').split(","); diff --git a/instrument/src/main/java/janala/instrument/FastCoverageListener.java b/instrument/src/main/java/janala/instrument/FastCoverageListener.java new file mode 100644 index 000000000..e5eec20fe --- /dev/null +++ b/instrument/src/main/java/janala/instrument/FastCoverageListener.java @@ -0,0 +1,5 @@ +package janala.instrument; + +public interface FastCoverageListener { + public void logCoverage(int iid, int arm); +} diff --git a/instrument/src/main/java/janala/instrument/FastCoverageMethodAdapter.java b/instrument/src/main/java/janala/instrument/FastCoverageMethodAdapter.java new file mode 100644 index 000000000..7bcd9d2aa --- /dev/null +++ b/instrument/src/main/java/janala/instrument/FastCoverageMethodAdapter.java @@ -0,0 +1,215 @@ +package janala.instrument; + +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +public class FastCoverageMethodAdapter extends MethodVisitor implements Opcodes { + boolean isInit; + boolean isSuperInitCalled; // Used to keep track of calls to super()/this() in () + int newStack = 0; // Used to keep-track of NEW instructions in () + + private final String className; + private final String superName; + + private final GlobalStateForInstrumentation instrumentationState; + + public FastCoverageMethodAdapter(MethodVisitor mv, String className, + String methodName, String descriptor, String superName, + GlobalStateForInstrumentation instrumentationState) { + super(ASM8, mv); + this.isInit = methodName.equals(""); + this.isSuperInitCalled = false; + this.className = className; + this.superName = superName; + + this.instrumentationState = instrumentationState; + } + + /** Push a value onto the stack. */ + private static void addBipushInsn(MethodVisitor mv, int val) { + Utils.addBipushInsn(mv, val); + } + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { + if (opcode == INVOKESPECIAL && name.equals("")) { + + + // The first call to within a constructor (`isInit`) on the same or super class, + // which is not associated with a NEW instruction (`newStack` == 0), + // will be considered as an invocation of super()/this(). + + if (isInit && isSuperInitCalled == false && newStack == 0 && + (owner.equals(className) || owner.equals(superName))) { + // Constructor calls to method of the same or super class. + // + // XXX: This is a hack. We assume that if we see an to same or + // super class, then it must be a super() or this() call. However, + // there are counter-examples such as `public Foo() { super(new Foo()); }`, + // which will cause broken class files. This comment is here as a forewarning + // for when this situation is eventually encountered due to a bytecode + // verification error due to stack-map frames not matching up. + // + // In this case, we do not wrap the method call in try catch block as + // it uses uninitialized this object. + isSuperInitCalled = true; + } else { + // Call to but not a super() or this(). Must have occurred after a NEW. + // This is an outer constructor call, so reduce the NEW stack + if (isInit) { + newStack--; + assert newStack >= 0; + } + } + } + mv.visitMethodInsn(opcode, owner, name, desc, itf); + } + + @Override + public void visitCode() { + super.visitCode(); + int iid = instrumentationState.incAndGetFastCoverageId(); + addBipushInsn(mv, iid); + mv.visitInsn(ICONST_0); + mv.visitMethodInsn(INVOKESTATIC, Config.instance.analysisClass, "LOGJUMP", "(II)V", false); + } + + private void addConditionalJumpInstrumentation(int opcode, Label finalBranchTarget, + String instMethodName, String instMethodDesc) { + int iid = instrumentationState.incAndGetFastCoverageId(); + instrumentationState.incAndGetId(); //reserve another counter for the other side of this branch + + Label intermediateBranchTarget = new Label(); + Label fallthrough = new Label(); + + // Perform the original jump, but branch to intermediate label + mv.visitJumpInsn(opcode, intermediateBranchTarget); + // If we did not jump, skip to the fallthrough + mv.visitJumpInsn(GOTO, fallthrough); + + // Now instrument the branch target + mv.visitLabel(intermediateBranchTarget); + addBipushInsn(mv, iid); + addBipushInsn(mv, 1); // Mark branch as taken + mv.visitMethodInsn(INVOKESTATIC, Config.instance.analysisClass, instMethodName, instMethodDesc, false); + mv.visitJumpInsn(GOTO, finalBranchTarget); // Go to actual branch target + + // Now instrument the fall through + mv.visitLabel(fallthrough); + addBipushInsn(mv, iid); + addBipushInsn(mv, 0); // Mark branch as not taken + mv.visitMethodInsn(INVOKESTATIC, Config.instance.analysisClass, instMethodName, instMethodDesc, false); + + // continue with fall-through code visiting + } + + @Override + public void visitJumpInsn(int opcode, Label label) { + if (isInit && !isSuperInitCalled) { + // Jumps in a constructor before super() or this() mess up the analysis + throw new RuntimeException("Cannot handle jumps before super/this"); + } + + switch (opcode) { + case IFEQ: + case IFNE: + case IFLT: + case IFGE: + case IFGT: + case IFLE: + case IF_ICMPEQ: + case IF_ICMPNE: + case IF_ICMPLT: + case IF_ICMPGE: + case IF_ICMPGT: + case IF_ICMPLE: + case IF_ACMPEQ: + case IF_ACMPNE: + case IFNULL: + case IFNONNULL: + addConditionalJumpInstrumentation(opcode, label, "LOGJUMP", "(II)V"); + break; + case GOTO: + case JSR: + mv.visitJumpInsn(opcode, label); + break; + default: + throw new RuntimeException("Unknown jump opcode " + opcode); + } + } + + private Integer lastLineNumber = 0; + + @Override + public void visitLineNumber(int lineNumber, Label label) { + lastLineNumber = lineNumber; + mv.visitLineNumber(lineNumber, label); + } + + + private int getLabelNum(Label label) { + return System.identityHashCode(label); + } + + + + + + @Override + public void visitTableSwitchInsn(int min, int max, Label dflt, Label... labels) { + // Save operand value + //addValueReadInsn(mv, "I", "GETVALUE_"); + mv.visitInsn(Opcodes.DUP); + // Log switch instruction + addBipushInsn(mv, instrumentationState.incAndGetFastCoverageId()); + addBipushInsn(mv, min); + addBipushInsn(mv, max); + addBipushInsn(mv, getLabelNum(dflt)); + + for (int i = 0; i < labels.length; i++) { + //create a coverage probe for each of the arms, we'll refer to it by offset + instrumentationState.incAndGetFastCoverageId(); + } + + + //create a coverage probe for the default case + instrumentationState.incAndGetFastCoverageId(); + mv.visitMethodInsn(INVOKESTATIC, Config.instance.analysisClass, "LOGTABLESWITCH", "(IIIII)V", false); + mv.visitTableSwitchInsn(min, max, dflt, labels); + } + + + @Override + public void visitLookupSwitchInsn(Label dflt, int[] keys, Label[] labels) { + // Save operand value + mv.visitInsn(Opcodes.DUP); + + // Log switch instruction + addBipushInsn(mv, instrumentationState.incAndGetFastCoverageId()); + addBipushInsn(mv, getLabelNum(dflt)); + + addBipushInsn(mv, keys.length); + mv.visitIntInsn(NEWARRAY, T_INT); + for (int i = 0; i < keys.length; i++) { + mv.visitInsn(DUP); + addBipushInsn(mv, i); + addBipushInsn(mv, keys[i]); + mv.visitInsn(IASTORE); + //create a coverage probe for each of the arms, we'll refer to it by offset + instrumentationState.incAndGetFastCoverageId(); + } + + + //create a coverage probe for the default case + instrumentationState.incAndGetFastCoverageId(); + mv.visitMethodInsn(INVOKESTATIC, Config.instance.analysisClass, "LOGLOOKUPSWITCH", "(III[I)V", false); + mv.visitLookupSwitchInsn(dflt, keys, labels); + } + + @Override + public void visitMaxs(int maxStack, int maxLocals) { + //Allow ASM to calculate the correct maxStack by passing '0' as the maximum stack value. + mv.visitMaxs(0, maxLocals); + } +} diff --git a/instrument/src/main/java/janala/instrument/GlobalStateForInstrumentation.java b/instrument/src/main/java/janala/instrument/GlobalStateForInstrumentation.java index 9e1218210..5cc611ac9 100644 --- a/instrument/src/main/java/janala/instrument/GlobalStateForInstrumentation.java +++ b/instrument/src/main/java/janala/instrument/GlobalStateForInstrumentation.java @@ -8,6 +8,14 @@ public class GlobalStateForInstrumentation { private int mid = 0; private int cid = 0; + // JQF's Fast Coverage implementation uses a plain int, no bit packing, no truncation errors + private int fastCoverageIID = 0; + public int incAndGetFastCoverageId(){ + fastCoverageIID++; + return fastCoverageIID; + } + + // When one gets the id, she gets the result of merging all three ids. // NOTE: Beaware of truncation errors. private final static int CBITS = 10; // CID occupies the upper 10 bits diff --git a/instrument/src/main/java/janala/instrument/SnoopInstructionClassAdapter.java b/instrument/src/main/java/janala/instrument/SnoopInstructionClassAdapter.java index e87130eda..62eace23e 100644 --- a/instrument/src/main/java/janala/instrument/SnoopInstructionClassAdapter.java +++ b/instrument/src/main/java/janala/instrument/SnoopInstructionClassAdapter.java @@ -28,12 +28,16 @@ public void visit(int version, } @Override - public MethodVisitor visitMethod(int access, String name, String desc, + public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { MethodVisitor mv = cv.visitMethod(access, name, desc, signature, exceptions); if (mv != null) { - return new SnoopInstructionMethodAdapter(mv, className, name, desc, superName, - GlobalStateForInstrumentation.instance); + if(Config.instance.useFastCoverageInstrumentation){ + return new FastCoverageMethodAdapter(mv, className, name, desc, superName, GlobalStateForInstrumentation.instance); + }else { + return new SnoopInstructionMethodAdapter(mv, className, name, desc, superName, + GlobalStateForInstrumentation.instance); + } } return null; } diff --git a/instrument/src/main/java/janala/instrument/SnoopInstructionTransformer.java b/instrument/src/main/java/janala/instrument/SnoopInstructionTransformer.java index 82a5c4cf2..490d3f4c1 100644 --- a/instrument/src/main/java/janala/instrument/SnoopInstructionTransformer.java +++ b/instrument/src/main/java/janala/instrument/SnoopInstructionTransformer.java @@ -20,7 +20,7 @@ public class SnoopInstructionTransformer implements ClassFileTransformer { private static final String instDir = Config.instance.instrumentationCacheDir; private static final boolean verbose = Config.instance.verbose; - private static String[] banned = {"[", "java/lang", "janala", "org/objectweb/asm", "sun", "jdk", "java/util/function"}; + private static String[] banned = {"[", "java/lang", "org/eclipse/collections", "janala", "org/objectweb/asm", "sun", "jdk", "java/util/function"}; private static String[] excludes = Config.instance.excludeInst; private static String[] includes = Config.instance.includeInst; public static void premain(String agentArgs, Instrumentation inst) throws ClassNotFoundException { From 65713f7c62246a2a4308f68cfffbe6cd119d6136 Mon Sep 17 00:00:00 2001 From: Jonathan Bell Date: Fri, 31 Dec 2021 12:47:54 -0500 Subject: [PATCH 2/4] :facepalm: remove the obvious bug that was making this extremely collision-prone, rather htan collision free --- .../main/java/janala/instrument/FastCoverageMethodAdapter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrument/src/main/java/janala/instrument/FastCoverageMethodAdapter.java b/instrument/src/main/java/janala/instrument/FastCoverageMethodAdapter.java index 7bcd9d2aa..0c565aafb 100644 --- a/instrument/src/main/java/janala/instrument/FastCoverageMethodAdapter.java +++ b/instrument/src/main/java/janala/instrument/FastCoverageMethodAdapter.java @@ -78,7 +78,7 @@ public void visitCode() { private void addConditionalJumpInstrumentation(int opcode, Label finalBranchTarget, String instMethodName, String instMethodDesc) { int iid = instrumentationState.incAndGetFastCoverageId(); - instrumentationState.incAndGetId(); //reserve another counter for the other side of this branch + instrumentationState.incAndGetFastCoverageId(); //reserve another counter for the other side of this branch Label intermediateBranchTarget = new Label(); Label fallthrough = new Label(); From 2e103307d72d5bb489f5c621900e73d2e77d601b Mon Sep 17 00:00:00 2001 From: Jonathan Bell Date: Sun, 9 Jan 2022 09:20:05 -0500 Subject: [PATCH 3/4] Performance optimization: Don't track coverage probes in coverage utility classes --- .../java/janala/instrument/SnoopInstructionTransformer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrument/src/main/java/janala/instrument/SnoopInstructionTransformer.java b/instrument/src/main/java/janala/instrument/SnoopInstructionTransformer.java index 490d3f4c1..9dde23d0b 100644 --- a/instrument/src/main/java/janala/instrument/SnoopInstructionTransformer.java +++ b/instrument/src/main/java/janala/instrument/SnoopInstructionTransformer.java @@ -20,7 +20,7 @@ public class SnoopInstructionTransformer implements ClassFileTransformer { private static final String instDir = Config.instance.instrumentationCacheDir; private static final boolean verbose = Config.instance.verbose; - private static String[] banned = {"[", "java/lang", "org/eclipse/collections", "janala", "org/objectweb/asm", "sun", "jdk", "java/util/function"}; + private static String[] banned = {"[", "java/lang", "org/eclipse/collections", "edu/berkeley/cs/jqf/fuzz/util", "janala", "org/objectweb/asm", "sun", "jdk", "java/util/function"}; private static String[] excludes = Config.instance.excludeInst; private static String[] includes = Config.instance.includeInst; public static void premain(String agentArgs, Instrumentation inst) throws ClassNotFoundException { From e91bd9446eac329c31ee1259f87cf0602e18ea83 Mon Sep 17 00:00:00 2001 From: Jonathan Bell Date: Mon, 17 Jan 2022 09:29:43 -0500 Subject: [PATCH 4/4] Reduce memory overhead for storing failures by only storing a hash of the failure; Add a branch probe before each call site --- .../berkeley/cs/jqf/fuzz/ei/ZestGuidance.java | 25 +++++++++++++++++-- .../instrument/FastCoverageMethodAdapter.java | 9 ++++--- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ZestGuidance.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ZestGuidance.java index 4ddf193d1..89bc5e535 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ZestGuidance.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ZestGuidance.java @@ -40,6 +40,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintWriter; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.time.Duration; import java.util.ArrayDeque; import java.util.ArrayList; @@ -164,7 +166,7 @@ public class ZestGuidance implements Guidance { protected Map responsibleInputs = new HashMap<>(totalCoverage.size()); /** The set of unique failures found so far. */ - protected Set> uniqueFailures = new HashSet<>(); + protected Set uniqueFailures = new HashSet<>(); /** save crash to specific location (should be used with EXIT_ON_CRASH) **/ protected final String EXACT_CRASH_PATH = System.getProperty("jqf.ei.EXACT_CRASH_PATH"); @@ -785,7 +787,7 @@ public void handleResult(Result result, Throwable error) throws GuidanceExceptio } // Attempt to add this to the set of unique failures - if (uniqueFailures.add(Arrays.asList(rootCause.getStackTrace()))) { + if (uniqueFailures.add(failureDigest(rootCause.getStackTrace()))) { // Trim input (remove unused keys) currentInput.gc(); @@ -1047,6 +1049,25 @@ protected void conditionallySynchronize(boolean cond, Runnable task) { } } + private static MessageDigest sha1; + + private static String failureDigest(StackTraceElement[] stackTrace) { + if (sha1 == null) { + try { + sha1 = MessageDigest.getInstance("SHA-1"); + } catch (NoSuchAlgorithmException e) { + throw new GuidanceException(e); + } + } + byte[] bytes = sha1.digest(Arrays.deepToString(stackTrace).getBytes()); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < bytes.length; i++) { + sb.append(Integer.toString((bytes[i] & 0xff) + 0x100, 16) + .substring(1)); + } + return sb.toString(); + } + /** * A candidate or saved test input that maps objects of type K to bytes. */ diff --git a/instrument/src/main/java/janala/instrument/FastCoverageMethodAdapter.java b/instrument/src/main/java/janala/instrument/FastCoverageMethodAdapter.java index 0c565aafb..e80a5b194 100644 --- a/instrument/src/main/java/janala/instrument/FastCoverageMethodAdapter.java +++ b/instrument/src/main/java/janala/instrument/FastCoverageMethodAdapter.java @@ -33,6 +33,11 @@ private static void addBipushInsn(MethodVisitor mv, int val) { @Override public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { + int iid = instrumentationState.incAndGetFastCoverageId(); + addBipushInsn(mv, iid); + mv.visitInsn(ICONST_0); + mv.visitMethodInsn(INVOKESTATIC, Config.instance.analysisClass, "LOGJUMP", "(II)V", false); + if (opcode == INVOKESPECIAL && name.equals("")) { @@ -69,10 +74,6 @@ public void visitMethodInsn(int opcode, String owner, String name, String desc, @Override public void visitCode() { super.visitCode(); - int iid = instrumentationState.incAndGetFastCoverageId(); - addBipushInsn(mv, iid); - mv.visitInsn(ICONST_0); - mv.visitMethodInsn(INVOKESTATIC, Config.instance.analysisClass, "LOGJUMP", "(II)V", false); } private void addConditionalJumpInstrumentation(int opcode, Label finalBranchTarget,