Skip to content

Commit

Permalink
Pull out FingerprintValueStore and PutOperation from NestedSetStore.
Browse files Browse the repository at this point in the history
These are more broadly useful in the implementation of shared values.

PiperOrigin-RevId: 609006068
Change-Id: I2e03a75b84f75b6b068c07e293fc1fda7b52af78
  • Loading branch information
aoeui authored and copybara-github committed Feb 21, 2024
1 parent cb901b7 commit ea26c85
Show file tree
Hide file tree
Showing 11 changed files with 274 additions and 238 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@
import com.google.devtools.build.lib.bugreport.Crash;
import com.google.devtools.build.lib.bugreport.CrashContext;
import com.google.devtools.build.lib.collect.compacthashset.CompactHashSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetStore.MissingNestedSetException;
import com.google.devtools.build.lib.concurrent.MoreFutures;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.server.FailureDetails.Interrupted;
import com.google.devtools.build.lib.server.FailureDetails.Interrupted.Code;
import com.google.devtools.build.lib.skyframe.serialization.FingerprintValueStore.MissingFingerprintValueException;
import com.google.devtools.build.lib.skyframe.serialization.VisibleForSerialization;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant;
Expand Down Expand Up @@ -408,16 +408,16 @@ public E getSingleton() {

/**
* Returns an immutable list of all unique elements of this set, similar to {@link #toList}, but
* will propagate an {@code InterruptedException} or {@link MissingNestedSetException} if one is
* thrown.
* will propagate an {@code InterruptedException} or {@link MissingFingerprintValueException} if
* one is thrown.
*/
public ImmutableList<E> toListInterruptibly()
throws InterruptedException, MissingNestedSetException {
throws InterruptedException, MissingFingerprintValueException {
Object actualChildren;
if (children instanceof ListenableFuture) {
actualChildren =
MoreFutures.waitForFutureAndGetWithCheckedException(
(ListenableFuture<Object[]>) children, MissingNestedSetException.class);
(ListenableFuture<Object[]>) children, MissingFingerprintValueException.class);
} else {
actualChildren = children;
}
Expand All @@ -430,22 +430,22 @@ public ImmutableList<E> toListInterruptibly()
* TimeoutException} if this set is deserializing and does not become ready within the given
* timeout.
*
* <p>Additionally, throws {@link MissingNestedSetException} if this nested set {@link
* <p>Additionally, throws {@link MissingFingerprintValueException} if this nested set {@link
* #isFromStorage} and could not be retrieved.
*
* <p>Note that the timeout only applies to blocking for the deserialization future to become
* available. The actual list transformation is untimed.
*/
public ImmutableList<E> toListWithTimeout(Duration timeout)
throws InterruptedException, TimeoutException, MissingNestedSetException {
throws InterruptedException, TimeoutException, MissingFingerprintValueException {
Object actualChildren;
if (children instanceof ListenableFuture) {
try {
actualChildren =
((ListenableFuture<Object[]>) children).get(timeout.toNanos(), TimeUnit.NANOSECONDS);
} catch (ExecutionException e) {
Throwables.propagateIfPossible(
e.getCause(), InterruptedException.class, MissingNestedSetException.class);
e.getCause(), InterruptedException.class, MissingFingerprintValueException.class);
throw new IllegalStateException(e);
}
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@
import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.devtools.build.lib.collect.nestedset.NestedSetStore.FingerprintComputationResult;
import com.google.devtools.build.lib.skyframe.serialization.DeserializationContext;
import com.google.devtools.build.lib.skyframe.serialization.ObjectCodec;
import com.google.devtools.build.lib.skyframe.serialization.PutOperation;
import com.google.devtools.build.lib.skyframe.serialization.SerializationContext;
import com.google.devtools.build.lib.skyframe.serialization.SerializationException;
import com.google.protobuf.ByteString;
Expand Down Expand Up @@ -83,7 +83,7 @@ public void serialize(SerializationContext context, NestedSet<?> obj, CodedOutpu
} else {
codedOut.writeEnumNoTag(NestedSetSize.NONLEAF.ordinal());
codedOut.writeInt32NoTag(obj.getApproxDepth());
FingerprintComputationResult fingerprintComputationResult =
PutOperation fingerprintComputationResult =
nestedSetStore.computeFingerprintAndStore((Object[]) obj.getChildren(), context);
context.addFutureToBlockWritingOn(fingerprintComputationResult.writeStatus());
codedOut.writeByteArrayNoTag(fingerprintComputationResult.fingerprint().toByteArray());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,20 +26,20 @@
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import com.google.devtools.build.lib.bugreport.BugReporter;
import com.google.devtools.build.lib.collect.nestedset.NestedSetStore.FingerprintComputationResult;
import com.google.devtools.build.lib.skyframe.serialization.PutOperation;
import com.google.devtools.build.lib.skyframe.serialization.SerializationConstants;
import com.google.protobuf.ByteString;
import javax.annotation.Nullable;

/**
* A bidirectional, in-memory, weak cache for fingerprint {@link NestedSet} associations.
* A bidirectional, in-memory, weak cache for fingerprint {@link NestedSet} associations.
*
* <p>For use by {@link NestedSetStore} to minimize work during {@link NestedSet} (de)serialization.
*
* <p>The cache supports the possibility of semantically different arrays having the same serialized
* representation. For this reason, a context object is included in the key for the fingerprint
* representation. For this reason, a context object is included in the key for the fingerprint
* array mapping. This object should encapsulate all additional context necessary to deserialize a
* {@link NestedSet} element. The array fingerprint mapping, on the other hand, is expected to be
* {@link NestedSet} element. The array fingerprint mapping, on the other hand, is expected to be
* deterministic.
*/
class NestedSetSerializationCache {
Expand All @@ -63,7 +63,7 @@ class NestedSetSerializationCache {
.build();

/** {@code Object[]} contents to fingerprint. Maintained for fast fingerprinting. */
private final Cache<Object[], FingerprintComputationResult> contentsToFingerprint =
private final Cache<Object[], PutOperation> contentsToFingerprint =
Caffeine.newBuilder()
.initialCapacity(SerializationConstants.DESERIALIZATION_POOL_SIZE)
.weakKeys()
Expand Down Expand Up @@ -116,18 +116,17 @@ private void unwrapWhenDone(
new FutureCallback<Object[]>() {
@Override
public void onSuccess(Object[] contents) {
// Store a FingerprintComputationResult so that we can skip fingerprinting this array
// and writing it to storage (it's already there - we just fetched it). Also replace the
// cached future with the unwrapped contents, since the future may be GC'd. If there was
// a call to putIfAbsent with this fingerprint while the future was pending, we may
// overwrite a fingerprint ⟹ array mapping, but this is fine since both arrays have
// the same contents. In this case, it would be nice to also complete the other array's
// write future, but the semantics of SettableFuture makes this difficult (set after
// setFuture has no effect).
putIfAbsent(
contents,
FingerprintComputationResult.create(fingerprint, immediateVoidFuture()),
context);
// Store a PutOperation so that we can skip fingerprinting this array and writing it to
// storage (it's already there - we just fetched it). Also replace the cached future
// with the unwrapped contents, since the future may be GC'd. If there was a call to
// putIfAbsent with this fingerprint while the future was pending, we may overwrite a
// fingerprint ⇒ array mapping, but this is fine since both arrays have the same
// contents. In this case, it would be nice to also complete the other array's write
// future, but the semantics of SettableFuture makes this difficult (set after setFuture
// has no effect).
var unused =
putIfAbsent(
contents, PutOperation.create(fingerprint, immediateVoidFuture()), context);
}

@Override
Expand All @@ -146,25 +145,23 @@ public void onFailure(Throwable t) {
* if the given contents are not known.
*/
@Nullable
FingerprintComputationResult fingerprintForContents(Object[] contents) {
PutOperation fingerprintForContents(Object[] contents) {
return contentsToFingerprint.getIfPresent(contents);
}

/**
* Ensures that a fingerprint ⟺ contents association is cached in both directions.
*
* <p>If the given fingerprint and array are already <em>fully</em> cached, returns the existing
* {@link FingerprintComputationResult}. Otherwise returns {@code null}.
* {@link PutOperation}. Otherwise returns {@code null}.
*
* <p>If the given fingerprint is only <em>partially</em> cached (meaning that {@link
* #putFutureIfAbsent} has been called but the associated future has not yet completed), then the
* cached future is overwritten in favor of the actual contents.
*/
@Nullable
FingerprintComputationResult putIfAbsent(
Object[] contents, FingerprintComputationResult result, Object context) {
FingerprintComputationResult existingResult =
contentsToFingerprint.asMap().putIfAbsent(contents, result);
PutOperation putIfAbsent(Object[] contents, PutOperation result, Object context) {
PutOperation existingResult = contentsToFingerprint.asMap().putIfAbsent(contents, result);
if (existingResult != null) {
return existingResult;
}
Expand Down
Loading

0 comments on commit ea26c85

Please sign in to comment.