Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move storage to new BlobSidecar #7737

Merged
merged 5 commits into from
Nov 21, 2023
Merged

Move storage to new BlobSidecar #7737

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7a96096
Move storage to new BlobSidecar
zilm13 Nov 20, 2023
27046fb
Merge branch 'master' into new-blobsidecar-storage
zilm13 Nov 20, 2023
468f0d6
Fix assemble
zilm13 Nov 20, 2023
31eac62
Merge branch 'master' into new-blobsidecar-storage
zilm13 Nov 21, 2023
b85a343
Revert accidentally removed line
zilm13 Nov 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions ...nc/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import java.util.Optional;
import java.util.stream.Collectors;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
Expand Down Expand Up @@ -185,7 +184,6 @@ public void run_returnAllBlocksOnFirstRequest() {
}

@Test
@Disabled("TODO for new BlobSidecars")
public void run_returnAllBlocksAndBlobSidecarsOnFirstRequest() {
when(blobSidecarManager.isAvailabilityRequiredAtSlot(any())).thenReturn(true);

Expand All @@ -208,7 +206,6 @@ public void run_returnAllBlocksAndBlobSidecarsOnFirstRequest() {
}

@Test
@Disabled("TODO for new BlobSidecars")
public void run_failsOnBlobSidecarsValidationFailure() {
when(blobSidecarManager.isAvailabilityRequiredAtSlot(any())).thenReturn(true);
when(blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(any(), anyList()))
Expand Down
19 changes: 11 additions & 8 deletions ...ech/pegasys/teku/beaconrestapi/tekuv1/beacon/GetAllBlobSidecarsAtSlotIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import okhttp3.Response;
import org.apache.tuweni.bytes.Bytes;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import tech.pegasys.teku.beaconrestapi.AbstractDataBackedRestAPIIntegrationTest;
import tech.pegasys.teku.beaconrestapi.handlers.tekuv1.beacon.GetAllBlobSidecarsAtSlot;
Expand All @@ -39,12 +40,14 @@
import tech.pegasys.teku.infrastructure.unsigned.UInt64;
import tech.pegasys.teku.spec.SpecMilestone;
import tech.pegasys.teku.spec.config.SpecConfigDeneb;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarOld;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarSchemaOld;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState;
import tech.pegasys.teku.spec.generator.ChainBuilder;
import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb;

@Disabled("Fix for new BlobSidecars")
public class GetAllBlobSidecarsAtSlotIntegrationTest
extends AbstractDataBackedRestAPIIntegrationTest {

Expand All @@ -64,14 +67,14 @@ public void shouldGetNonCanonicalBlobSidecars() throws IOException {
final ChainBuilder fork = chainBuilder.fork();
final SignedBlockAndState nonCanonicalBlock = fork.generateNextBlock(chainUpdater.blockOptions);

final List<BlobSidecarOld> nonCanonicalBlobSidecars =
fork.getBlobSidecarsOld(nonCanonicalBlock.getRoot());
final List<BlobSidecar> nonCanonicalBlobSidecars =
fork.getBlobSidecars(nonCanonicalBlock.getRoot());
chainUpdater.saveBlock(nonCanonicalBlock, nonCanonicalBlobSidecars);

final SignedBlockAndState canonicalBlock =
chainBuilder.generateNextBlock(1, chainUpdater.blockOptions);
final List<BlobSidecarOld> canonicalBlobSidecars =
chainBuilder.getBlobSidecarsOld(canonicalBlock.getRoot());
final List<BlobSidecar> canonicalBlobSidecars =
chainBuilder.getBlobSidecars(canonicalBlock.getRoot());
chainUpdater.saveBlock(canonicalBlock, canonicalBlobSidecars);
chainUpdater.updateBestBlock(canonicalBlock);
chainUpdater.finalizeEpoch(targetSlot.plus(1));
Expand Down Expand Up @@ -106,14 +109,14 @@ public void shouldGetNonCanonicalBlobSidecarsAsSsz() throws Exception {
final ChainBuilder fork = chainBuilder.fork();
final SignedBlockAndState nonCanonicalBlock = fork.generateNextBlock(chainUpdater.blockOptions);

final List<BlobSidecarOld> nonCanonicalBlobSidecars =
fork.getBlobSidecarsOld(nonCanonicalBlock.getRoot());
final List<BlobSidecar> nonCanonicalBlobSidecars =
fork.getBlobSidecars(nonCanonicalBlock.getRoot());
chainUpdater.saveBlock(nonCanonicalBlock, nonCanonicalBlobSidecars);

final SignedBlockAndState canonicalBlock =
chainBuilder.generateNextBlock(1, chainUpdater.blockOptions);
final List<BlobSidecarOld> canonicalBlobSidecars =
chainBuilder.getBlobSidecarsOld(canonicalBlock.getRoot());
final List<BlobSidecar> canonicalBlobSidecars =
chainBuilder.getBlobSidecars(canonicalBlock.getRoot());
chainUpdater.saveBlock(canonicalBlock, canonicalBlobSidecars);
chainUpdater.updateBestBlock(canonicalBlock);
chainUpdater.finalizeEpoch(targetSlot.plus(1));
Expand Down
29 changes: 23 additions & 6 deletions ...n-test/java/tech/pegasys/teku/beaconrestapi/v1/beacon/GetBlobSidecarsIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,13 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import okhttp3.Response;
import org.apache.tuweni.bytes.Bytes;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import tech.pegasys.teku.beaconrestapi.AbstractDataBackedRestAPIIntegrationTest;
import tech.pegasys.teku.beaconrestapi.handlers.v1.beacon.GetBlobSidecars;
Expand All @@ -39,14 +42,25 @@
import tech.pegasys.teku.infrastructure.unsigned.UInt64;
import tech.pegasys.teku.spec.SpecMilestone;
import tech.pegasys.teku.spec.config.SpecConfigDeneb;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarOld;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarSchemaOld;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState;
import tech.pegasys.teku.spec.generator.ChainBuilder;
import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb;

@Disabled("Remove blobSidecarRepacker and fix for new BlobSidecars")
public class GetBlobSidecarsIntegrationTest extends AbstractDataBackedRestAPIIntegrationTest {

@Deprecated
private final Supplier<Function<BlobSidecar, BlobSidecarOld>> blobSidecarRepacker =
() ->
blobSidecar ->
SchemaDefinitionsDeneb.required(
spec.forMilestone(SpecMilestone.DENEB).getSchemaDefinitions())
.getBlobSidecarOldSchema()
.create(blobSidecar);

@BeforeEach
public void beforeEach() {
startRestApiAtGenesisStoringNonCanonicalBlocks(SpecMilestone.DENEB);
Expand All @@ -63,7 +77,9 @@ public void shouldGetBlobSidecars() throws Exception {
final SignedBlockAndState lastBlock = chainUpdater.advanceChainUntil(targetSlot);
chainUpdater.updateBestBlock(lastBlock);
final List<BlobSidecarOld> expected =
recentChainData.getBlobSidecars(lastBlock.getSlotAndBlockRoot()).get();
recentChainData.getBlobSidecars(lastBlock.getSlotAndBlockRoot()).get().stream()
.map(blobSidecar -> blobSidecarRepacker.get().apply(blobSidecar))
.toList();

final Response responseAll = get("head");
assertThat(responseAll.code()).isEqualTo(SC_OK);
Expand Down Expand Up @@ -116,7 +132,9 @@ public void shouldGetBlobSidecarsAsSsz() throws Exception {
final SignedBlockAndState lastBlock = chainUpdater.advanceChainUntil(targetSlot);
chainUpdater.updateBestBlock(lastBlock);
final List<BlobSidecarOld> expected =
recentChainData.getBlobSidecars(lastBlock.getSlotAndBlockRoot()).get();
recentChainData.getBlobSidecars(lastBlock.getSlotAndBlockRoot()).get().stream()
.map(blobSidecar -> blobSidecarRepacker.get().apply(blobSidecar))
.toList();

final Response response = get("head", OCTET_STREAM);
assertThat(response.code()).isEqualTo(SC_OK);
Expand Down Expand Up @@ -149,14 +167,13 @@ public void shouldGetNonCanonicalBlobSidecarsByRoot() throws IOException {
final ChainBuilder fork = chainBuilder.fork();
final SignedBlockAndState nonCanonicalBlock = fork.generateNextBlock(chainUpdater.blockOptions);

final List<BlobSidecarOld> nonCanonicalBlobSidecars =
fork.getBlobSidecarsOld(nonCanonicalBlock.getRoot());
final List<BlobSidecar> nonCanonicalBlobSidecars =
fork.getBlobSidecars(nonCanonicalBlock.getRoot());
chainUpdater.saveBlock(nonCanonicalBlock, nonCanonicalBlobSidecars);

final SignedBlockAndState canonicalBlock =
chainBuilder.generateNextBlock(1, chainUpdater.blockOptions);
chainUpdater.saveBlock(
canonicalBlock, chainBuilder.getBlobSidecarsOld(canonicalBlock.getRoot()));
chainUpdater.saveBlock(canonicalBlock, chainBuilder.getBlobSidecars(canonicalBlock.getRoot()));
chainUpdater.updateBestBlock(canonicalBlock);

final Response byRootResponse =
Expand Down
2 changes: 1 addition & 1 deletion ...java/tech/pegasys/teku/beaconrestapi/handlers/tekuv1/beacon/GetAllBlobSidecarsAtSlot.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,8 @@
public void handleRequest(RestApiRequest request) throws JsonProcessingException {
final List<UInt64> indices = request.getQueryParameterList(BLOB_INDICES_PARAMETER);
final SafeFuture<Optional<List<BlobSidecarOld>>> future =
chainDataProvider.getAllBlobSidecarsAtSlot(
chainDataProvider.getAllBlobSidecarsAtSlotOld(
request.getPathParameter(SLOT_PARAMETER), indices);

Check notice

Code scanning / CodeQL

Deprecated method or constructor invocation Note

Invoking
ChainDataProvider.getAllBlobSidecarsAtSlotOld
Loading
should be avoided because it has been deprecated.

request.respondAsync(
future.thenApply(
Expand Down
2 changes: 1 addition & 1 deletion ...api/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/beacon/GetBlobSidecars.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@
public void handleRequest(RestApiRequest request) throws JsonProcessingException {
final List<UInt64> indices = request.getQueryParameterList(BLOB_INDICES_PARAMETER);
final SafeFuture<Optional<List<BlobSidecarOld>>> future =
chainDataProvider.getBlobSidecars(request.getPathParameter(PARAMETER_BLOCK_ID), indices);
chainDataProvider.getBlobSidecarsOld(request.getPathParameter(PARAMETER_BLOCK_ID), indices);

Check notice

Code scanning / CodeQL

Deprecated method or constructor invocation Note

Invoking
ChainDataProvider.getBlobSidecarsOld
Loading
should be avoided because it has been deprecated.

request.respondAsync(
future.thenApply(
Expand Down
2 changes: 1 addition & 1 deletion .../tech/pegasys/teku/beaconrestapi/handlers/tekuv1/beacon/GetAllBlobSidecarsAtSlotTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@
void shouldReturnAllBlobSidecarsAtSlot() throws JsonProcessingException {
final List<BlobSidecarOld> nonCanonicalBlobSidecars =
dataStructureUtil.randomBlobSidecarsOld(4);
when(chainDataProvider.getAllBlobSidecarsAtSlot(eq(UInt64.ONE), eq(indices)))
when(chainDataProvider.getAllBlobSidecarsAtSlotOld(eq(UInt64.ONE), eq(indices)))

Check notice

Code scanning / CodeQL

Deprecated method or constructor invocation Note test

Invoking
ChainDataProvider.getAllBlobSidecarsAtSlotOld
Loading
should be avoided because it has been deprecated.
.thenReturn(SafeFuture.completedFuture(Optional.of(nonCanonicalBlobSidecars)));

handler.handleRequest(request);
Expand Down
18 changes: 17 additions & 1 deletion ...src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/beacon/GetBlobSidecarsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,17 @@
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
import java.util.function.Supplier;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import tech.pegasys.teku.beaconrestapi.AbstractMigratedBeaconHandlerWithChainDataProviderTest;
import tech.pegasys.teku.spec.SpecMilestone;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarOld;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
import tech.pegasys.teku.spec.datastructures.metadata.ObjectAndMetaData;
import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb;

class GetBlobSidecarsTest extends AbstractMigratedBeaconHandlerWithChainDataProviderTest {

Expand All @@ -47,6 +51,15 @@ void setup() {
request.setPathParameter("block_id", "head");
}

@Deprecated
private final Supplier<Function<BlobSidecar, BlobSidecarOld>> blobSidecarRepacker =
() ->
blobSidecar ->
SchemaDefinitionsDeneb.required(
spec.forMilestone(SpecMilestone.DENEB).getSchemaDefinitions())
.getBlobSidecarOldSchema()
.create(blobSidecar);

@Test
void shouldReturnBlobSidecars()
throws JsonProcessingException, ExecutionException, InterruptedException {
Expand All @@ -56,7 +69,10 @@ void shouldReturnBlobSidecars()
combinedChainDataClient
.getBlobSidecars(
blockAndMetaData.getData().getSlotAndBlockRoot(), Collections.emptyList())
.get();
.get()
.stream()
.map(blobSidecar -> blobSidecarRepacker.get().apply(blobSidecar))
.toList();

handler.handleRequest(request);

Expand Down
50 changes: 48 additions & 2 deletions data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.util.function.Function;
import java.util.function.IntPredicate;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.IntStream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand Down Expand Up @@ -58,6 +59,7 @@
import tech.pegasys.teku.infrastructure.unsigned.UInt64;
import tech.pegasys.teku.spec.Spec;
import tech.pegasys.teku.spec.SpecMilestone;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarOld;
import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
Expand All @@ -74,6 +76,7 @@
import tech.pegasys.teku.spec.logic.common.statetransition.epoch.status.ValidatorStatuses;
import tech.pegasys.teku.spec.logic.common.statetransition.exceptions.EpochProcessingException;
import tech.pegasys.teku.spec.logic.common.statetransition.exceptions.SlotProcessingException;
import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb;
import tech.pegasys.teku.storage.client.ChainDataUnavailableException;
import tech.pegasys.teku.storage.client.CombinedChainDataClient;
import tech.pegasys.teku.storage.client.RecentChainData;
Expand Down Expand Up @@ -179,18 +182,61 @@ public SafeFuture<Optional<ObjectAndMetaData<SignedBeaconBlock>>> getBlock(
return fromBlock(blockIdParam, Function.identity());
}

public SafeFuture<Optional<List<BlobSidecarOld>>> getBlobSidecars(
public SafeFuture<Optional<List<BlobSidecar>>> getBlobSidecars(
final String blockIdParam, final List<UInt64> indices) {
return blobSidecarSelectorFactory
.createSelectorForBlockId(blockIdParam)
.getBlobSidecars(indices);
}

public SafeFuture<Optional<List<BlobSidecarOld>>> getAllBlobSidecarsAtSlot(
@Deprecated
private final Supplier<Function<BlobSidecar, BlobSidecarOld>> blobSidecarRepacker =
new Supplier<>() {
@Override
public Function<BlobSidecar, BlobSidecarOld> get() {
return blobSidecar ->
SchemaDefinitionsDeneb.required(
spec.forMilestone(SpecMilestone.DENEB).getSchemaDefinitions())
.getBlobSidecarOldSchema()
.create(blobSidecar);
}
};

@Deprecated
public SafeFuture<Optional<List<BlobSidecarOld>>> getBlobSidecarsOld(
final String blockIdParam, final List<UInt64> indices) {
return blobSidecarSelectorFactory
.createSelectorForBlockId(blockIdParam)
.getBlobSidecars(indices)
.thenApply(
maybeBlobSidecars ->
maybeBlobSidecars.map(
blobSidecars ->
blobSidecars.stream()
.map(blobSidecar -> blobSidecarRepacker.get().apply(blobSidecar))
.toList()));
}

public SafeFuture<Optional<List<BlobSidecar>>> getAllBlobSidecarsAtSlot(
final UInt64 slot, final List<UInt64> indices) {
return blobSidecarSelectorFactory.slotSelectorForAll(slot).getBlobSidecars(indices);
}

@Deprecated
public SafeFuture<Optional<List<BlobSidecarOld>>> getAllBlobSidecarsAtSlotOld(
final UInt64 slot, final List<UInt64> indices) {
return blobSidecarSelectorFactory
.slotSelectorForAll(slot)
.getBlobSidecars(indices)
.thenApply(
maybeBlobSidecars ->
maybeBlobSidecars.map(
blobSidecars ->
blobSidecars.stream()
.map(blobSidecar -> blobSidecarRepacker.get().apply(blobSidecar))
.toList()));
}

public SafeFuture<Optional<ObjectAndMetaData<Bytes32>>> getBlockRoot(final String blockIdParam) {
return fromBlock(blockIdParam, SignedBeaconBlock::getRoot);
}
Expand Down
4 changes: 2 additions & 2 deletions data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
import java.util.Optional;
import tech.pegasys.teku.infrastructure.async.SafeFuture;
import tech.pegasys.teku.infrastructure.unsigned.UInt64;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarOld;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;

public interface BlobSidecarSelector {
SafeFuture<Optional<List<BlobSidecarOld>>> getBlobSidecars(List<UInt64> indices);
SafeFuture<Optional<List<BlobSidecar>>> getBlobSidecars(List<UInt64> indices);
}
8 changes: 4 additions & 4 deletions ...provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import tech.pegasys.teku.api.AbstractSelectorFactory;
import tech.pegasys.teku.infrastructure.async.SafeFuture;
import tech.pegasys.teku.infrastructure.unsigned.UInt64;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarOld;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot;
import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.BeaconBlockBodyDeneb;
Expand Down Expand Up @@ -99,7 +99,7 @@ public BlobSidecarSelector slotSelectorForAll(final UInt64 slot) {
blobSidecars.isEmpty() ? Optional.empty() : Optional.of(blobSidecars));
}

private SafeFuture<Optional<List<BlobSidecarOld>>> getBlobSidecarsForBlock(
private SafeFuture<Optional<List<BlobSidecar>>> getBlobSidecarsForBlock(
final Optional<SignedBeaconBlock> maybeBlock, final List<UInt64> indices) {
if (maybeBlock.isEmpty()) {
return SafeFuture.completedFuture(Optional.empty());
Expand All @@ -116,12 +116,12 @@ private SafeFuture<Optional<List<BlobSidecarOld>>> getBlobSidecarsForBlock(
return getBlobSidecars(block.getSlotAndBlockRoot(), indices);
}

private SafeFuture<Optional<List<BlobSidecarOld>>> getBlobSidecars(
private SafeFuture<Optional<List<BlobSidecar>>> getBlobSidecars(
final SlotAndBlockRoot slotAndBlockRoot, final List<UInt64> indices) {
return client.getBlobSidecars(slotAndBlockRoot, indices).thenApply(Optional::of);
}

private SafeFuture<Optional<List<BlobSidecarOld>>> getBlobSidecars(
private SafeFuture<Optional<List<BlobSidecar>>> getBlobSidecars(
final UInt64 slot, final List<UInt64> indices) {
return client.getBlobSidecars(slot, indices).thenApply(Optional::of);
}
Expand Down
Loading
Loading