Merge branch 'develop' into 11751-determine-node-ids-to-start-without…

…-network

platform-sdk/swirlds-merkledb/src/main/java/com/swirlds/merkledb/MerkleDb.java

@@ -398,7 +398,7 @@ public MerkleDbDataSource copyDataSource(
         final String label = dataSource.getTableName();
         final int tableId = getNextTableId();
         importDataSource(dataSource, tableId, !makeCopyPrimary, makeCopyPrimary); // import to itself == copy
-        return getDataSource(tableId, label, false, offlineUse);
+        return getDataSource(tableId, label, makeCopyPrimary, offlineUse);
     }
 
     private void importDataSource(

platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/cli/GenesisPlatformStateCommand.java

@@ -22,6 +22,7 @@
 import com.swirlds.cli.commands.StateCommand;
 import com.swirlds.cli.utility.AbstractCommand;
 import com.swirlds.cli.utility.SubcommandOf;
+import com.swirlds.common.RosterStateId;
 import com.swirlds.common.context.PlatformContext;
 import com.swirlds.common.merkle.crypto.MerkleCryptoFactory;
 import com.swirlds.config.api.Configuration;
@@ -30,10 +31,14 @@
 import com.swirlds.platform.consensus.SyntheticSnapshot;
 import com.swirlds.platform.state.PlatformStateAccessor;
 import com.swirlds.platform.state.PlatformStateModifier;
+import com.swirlds.platform.state.service.WritableRosterStore;
 import com.swirlds.platform.state.signed.ReservedSignedState;
 import com.swirlds.platform.state.snapshot.DeserializedSignedState;
 import com.swirlds.platform.state.snapshot.SignedStateFileReader;
 import com.swirlds.platform.util.BootstrapUtils;
+import com.swirlds.state.State;
+import com.swirlds.state.spi.CommittableWritableStates;
+import com.swirlds.state.spi.WritableStates;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.concurrent.ExecutionException;
@@ -81,10 +86,21 @@ public Integer call() throws IOException, ExecutionException, InterruptedExcepti
                 System.out.printf("Replacing platform data %n");
                 v.setRound(PlatformStateAccessor.GENESIS_ROUND);
                 v.setSnapshot(SyntheticSnapshot.getGenesisSnapshot());
+
+                // FUTURE WORK: remove once the AddressBook setters are deprecated and the fields are nullified.
+                // For now, we have to keep these calls to ensure RosterRetriever won't fall back to using these values.
                 System.out.printf("Nullifying Address Books %n");
                 v.setAddressBook(null);
                 v.setPreviousAddressBook(null);
             });
+            {
+                System.out.printf("Resetting the RosterService state %n");
+                final State state = (State) reservedSignedState.get().getState().getSwirldState();
+                final WritableStates writableStates = state.getWritableStates(RosterStateId.NAME);
+                final WritableRosterStore writableRosterStore = new WritableRosterStore(writableStates);
+                writableRosterStore.resetRosters();
+                ((CommittableWritableStates) writableStates).commit();
+            }
             System.out.printf("Hashing state %n");
             MerkleCryptoFactory.getInstance()
                     .digestTreeAsync(reservedSignedState.get().getState())

platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/RosterUtils.java

@@ -1,4 +1,19 @@
-// SPDX-License-Identifier: Apache-2.0
+/*
+ * Copyright (C) 2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.swirlds.platform.roster;
 
 import com.hedera.hapi.node.base.ServiceEndpoint;
@@ -293,6 +308,17 @@ public static void setActiveRoster(@NonNull final State state, @NonNull final Ro
         ((CommittableWritableStates) writableStates).commit();
     }
 
+    /**
+     * Formats a human-readable Roster representation, currently using its JSON codec,
+     * or returns {@code null} if the given roster object is null.
+     * @param roster a roster to format
+     * @return roster JSON string, or null
+     */
+    @Nullable
+    public static String toString(@Nullable final Roster roster) {
+        return roster == null ? null : Roster.JSON.toJSON(roster);
+    }
+
     /**
      * Build an Address object out of a given RosterEntry object.
      *

platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/service/WritableRosterStore.java

@@ -33,6 +33,9 @@
 import edu.umd.cs.findbugs.annotations.NonNull;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.StreamSupport;
 
 /**
  * Read-write implementation for accessing rosters states.
@@ -114,19 +117,28 @@ public void putActiveRoster(@NonNull final Roster roster, final long round) {
         requireNonNull(roster);
         RosterValidator.validate(roster);
 
+        final Bytes rosterHash = RosterUtils.hash(roster).getBytes();
+
         // update the roster state
         final RosterState previousRosterState = rosterStateOrDefault();
         final List<RoundRosterPair> roundRosterPairs = new LinkedList<>(previousRosterState.roundRosterPairs());
         if (!roundRosterPairs.isEmpty()) {
             final RoundRosterPair activeRosterPair = roundRosterPairs.getFirst();
+            if (activeRosterPair.activeRosterHash().equals(rosterHash)) {
+                // We're trying to set the exact same active roster, maybe even with the same roundNumber.
+                // This may happen if, for whatever reason, roster updates come from different code paths.
+                // This shouldn't be considered an error because the system wants to use the exact same
+                // roster that is currently active anyway. So we silently ignore such a putActiveRoster request
+                // because it's a no-op:
+                return;
+            }
             if (round < 0 || round <= activeRosterPair.roundNumber()) {
                 throw new IllegalArgumentException("incoming round number = " + round
                         + " must be greater than the round number of the current active roster = "
                         + activeRosterPair.roundNumber() + ".");
             }
         }
-        final Bytes activeRosterHash = RosterUtils.hash(roster).getBytes();
-        roundRosterPairs.addFirst(new RoundRosterPair(round, activeRosterHash));
+        roundRosterPairs.addFirst(new RoundRosterPair(round, rosterHash));
 
         if (roundRosterPairs.size() > MAXIMUM_ROSTER_HISTORY_SIZE) {
             final RoundRosterPair lastRemovedRoster = roundRosterPairs.removeLast();
@@ -149,7 +161,22 @@ public void putActiveRoster(@NonNull final Roster roster, final long round) {
         // so we remove it if it meets removal criteria.
         removeRoster(previousRosterState.candidateRosterHash());
         rosterState.put(newRosterStateBuilder.build());
-        rosterMap.put(ProtoBytes.newBuilder().value(activeRosterHash).build(), roster);
+        rosterMap.put(ProtoBytes.newBuilder().value(rosterHash).build(), roster);
+    }
+
+    /**
+     * Reset the roster state to an empty list and remove all entries from the roster map.
+     * This method is primarily intended to be used in CLI tools that may need to reset
+     * the RosterService states to a vanilla state, for example to reproduce the genesis state.
+     */
+    public void resetRosters() {
+        rosterState.put(RosterState.DEFAULT);
+
+        // To avoid modifying the map while iterating over all the keys, collect them into a list first:
+        final List<ProtoBytes> keys = StreamSupport.stream(
+                        Spliterators.spliteratorUnknownSize(rosterMap.keys(), Spliterator.ORDERED), false)
+                .toList();
+        keys.forEach(rosterMap::remove);
     }
 
     /**

platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/state/signed/StartupStateUtils.java

@@ -35,13 +35,16 @@
 import com.swirlds.platform.config.StateConfig;
 import com.swirlds.platform.crypto.CryptoStatic;
 import com.swirlds.platform.internal.SignedStateLoadingException;
+import com.swirlds.platform.roster.RosterRetriever;
+import com.swirlds.platform.roster.RosterUtils;
 import com.swirlds.platform.state.MerkleRoot;
 import com.swirlds.platform.state.PlatformStateModifier;
 import com.swirlds.platform.state.snapshot.DeserializedSignedState;
 import com.swirlds.platform.state.snapshot.SavedStateInfo;
 import com.swirlds.platform.state.snapshot.SignedStateFilePath;
 import com.swirlds.platform.system.SoftwareVersion;
 import com.swirlds.platform.system.address.AddressBook;
+import com.swirlds.state.State;
 import edu.umd.cs.findbugs.annotations.NonNull;
 import edu.umd.cs.findbugs.annotations.Nullable;
 import java.io.IOException;
@@ -326,7 +329,12 @@ private static ReservedSignedState buildGenesisState(
             @NonNull final MerkleRoot stateRoot) {
 
         if (!configuration.getConfigData(AddressBookConfig.class).useRosterLifecycle()) {
-            initGenesisPlatformState(configuration, stateRoot.getWritablePlatformState(), addressBook, appVersion);
+            initGenesisState(
+                    configuration,
+                    (State) stateRoot.getSwirldState(),
+                    stateRoot.getWritablePlatformState(),
+                    addressBook,
+                    appVersion);
         }
 
         final SignedState signedState = new SignedState(
@@ -335,21 +343,24 @@ private static ReservedSignedState buildGenesisState(
     }
 
     /**
-     * Initializes a genesis platform state.
+     * Initializes a genesis platform state and RosterService state.
      * @param configuration the configuration for this node
+     * @param state the State instance to initialize
      * @param platformState the platform state to initialize
      * @param addressBook the current address book
      * @param appVersion the software version of the app
      */
-    private static void initGenesisPlatformState(
+    private static void initGenesisState(
             final Configuration configuration,
+            final State state,
             final PlatformStateModifier platformState,
             final AddressBook addressBook,
             final SoftwareVersion appVersion) {
+        final long round = 0L;
+
         platformState.bulkUpdate(v -> {
-            v.setAddressBook(addressBook.copy());
             v.setCreationSoftwareVersion(appVersion);
-            v.setRound(0);
+            v.setRound(round);
             v.setLegacyRunningEventHash(null);
             v.setConsensusTimestamp(Instant.ofEpochSecond(0L));
 
@@ -360,5 +371,7 @@ private static void initGenesisPlatformState(
                 v.setFreezeTime(Instant.ofEpochSecond(genesisFreezeTime));
             }
         });
+
+        RosterUtils.setActiveRoster(state, RosterRetriever.buildRoster(addressBook), round);
     }
 }

platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/system/address/AddressBookUtils.java

@@ -1,18 +1,31 @@
-// SPDX-License-Identifier: Apache-2.0
+/*
+ * Copyright (C) 2023-2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.swirlds.platform.system.address;
 
-import static com.swirlds.platform.roster.RosterRetriever.retrieveActiveOrGenesisRoster;
-import static com.swirlds.platform.roster.RosterUtils.buildAddressBook;
 import static com.swirlds.platform.util.BootstrapUtils.detectSoftwareUpgrade;
 
 import com.hedera.hapi.node.base.ServiceEndpoint;
+import com.hedera.hapi.node.state.roster.Roster;
 import com.hedera.pbj.runtime.io.buffer.Bytes;
 import com.swirlds.common.context.PlatformContext;
 import com.swirlds.common.formatting.TextTable;
 import com.swirlds.common.platform.NodeId;
-import com.swirlds.platform.config.AddressBookConfig;
-import com.swirlds.platform.state.MerkleRoot;
-import com.swirlds.platform.state.PlatformStateModifier;
+import com.swirlds.platform.roster.RosterRetriever;
+import com.swirlds.platform.roster.RosterUtils;
 import com.swirlds.platform.state.address.AddressBookInitializer;
 import com.swirlds.platform.state.signed.ReservedSignedState;
 import com.swirlds.platform.system.SoftwareVersion;
@@ -238,31 +251,38 @@ public static ServiceEndpoint endpointFor(@NonNull final String host, final int
         // Initialize the address book from the configuration and platform saved state.
         final AddressBookInitializer addressBookInitializer = new AddressBookInitializer(
                 selfId, version, softwareUpgrade, initialState.get(), bootstrapAddressBook.copy(), platformContext);
+        final State state = (State) initialState.get().getState().getSwirldState();
+
+        if (addressBookInitializer.hasAddressBookChanged()) {
+            if (addressBookInitializer.getPreviousAddressBook() != null) {
+                // We cannot really "update" the previous roster because we don't know the round number
+                // at which it became active. And we shouldn't do that anyway because under normal circumstances
+                // the RosterService tracks the roster history correctly. However, since we're given a non-null
+                // previous AddressBook, and per the current implementation we know it comes from the state,
+                // we might as well validate this fact here just to ensure the update is correct.
+                final Roster previousRoster =
+                        RosterRetriever.buildRoster(addressBookInitializer.getPreviousAddressBook());
+                if (!previousRoster.equals(RosterRetriever.retrieveActiveOrGenesisRoster(state))
+                        && !previousRoster.equals(RosterRetriever.retrievePreviousRoster(state))) {
+                    throw new IllegalStateException(
+                            "The previousRoster in the AddressBookInitializer doesn't match either the active or previous roster in state."
+                                    + " AddressBookInitializer previousRoster = " + RosterUtils.toString(previousRoster)
+                                    + ", state currentRoster = "
+                                    + RosterUtils.toString(RosterRetriever.retrieveActiveOrGenesisRoster(state))
+                                    + ", state previousRoster = "
+                                    + RosterUtils.toString(RosterRetriever.retrievePreviousRoster(state)));
+                }
+            }
 
-        final boolean useRosterLifecycle = platformContext
-                .getConfiguration()
-                .getConfigData(AddressBookConfig.class)
-                .useRosterLifecycle();
-        if (!useRosterLifecycle && addressBookInitializer.hasAddressBookChanged()) {
-            final MerkleRoot state = initialState.get().getState();
-            // Update the address book with the current address book read from config.txt.
-            // Eventually we will not do this, and only transactions will be capable of
-            // modifying the address book.
-            final PlatformStateModifier platformState = state.getWritablePlatformState();
-            platformState.bulkUpdate(v -> {
-                v.setAddressBook(addressBookInitializer.getCurrentAddressBook().copy());
-                v.setPreviousAddressBook(
-                        addressBookInitializer.getPreviousAddressBook() == null
-                                ? null
-                                : addressBookInitializer
-                                        .getPreviousAddressBook()
-                                        .copy());
-            });
+            RosterUtils.setActiveRoster(
+                    state,
+                    RosterRetriever.buildRoster(addressBookInitializer.getCurrentAddressBook()),
+                    RosterRetriever.getRound(state));
         }
 
         // At this point the initial state must have the current address book set.  If not, something is wrong.
-        final AddressBook addressBook = buildAddressBook(retrieveActiveOrGenesisRoster(
-                (State) initialState.get().getState().getSwirldState()));
+        final AddressBook addressBook =
+                RosterUtils.buildAddressBook(RosterRetriever.retrieveActiveOrGenesisRoster(state));
         if (addressBook == null) {
             throw new IllegalStateException("The current address book of the initial state is null.");
         }

platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/state/SignedStateTests.java

@@ -32,10 +32,11 @@
 import com.swirlds.common.test.fixtures.platform.TestPlatformContextBuilder;
 import com.swirlds.merkledb.MerkleDb;
 import com.swirlds.platform.crypto.SignatureVerifier;
+import com.swirlds.platform.roster.RosterUtils;
 import com.swirlds.platform.state.signed.ReservedSignedState;
 import com.swirlds.platform.state.signed.SignedState;
 import com.swirlds.platform.system.BasicSoftwareVersion;
-import com.swirlds.platform.system.address.AddressBook;
+import com.swirlds.platform.test.fixtures.addressbook.RandomRosterBuilder;
 import com.swirlds.platform.test.fixtures.state.RandomSignedStateGenerator;
 import java.time.Duration;
 import java.util.ArrayList;
@@ -73,14 +74,15 @@ void tearDown() {
      * @param reserveCallback this method is called when the State is reserved
      * @param releaseCallback this method is called when the State is released
      */
-    private PlatformMerkleStateRoot buildMockState(final Runnable reserveCallback, final Runnable releaseCallback) {
+    private PlatformMerkleStateRoot buildMockState(
+            final Random random, final Runnable reserveCallback, final Runnable releaseCallback) {
         final var real = new PlatformMerkleStateRoot(
                 FAKE_MERKLE_STATE_LIFECYCLES, version -> new BasicSoftwareVersion(version.major()));
-        FAKE_MERKLE_STATE_LIFECYCLES.initPlatformState(real);
+        FAKE_MERKLE_STATE_LIFECYCLES.initStates(real);
+        RosterUtils.setActiveRoster(real, RandomRosterBuilder.create(random).build(), 0L);
         final PlatformMerkleStateRoot state = spy(real);
 
         final PlatformStateModifier platformState = new PlatformState();
-        platformState.setAddressBook(mock(AddressBook.class));
         when(state.getWritablePlatformState()).thenReturn(platformState);
         if (reserveCallback != null) {
             doAnswer(invocation -> {
@@ -112,6 +114,7 @@ void reservationTest() throws InterruptedException {
         final AtomicBoolean released = new AtomicBoolean(false);
 
         final PlatformMerkleStateRoot state = buildMockState(
+                random,
                 () -> {
                     assertFalse(reserved.get(), "should only be reserved once");
                     reserved.set(true);
@@ -173,6 +176,7 @@ void noGarbageCollectorTest() {
         final Thread mainThread = Thread.currentThread();
 
         final PlatformMerkleStateRoot state = buildMockState(
+                random,
                 () -> {
                     assertFalse(reserved.get(), "should only be reserved once");
                     reserved.set(true);

platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/addressbook/RandomAddressBuilder.java

@@ -82,7 +82,7 @@ public Address build() {
         }
 
         if (port == null) {
-            port = random.nextInt(0, 65535);
+            port = random.nextInt(1, 65535);
         }
 
         if (hostname == null) {

platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/addressbook/RandomRosterEntryBuilder.java

@@ -79,7 +79,7 @@ public RosterEntry build() {
         }
 
         if (port == null) {
-            port = random.nextInt(0, 65535);
+            port = random.nextInt(1, 65535);
         }
 
         if (hostname == null) {