[WEEKLY RELEASE] HotShot 0.5.57 (#1566)

Closes #<ISSUE_NUMBER>
<!-- These comments should help create a useful PR message, please
delete any remaining comments before opening the PR. -->
<!-- If there is no issue number make sure to describe clearly *why*
this PR is necessary. -->
<!-- Mention open questions, remaining TODOs, if any -->

### This PR:
<!-- Describe what this PR adds to this repo and why -->
<!-- E.g. -->
<!-- * Implements feature 1 -->
<!-- * Fixes bug 3 -->

### This PR does not:
<!-- Describe what is out of scope for this PR, if applicable. Leave
this section blank if it's not applicable -->
<!-- This section helps avoid the reviewer having to needlessly point
out missing parts -->
<!-- * Implement feature 3 because that feature is blocked by Issue 4
-->
<!-- * Implement xyz because that is tracked in issue #123. -->
<!-- * Address xzy for which I opened issue #456 -->

### Key places to review:
<!-- Describe key places for reviewers to pay close attention to -->
<!-- * file.rs, `add_integers` function -->
<!-- Or directly comment on those files/lines to make it easier for the
reviewers -->

<!-- ### How to test this PR:  -->
<!-- Optional, uncomment the above line if this is relevant to your PR
-->
<!-- If your PR is fully tested through CI there is no need to add this
section -->
<!-- * E.g. `just test` -->

<!-- ### Things tested -->
<!-- Anything that was manually tested (that is not tested in CI). -->
<!-- E.g. building/running of docker containers. Changes to docker demo,
... -->
<!-- Especially mention anything untested, with reasoning and link an
issue to resolve this. -->

<!-- Complete the following items before creating this PR -->
<!-- [ ] Issue linked or PR description mentions why this change is
necessary. -->
<!-- [ ] PR description is clear enough for reviewers. -->
<!-- [ ] Documentation for changes (additions) has been updated (added).
-->
<!-- [ ] If this is a draft it is marked as "draft".  -->

<!-- To make changes to this template edit
https://github.com/EspressoSystems/.github/blob/main/PULL_REQUEST_TEMPLATE.md
-->

Cargo.toml

@@ -47,18 +47,18 @@ dotenvy = "0.15"
 ethers = { version = "2.0", features = ["solc"] }
 futures = "0.3"
 
-hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
+hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" }
 # Hotshot imports
-hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "rc-0.5.56" }
-hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.25" }
-hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.25" }
-hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
-hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.34" }
-hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
+hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "rc-0.5.57" }
+hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.27" }
+hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.27" }
+hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" }
+hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.37" }
+hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" }
 hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" }
-hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
-hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
-hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
+hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" }
+hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" }
+hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" }
 hotshot-contract-adapter = { version = "0.1.0", path = "contracts/rust/adapter" }
 
 # Push CDN imports
@@ -99,10 +99,10 @@ jf-relation = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4
 jf-utils = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.5" }
 snafu = "0.8"
 strum = { version = "0.26", features = ["derive"] }
-surf-disco = "0.7"
+surf-disco = "0.8"
 tagged-base64 = "0.4"
+tide-disco = "0.8"
 thiserror = "1.0.61"
-tide-disco = "0.7"
 time = "0.3"
 tracing = "0.1"
 bytesize = "1.3"

sequencer/api/migrations/V32__saved_proposals.sql

@@ -0,0 +1,4 @@
+CREATE TABLE quorum_proposals (
+    view BIGINT PRIMARY KEY,
+    data BYTEA
+);

sequencer/src/api/endpoints.rs

@@ -108,7 +108,7 @@ where
                 )
                 .context(CustomSnafu {
                     message: format!("failed to make proof for namespace {ns_id}"),
-                    status: StatusCode::NotFound,
+                    status: StatusCode::NOT_FOUND,
                 })?;
 
             let transactions = if let NamespaceProof::Existence {
@@ -184,7 +184,7 @@ where
             if tx.namespace() > NamespaceId::from(u32::MAX as u64) {
                 return Err(Error::Custom {
                     message: "Transaction namespace > u32::MAX".to_string(),
-                    status: StatusCode::BadRequest,
+                    status: StatusCode::BAD_REQUEST,
                 });
             }
 
@@ -221,7 +221,7 @@ where
                 .get_state_signature(height)
                 .await
                 .ok_or(tide_disco::Error::catch_all(
-                    StatusCode::NotFound,
+                    StatusCode::NOT_FOUND,
                     "Signature not found.".to_owned(),
                 ))
         }
@@ -252,15 +252,15 @@ where
                 .map_err(Error::from_request_error)?;
             let account = account.parse().map_err(|err| {
                 Error::catch_all(
-                    StatusCode::BadRequest,
+                    StatusCode::BAD_REQUEST,
                     format!("malformed account {account}: {err}"),
                 )
             })?;
 
             state
                 .get_account(height, ViewNumber::new(view), account)
                 .await
-                .map_err(|err| Error::catch_all(StatusCode::NotFound, format!("{err:#}")))
+                .map_err(|err| Error::catch_all(StatusCode::NOT_FOUND, format!("{err:#}")))
         }
         .boxed()
     })?
@@ -276,7 +276,7 @@ where
             state
                 .get_frontier(height, ViewNumber::new(view))
                 .await
-                .map_err(|err| Error::catch_all(StatusCode::NotFound, format!("{err:#}")))
+                .map_err(|err| Error::catch_all(StatusCode::NOT_FOUND, format!("{err:#}")))
         }
         .boxed()
     })?;
@@ -314,7 +314,7 @@ where
     let mut api = Api::<S, Error, Ver>::new(toml)?;
 
     let env_variables = get_public_env_vars()
-        .map_err(|err| Error::catch_all(StatusCode::InternalServerError, format!("{err:#}")))?;
+        .map_err(|err| Error::catch_all(StatusCode::INTERNAL_SERVER_ERROR, format!("{err:#}")))?;
 
     api.get("hotshot", |_, state| {
         async move { Ok(state.get_config().await) }.boxed()

sequencer/src/bin/nasty-client.rs

@@ -510,7 +510,7 @@ impl<T: Queryable> ResourceManager<T> {
         let elapsed = start.elapsed();
 
         let status = match &res {
-            Ok(_) => StatusCode::Ok,
+            Ok(_) => StatusCode::OK,
             Err(err) => err.status(),
         };
         tracing::debug!("<- GET {path} {} ({elapsed:?})", u16::from(status));

sequencer/src/hotshot_commitment.rs

@@ -320,7 +320,7 @@ mod test {
                 .flatten()
                 .ok_or_else(|| {
                     Self::Error::catch_all(
-                        StatusCode::NotFound,
+                        StatusCode::NOT_FOUND,
                         format!("no leaf for height {height}"),
                     )
                 })

sequencer/src/lib.rs

@@ -54,7 +54,7 @@ use hotshot_orchestrator::{
 };
 use hotshot_types::{
     consensus::CommitmentMap,
-    data::{DaProposal, VidDisperseShare, ViewNumber},
+    data::{DaProposal, QuorumProposal, VidDisperseShare, ViewNumber},
     event::HotShotAction,
     light_client::{StateKeyPair, StateSignKey},
     message::Proposal,
@@ -162,6 +162,13 @@ impl<P: SequencerPersistence> Storage<SeqTypes> for Arc<RwLock<P>> {
             .update_undecided_state(leaves, state)
             .await
     }
+
+    async fn append_proposal(
+        &self,
+        proposal: &Proposal<SeqTypes, QuorumProposal<SeqTypes>>,
+    ) -> anyhow::Result<()> {
+        self.write().await.append_quorum_proposal(proposal).await
+    }
 }
 
 #[derive(Debug, Clone)]

sequencer/src/persistence.rs

@@ -20,7 +20,7 @@ use hotshot::{
 };
 use hotshot_types::{
     consensus::CommitmentMap,
-    data::{DaProposal, VidDisperseShare},
+    data::{DaProposal, QuorumProposal, VidDisperseShare},
     event::{HotShotAction, LeafInfo},
     message::Proposal,
     simple_certificate::QuorumCertificate,
@@ -87,6 +87,11 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static {
         &self,
     ) -> anyhow::Result<Option<(CommitmentMap<Leaf>, BTreeMap<ViewNumber, View<SeqTypes>>)>>;
 
+    /// Load the proposals saved by consensus
+    async fn load_quorum_proposals(
+        &self,
+    ) -> anyhow::Result<Option<BTreeMap<ViewNumber, Proposal<SeqTypes, QuorumProposal<SeqTypes>>>>>;
+
     async fn load_vid_share(
         &self,
         view: ViewNumber,
@@ -168,20 +173,29 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static {
             .context("loading undecided state")?
             .unwrap_or_default();
 
+        let saved_proposals = self
+            .load_quorum_proposals()
+            .await
+            .context("loading saved proposals")
+            .unwrap_or_default()
+            .unwrap_or_default();
+
         tracing::info!(
             ?leaf,
             ?view,
             ?high_qc,
             ?validated_state,
             ?undecided_leaves,
             ?undecided_state,
+            ?saved_proposals,
             "loaded consensus state"
         );
         Ok(HotShotInitializer::from_reload(
             leaf,
             state,
             validated_state,
             view,
+            saved_proposals,
             high_qc,
             undecided_leaves.into_values().collect(),
             undecided_state,
@@ -233,6 +247,10 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static {
         leaves: CommitmentMap<Leaf>,
         state: BTreeMap<ViewNumber, View<SeqTypes>>,
     ) -> anyhow::Result<()>;
+    async fn append_quorum_proposal(
+        &mut self,
+        proposal: &Proposal<SeqTypes, QuorumProposal<SeqTypes>>,
+    ) -> anyhow::Result<()>;
 }
 
 #[cfg(test)]
@@ -368,13 +386,29 @@ mod persistence_tests {
         let bytes = payload.encode().to_vec();
         let disperse = vid_scheme(2).disperse(bytes).unwrap();
         let (pubkey, privkey) = BLSPubKey::generated_from_seed_indexed([0; 32], 1);
+        let signature = PubKey::sign(&privkey, &[]).unwrap();
         let mut vid = VidDisperseShare::<SeqTypes> {
             view_number: ViewNumber::new(1),
             payload_commitment: Default::default(),
             share: disperse.shares[0].clone(),
             common: disperse.common,
             recipient_key: pubkey,
         };
+        let mut quorum_proposal = Proposal {
+            data: QuorumProposal::<SeqTypes> {
+                block_header: leaf.block_header().clone(),
+                view_number: ViewNumber::genesis(),
+                justify_qc: QuorumCertificate::genesis(
+                    &ValidatedState::default(),
+                    &NodeState::mock(),
+                )
+                .await,
+                upgrade_certificate: None,
+                proposal_certificate: None,
+            },
+            signature,
+            _pd: Default::default(),
+        };
 
         let vid_share1 = vid.clone().to_proposal(&privkey).unwrap().clone();
 
@@ -457,6 +491,70 @@ mod persistence_tests {
             Some(da_proposal3.clone())
         );
 
+        let quorum_proposal1 = quorum_proposal.clone();
+        storage
+            .append_quorum_proposal(&quorum_proposal1)
+            .await
+            .unwrap();
+
+        assert_eq!(
+            storage.load_quorum_proposals().await.unwrap(),
+            Some(BTreeMap::from_iter([(
+                ViewNumber::genesis(),
+                quorum_proposal1.clone()
+            )]))
+        );
+
+        quorum_proposal.data.view_number = ViewNumber::new(1);
+        let quorum_proposal2 = quorum_proposal.clone();
+        storage
+            .append_quorum_proposal(&quorum_proposal2)
+            .await
+            .unwrap();
+
+        assert_eq!(
+            storage.load_quorum_proposals().await.unwrap(),
+            Some(BTreeMap::from_iter([
+                (ViewNumber::genesis(), quorum_proposal1.clone()),
+                (ViewNumber::new(1), quorum_proposal2.clone())
+            ]))
+        );
+
+        quorum_proposal.data.view_number = ViewNumber::new(2);
+        let quorum_proposal3 = quorum_proposal.clone();
+        storage
+            .append_quorum_proposal(&quorum_proposal3)
+            .await
+            .unwrap();
+
+        assert_eq!(
+            storage.load_quorum_proposals().await.unwrap(),
+            Some(BTreeMap::from_iter([
+                (ViewNumber::genesis(), quorum_proposal1.clone()),
+                (ViewNumber::new(1), quorum_proposal2.clone()),
+                (ViewNumber::new(2), quorum_proposal3.clone())
+            ]))
+        );
+
+        quorum_proposal.data.view_number = ViewNumber::new(10);
+
+        // This one should stick around after GC runs.
+        let quorum_proposal4 = quorum_proposal.clone();
+        storage
+            .append_quorum_proposal(&quorum_proposal4)
+            .await
+            .unwrap();
+
+        assert_eq!(
+            storage.load_quorum_proposals().await.unwrap(),
+            Some(BTreeMap::from_iter([
+                (ViewNumber::genesis(), quorum_proposal1),
+                (ViewNumber::new(1), quorum_proposal2),
+                (ViewNumber::new(2), quorum_proposal3),
+                (ViewNumber::new(10), quorum_proposal4.clone())
+            ]))
+        );
+
         // Test garbage collection
         // Deleting da proposals and vid shares with view number <=2
         storage.collect_garbage(ViewNumber::new(2)).await.unwrap();
@@ -482,5 +580,14 @@ mod persistence_tests {
             storage.load_vid_share(ViewNumber::new(3)).await.unwrap(),
             Some(vid_share3)
         );
+
+        let proposals = storage.load_quorum_proposals().await.unwrap();
+        assert_eq!(
+            proposals,
+            Some(BTreeMap::from_iter([(
+                ViewNumber::new(10),
+                quorum_proposal4
+            )]))
+        )
     }
 }