Implements new ProveCommit batch+aggregate for direct onboarding (#1380)

Co-authored-by: zenground0 <ZenGround0@users.noreply.github.com>

actors/miner/src/monies.rs

@@ -297,21 +297,21 @@ lazy_static! {
 }
 
 pub fn aggregate_prove_commit_network_fee(
-    aggregate_size: i64,
+    aggregate_size: usize,
     base_fee: &TokenAmount,
 ) -> TokenAmount {
     aggregate_network_fee(aggregate_size, &ESTIMATED_SINGLE_PROVE_COMMIT_GAS_USAGE, base_fee)
 }
 
 pub fn aggregate_pre_commit_network_fee(
-    aggregate_size: i64,
+    aggregate_size: usize,
     base_fee: &TokenAmount,
 ) -> TokenAmount {
     aggregate_network_fee(aggregate_size, &ESTIMATED_SINGLE_PRE_COMMIT_GAS_USAGE, base_fee)
 }
 
 pub fn aggregate_network_fee(
-    aggregate_size: i64,
+    aggregate_size: usize,
     gas_usage: &BigInt,
     base_fee: &TokenAmount,
 ) -> TokenAmount {

actors/miner/src/state.rs

@@ -1,6 +1,7 @@
 // Copyright 2019-2022 ChainSafe Systems
 // SPDX-License-Identifier: Apache-2.0, MIT
 
+use std::borrow::Borrow;
 use std::cmp;
 use std::ops::Neg;
 
@@ -10,7 +11,7 @@ use cid::Cid;
 use fil_actors_runtime::runtime::Policy;
 use fil_actors_runtime::{
     actor_error, make_empty_map, make_map_with_root_and_bitwidth, u64_key, ActorDowncast,
-    ActorError, Array,
+    ActorError, Array, AsActorError,
 };
 use fvm_ipld_amt::Error as AmtError;
 use fvm_ipld_bitfield::BitField;
@@ -1169,22 +1170,24 @@ impl State {
         })
     }
 
-    pub fn get_all_precommitted_sectors<BS: Blockstore>(
+    // Loads sectors precommit information from store, requiring it to exist.
+    pub fn get_precommitted_sectors<BS: Blockstore>(
         &self,
         store: &BS,
-        sector_nos: &BitField,
-    ) -> anyhow::Result<Vec<SectorPreCommitOnChainInfo>> {
+        sector_nos: impl IntoIterator<Item = impl Borrow<SectorNumber>>,
+    ) -> Result<Vec<SectorPreCommitOnChainInfo>, ActorError> {
         let mut precommits = Vec::new();
         let precommitted =
-            make_map_with_root_and_bitwidth(&self.pre_committed_sectors, store, HAMT_BIT_WIDTH)?;
-        for sector_no in sector_nos.iter() {
+            make_map_with_root_and_bitwidth(&self.pre_committed_sectors, store, HAMT_BIT_WIDTH)
+                .context_code(ExitCode::USR_ILLEGAL_STATE, "failed to load precommitted sectors")?;
+        for sector_no in sector_nos.into_iter() {
+            let sector_no = *sector_no.borrow();
             if sector_no > MAX_SECTOR_NUMBER {
-                return Err(
-                    actor_error!(illegal_argument; "sector number greater than maximum").into()
-                );
+                return Err(actor_error!(illegal_argument; "sector number greater than maximum"));
             }
             let info: &SectorPreCommitOnChainInfo = precommitted
-                .get(&u64_key(sector_no))?
+                .get(&u64_key(sector_no))
+                .exit_code(ExitCode::USR_ILLEGAL_STATE)?
                 .ok_or_else(|| actor_error!(not_found, "sector {} not found", sector_no))?;
             precommits.push(info.clone());
         }

actors/miner/src/types.rs

@@ -3,20 +3,24 @@
 
 use cid::Cid;
 use fvm_ipld_bitfield::BitField;
-use fvm_ipld_encoding::tuple::*;
 use fvm_ipld_encoding::{strict_bytes, BytesDe};
+use fvm_ipld_encoding::{tuple::*, RawBytes};
 use fvm_shared::address::Address;
 use fvm_shared::bigint::bigint_ser;
 use fvm_shared::clock::ChainEpoch;
 use fvm_shared::deal::DealID;
 use fvm_shared::econ::TokenAmount;
+use fvm_shared::error::ExitCode;
+use fvm_shared::piece::PaddedPieceSize;
 use fvm_shared::randomness::Randomness;
 use fvm_shared::sector::{
     PoStProof, RegisteredPoStProof, RegisteredSealProof, RegisteredUpdateProof, SectorNumber,
     SectorSize, StoragePower,
 };
 use fvm_shared::smooth::FilterEstimate;
+use fvm_shared::ActorID;
 
+use crate::ext::verifreg::AllocationID;
 use fil_actors_runtime::DealWeight;
 use serde::{Deserialize, Serialize};
 
@@ -129,8 +133,96 @@ pub struct SubmitWindowedPoStParams {
 #[derive(Serialize_tuple, Deserialize_tuple)]
 pub struct ProveCommitSectorParams {
     pub sector_number: SectorNumber,
-    #[serde(with = "strict_bytes")]
-    pub proof: Vec<u8>,
+    pub proof: RawBytes,
+}
+
+#[derive(Serialize_tuple, Deserialize_tuple)]
+pub struct ProveCommitSectors2Params {
+    // Activation manifest for each sector being proven.
+    pub sector_activations: Vec<SectorActivationManifest>,
+    // Proofs for each sector, parallel to activation manifests.
+    // Exactly one of sector_proofs or aggregate_proof must be non-empty.
+    pub sector_proofs: Vec<RawBytes>,
+    // Aggregate proof for all sectors.
+    // Exactly one of sector_proofs or aggregate_proof must be non-empty.
+    pub aggregate_proof: RawBytes,
+    // Whether to abort if any sector activation fails.
+    pub require_activation_success: bool,
+    // Whether to abort if any notification returns a non-zero exit code.
+    pub require_notification_success: bool,
+}
+
+// Data to activate a commitment to one sector and its data.
+// All pieces of data must be specified, whether or not not claiming a FIL+ activation or being
+// notified to a data consumer.
+// An implicit zero piece fills any remaining sector capacity.
+// XXX: we should consider fast tracking the special case where there is only
+//  one piece not claiming or notifying other actors to allow an empty piece vector.
+//  We could interpret this as a single piece, size == sector size, cid == commD, empty allocation empty notify vector
+#[derive(Serialize_tuple, Deserialize_tuple, Clone)]
+pub struct SectorActivationManifest {
+    // Sector to be activated.
+    pub sector_number: SectorNumber,
+    // Pieces comprising the sector content, in order.
+    pub pieces: Vec<PieceActivationManifest>,
+}
+
+#[derive(Serialize_tuple, Deserialize_tuple, Clone)]
+pub struct PieceActivationManifest {
+    // Piece data commitment.
+    pub cid: Cid,
+    // Piece size.
+    pub size: PaddedPieceSize,
+    // Identifies a verified allocation to be claimed.
+    pub verified_allocation_key: Option<VerifiedAllocationKey>,
+    // Synchronous notifications to be sent to other actors after activation.
+    pub notify: Vec<DataActivationNotification>,
+}
+
+#[derive(Serialize_tuple, Deserialize_tuple, Clone)]
+pub struct VerifiedAllocationKey {
+    pub client: ActorID,
+    pub id: AllocationID,
+}
+
+#[derive(Serialize_tuple, Deserialize_tuple, Clone)]
+pub struct DataActivationNotification {
+    // Actor to be notified.
+    pub address: Address,
+    // Data to send in the notification.
+    pub payload: RawBytes,
+}
+
+#[derive(Serialize_tuple, Deserialize_tuple)]
+pub struct ProveCommit2Return {
+    // Sector activation results, parallel to input sector activation manifests.
+    pub sectors: Vec<SectorActivationReturn>,
+}
+
+#[derive(Serialize_tuple, Deserialize_tuple)]
+pub struct SectorActivationReturn {
+    // Whether the sector was activated.
+    activated: bool,
+    // Power of the activated sector (or zero).
+    power: StoragePower,
+    // Piece activation results, parallel to input piece activation manifests.
+    pieces: Vec<PieceActivationReturn>,
+}
+
+#[derive(Serialize_tuple, Deserialize_tuple)]
+pub struct PieceActivationReturn {
+    // Whether a verified allocation was successfully claimed by the piece.
+    claimed: bool,
+    // Results from notifications of piece activation, parallel to input notification requests.
+    notifications: Vec<DataActivationNotificationReturn>,
+}
+
+#[derive(Serialize_tuple, Deserialize_tuple)]
+pub struct DataActivationNotificationReturn {
+    // Exit code from the notified actor.
+    code: ExitCode,
+    // Return value from the notified actor.
+    data: RawBytes,
 }
 
 #[derive(Serialize_tuple, Deserialize_tuple)]
@@ -393,8 +485,7 @@ pub struct DisputeWindowedPoStParams {
 #[derive(Debug, Clone, Serialize_tuple, Deserialize_tuple)]
 pub struct ProveCommitAggregateParams {
     pub sector_numbers: BitField,
-    #[serde(with = "strict_bytes")]
-    pub aggregate_proof: Vec<u8>,
+    pub aggregate_proof: RawBytes,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)]
@@ -405,8 +496,7 @@ pub struct ReplicaUpdate {
     pub new_sealed_cid: Cid,
     pub deals: Vec<DealID>,
     pub update_proof_type: RegisteredUpdateProof,
-    #[serde(with = "strict_bytes")]
-    pub replica_proof: Vec<u8>,
+    pub replica_proof: RawBytes,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)]
@@ -423,8 +513,7 @@ pub struct ReplicaUpdate2 {
     pub new_unsealed_cid: Cid,
     pub deals: Vec<DealID>,
     pub update_proof_type: RegisteredUpdateProof,
-    #[serde(with = "strict_bytes")]
-    pub replica_proof: Vec<u8>,
+    pub replica_proof: RawBytes,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)]

actors/miner/tests/batch_method_network_fees_test.rs

@@ -53,7 +53,7 @@ fn insufficient_funds_for_aggregated_prove_commit_network_fee() {
     rt.set_balance(balance.clone());
     let base_fee = TokenAmount::from_atto(10u64.pow(16));
     rt.base_fee.replace(base_fee.clone());
-    assert!(aggregate_prove_commit_network_fee(precommits.len() as i64, &base_fee) > balance);
+    assert!(aggregate_prove_commit_network_fee(precommits.len(), &base_fee) > balance);
 
     let res = actor.prove_commit_aggregate_sector(
         &rt,
@@ -93,7 +93,7 @@ fn insufficient_funds_for_batch_precommit_network_fee() {
     rt.set_balance(balance.clone());
     let base_fee = TokenAmount::from_atto(10u64.pow(16));
     rt.base_fee.replace(base_fee.clone());
-    assert!(aggregate_pre_commit_network_fee(precommits.len() as i64, &base_fee) > balance);
+    assert!(aggregate_pre_commit_network_fee(precommits.len(), &base_fee) > balance);
 
     let res = actor.pre_commit_sector_batch(
         &rt,
@@ -140,7 +140,7 @@ fn insufficient_funds_for_batch_precommit_in_combination_of_fee_debt_and_network
     // set base fee extremely high so AggregateProveCommitNetworkFee is > 1000 FIL. Set balance to 1000 FIL to easily cover PCD but not network fee
     let base_fee = TokenAmount::from_atto(10u64.pow(16));
     rt.base_fee.replace(base_fee.clone());
-    let net_fee = aggregate_pre_commit_network_fee(precommits.len() as i64, &base_fee);
+    let net_fee = aggregate_pre_commit_network_fee(precommits.len(), &base_fee);
 
     // setup miner to have fee debt equal to net fee
     let mut state: State = rt.get_state();
@@ -196,7 +196,7 @@ fn enough_funds_for_fee_debt_and_network_fee_but_not_for_pcd() {
     // set base fee and fee debt high
     let base_fee = TokenAmount::from_atto(10u64.pow(16));
     rt.base_fee.replace(base_fee.clone());
-    let net_fee = aggregate_pre_commit_network_fee(precommits.len() as i64, &base_fee);
+    let net_fee = aggregate_pre_commit_network_fee(precommits.len(), &base_fee);
     // setup miner to have feed debt equal to net fee
     let mut state: State = rt.get_state();
     state.fee_debt = net_fee.clone();
@@ -251,7 +251,7 @@ fn enough_funds_for_everything() {
     // set base fee extremely high so AggregateProveCommitNetworkFee is > 1000 FIL. Set balance to 1000 FIL to easily cover PCD but not network fee
     let base_fee = TokenAmount::from_atto(10u64.pow(16));
     rt.base_fee.replace(base_fee.clone());
-    let net_fee = aggregate_pre_commit_network_fee(precommits.len() as i64, &base_fee);
+    let net_fee = aggregate_pre_commit_network_fee(precommits.len(), &base_fee);
 
     // setup miner to have fee debt equal to net fee
     let mut state: State = rt.get_state();

actors/miner/tests/miner_actor_test_precommit_batch.rs

@@ -81,7 +81,7 @@ fn assert_simple_batch(
             &pwr_estimate,
         );
     }
-    let net_fee = aggregate_pre_commit_network_fee(batch_size as i64, &base_fee);
+    let net_fee = aggregate_pre_commit_network_fee(batch_size, &base_fee);
     let total_deposit: TokenAmount = deposits.iter().sum();
     let total_balance = net_fee + &total_deposit;
     rt.set_balance(total_balance + balance_surplus);

actors/miner/tests/prove_commit.rs

@@ -612,8 +612,7 @@ fn verify_proof_does_not_vest_funds() {
     rt.set_epoch(precommit_epoch + rt.policy.pre_commit_challenge_delay + 1);
     rt.balance.replace(TokenAmount::from_whole(1000));
 
-    let mut prove_commit = h.make_prove_commit_params(sector_no);
-    prove_commit.proof.resize(192, 0);
+    let prove_commit = h.make_prove_commit_params(sector_no);
     // The below call expects exactly the pledge delta for the proven sector, zero for any other vesting.
     h.prove_commit_sector_and_confirm(&rt, &precommit, prove_commit, ProveCommitConfig::empty())
         .unwrap();

actors/miner/tests/util.rs

@@ -549,7 +549,7 @@ impl ActorHarness {
     }
 
     pub fn make_prove_commit_params(&self, sector_no: u64) -> ProveCommitSectorParams {
-        ProveCommitSectorParams { sector_number: sector_no, proof: vec![0u8; 192] }
+        ProveCommitSectorParams { sector_number: sector_no, proof: vec![0u8; 192].into() }
     }
 
     pub fn pre_commit_sector_batch(
@@ -630,7 +630,7 @@ impl ActorHarness {
 
         let mut expected_network_fee = TokenAmount::zero();
         if sectors.len() > 1 {
-            expected_network_fee = aggregate_pre_commit_network_fee(sectors.len() as i64, base_fee);
+            expected_network_fee = aggregate_pre_commit_network_fee(sectors.len(), base_fee);
         }
         // burn networkFee
         if state.fee_debt.is_positive() || expected_network_fee.is_positive() {
@@ -802,8 +802,8 @@ impl ActorHarness {
             sector_id: SectorID { miner: actor_id, number: pc.info.sector_number },
             sealed_cid: pc.info.sealed_cid,
             registered_proof: pc.info.seal_proof,
-            proof: params.proof.clone(),
-            deal_ids: pc.info.deal_ids.clone(),
+            proof: params.proof.clone().into(),
+            deal_ids: vec![],
             randomness: Randomness(seal_rand.into()),
             interactive_randomness: Randomness(seal_int_rand.into()),
             unsealed_cid: pc.info.unsealed_cid.get_cid(pc.info.seal_proof).unwrap(),
@@ -880,13 +880,13 @@ impl ActorHarness {
                 unsealed_cid: comm_ds[i],
             })
         }
-        rt.expect_aggregate_verify_seals(svis, params.aggregate_proof.clone(), Ok(()));
+        rt.expect_aggregate_verify_seals(svis, params.aggregate_proof.clone().into(), Ok(()));
 
         // confirm sector proofs valid
         self.confirm_sector_proofs_valid_internal(rt, config, &precommits);
 
         // burn network fee
-        let expected_fee = aggregate_prove_commit_network_fee(precommits.len() as i64, base_fee);
+        let expected_fee = aggregate_prove_commit_network_fee(precommits.len(), base_fee);
         assert!(expected_fee.is_positive());
         rt.expect_send_simple(
             BURNT_FUNDS_ACTOR_ADDR,
@@ -2765,7 +2765,7 @@ pub fn get_bitfield(ubf: &UnvalidatedBitField) -> BitField {
 pub fn make_prove_commit_aggregate(sector_nos: &BitField) -> ProveCommitAggregateParams {
     ProveCommitAggregateParams {
         sector_numbers: sector_nos.clone(),
-        aggregate_proof: vec![0; 1024],
+        aggregate_proof: vec![0; 1024].into(),
     }
 }
 

integration_tests/src/tests/batch_onboarding_deals_test.rs

@@ -233,7 +233,7 @@ pub fn prove_commit_aggregate(
     let sector_nos: Vec<u64> = precommits.iter().map(|p| p.info.sector_number).collect();
     let prove_commit_aggregate_params = ProveCommitAggregateParams {
         sector_numbers: make_bitfield(sector_nos.as_slice()),
-        aggregate_proof: vec![],
+        aggregate_proof: vec![].into(),
     };
 
     apply_ok(