More tests for direct data onboarding (#1440)

Tests that CommD computed from pieces must match that declared at precommit. Tests that updating an empty replica to be empty is permitted (later, when we allow updates to non-empty replicas, it won't be a no-op).

actors/miner/src/commd.rs

@@ -25,12 +25,26 @@ impl CompactCommD {
         CompactCommD(Some(c))
     }
 
+    // Whether this represents the zero CID.
+    pub fn is_zero(&self) -> bool {
+        self.0.is_none()
+    }
+
+    // Gets the full, non-compact CID.
     pub fn get_cid(&self, seal_proof: RegisteredSealProof) -> Result<Cid, ActorError> {
         match self.0 {
             Some(ref x) => Ok(*x),
             None => zero_commd(seal_proof),
         }
     }
+
+    // Gets the full, non-compact CID, panicking if the CID is zero.
+    pub fn get_nonzero_cid(&self) -> Cid {
+        match self.0 {
+            Some(ref x) => *x,
+            None => panic!("zero commd"),
+        }
+    }
 }
 
 /// Prefix for unsealed sector CIDs (CommD).

actors/miner/src/lib.rs

@@ -5362,7 +5362,7 @@ fn activate_sectors_pieces(
             if !declared_commd.eq(&computed_commd) {
                 return Err(actor_error!(
                     illegal_argument,
-                    "unsealed CID does not match deals for sector {}, computed {:?} declared {:?}",
+                    "unsealed CID does not match pieces for sector {}, computed {:?} declared {:?}",
                     activation_info.sector_number,
                     computed_commd,
                     declared_commd

actors/miner/tests/prove_commit2_failures_test.rs

@@ -111,13 +111,18 @@ fn reject_precommit_deals() {
     // Precommit sectors, one with a deal
     let precommit_epoch = *rt.epoch.borrow();
     let sector_expiry = precommit_epoch + DEFAULT_SECTOR_EXPIRATION_DAYS * EPOCHS_IN_DAY;
-    let mut precommits =
-        make_fake_commd_precommits(&h, FIRST_SECTOR_NUMBER, precommit_epoch - 1, sector_expiry, 2);
+    let piece_size = h.sector_size as u64;
+    let mut precommits = make_fake_precommits(
+        &h,
+        FIRST_SECTOR_NUMBER,
+        precommit_epoch - 1,
+        sector_expiry,
+        &[&[piece_size], &[piece_size]],
+    );
     precommits[0].deal_ids.push(1);
     h.pre_commit_sector_batch_v2(&rt, &precommits, true, &TokenAmount::zero()).unwrap();
     rt.set_epoch(precommit_epoch + rt.policy.pre_commit_challenge_delay + 1);
 
-    let piece_size = h.sector_size as u64;
     let manifests: Vec<SectorActivationManifest> = precommits
         .iter()
         .map(|s| make_activation_manifest(s.sector_number, &[(piece_size, 0, 0, 0)]))
@@ -172,6 +177,20 @@ fn reject_required_proof_failure() {
     h.check_state(&rt);
 }
 
+#[test]
+fn reject_mismatched_commd() {
+    let (h, rt, mut activations) = setup_precommits(&[(0, 0, 0); 2]);
+    // Set wrong CID for first sector.
+    activations[0].pieces[0].cid = activations[1].pieces[0].cid;
+
+    let cfg = ProveCommitSectors2Config::default();
+    expect_abort_contains_message(
+        ExitCode::USR_ILLEGAL_ARGUMENT,
+        "unsealed CID does not match pieces",
+        h.prove_commit_sectors2(&rt, &activations, false, false, false, cfg),
+    );
+}
+
 #[test]
 fn reject_required_claim_failure() {
     let (h, rt, activations) = setup_precommits(&[(0, 0, 0), (CLIENT_ID, 1, 0)]);
@@ -229,19 +248,21 @@ fn setup_precommits(
     let (h, rt) = setup_basic();
 
     // Precommit sectors
+    let piece_size = h.sector_size as u64; // All sectors have a single full-size piece.
     let precommit_epoch = *rt.epoch.borrow();
     let sector_expiry = *rt.epoch.borrow() + DEFAULT_SECTOR_EXPIRATION_DAYS * EPOCHS_IN_DAY;
-    let precommits = make_fake_commd_precommits(
+    let one_sector_piece_sizes = &[piece_size] as &[u64];
+    let piece_sizes = vec![one_sector_piece_sizes; confs.len()];
+    let precommits = make_fake_precommits(
         &h,
         FIRST_SECTOR_NUMBER,
         precommit_epoch - 1,
         sector_expiry,
-        confs.len(),
+        &piece_sizes,
     );
     h.pre_commit_sector_batch_v2(&rt, &precommits, true, &TokenAmount::zero()).unwrap();
     rt.set_epoch(precommit_epoch + rt.policy.pre_commit_challenge_delay + 1);
 
-    let piece_size = h.sector_size as u64;
     let manifests = precommits
         .iter()
         .zip(confs)

actors/miner/tests/prove_commit2_test.rs

@@ -22,12 +22,17 @@ const FIRST_SECTOR_NUMBER: SectorNumber = 100;
 #[test]
 fn commit_batch() {
     let (h, mut rt) = setup_basic();
-    let precommits = precommit_sectors(&mut rt, &h, 4);
+    let piece_size = h.sector_size as u64;
+    let precommits = precommit_sectors(
+        &mut rt,
+        &h,
+        &[&[piece_size], &[piece_size], &[piece_size], &[piece_size]],
+    );
     let snos: Vec<SectorNumber> =
         precommits.iter().map(|pci: &SectorPreCommitInfo| pci.sector_number).collect();
 
     // Prove them in batch, each with a single piece.
-    let piece_size = h.sector_size as u64;
+
     let manifests = vec![
         make_activation_manifest(snos[0], &[(piece_size, 0, 0, 0)]), // No alloc or deal
         make_activation_manifest(snos[1], &[(piece_size, CLIENT_ID, 1000, 0)]), // Just an alloc
@@ -117,12 +122,13 @@ fn commit_batch() {
 #[test]
 fn multiple_pieces_in_sector() {
     let (h, mut rt) = setup_basic();
-    let precommits = precommit_sectors(&mut rt, &h, 2);
+    // Half-size pieces
+    let piece_size = h.sector_size as u64 / 2;
+    let precommits =
+        precommit_sectors(&mut rt, &h, &[&[piece_size, piece_size], &[piece_size, piece_size]]);
     let snos: Vec<SectorNumber> =
         precommits.iter().map(|pci: &SectorPreCommitInfo| pci.sector_number).collect();
 
-    // Half-size pieces
-    let piece_size = h.sector_size as u64 / 2;
     let manifests = vec![
         make_activation_manifest(
             snos[0],
@@ -215,11 +221,11 @@ fn multiple_pieces_in_sector() {
 #[test]
 fn multiple_notifs_for_piece() {
     let (h, mut rt) = setup_basic();
-    let precommits = precommit_sectors(&mut rt, &h, 2);
+    let piece_size = h.sector_size as u64 / 2;
+    let precommits = precommit_sectors(&mut rt, &h, &[&[piece_size, piece_size], &[piece_size]]);
     let snos: Vec<SectorNumber> =
         precommits.iter().map(|pci: &SectorPreCommitInfo| pci.sector_number).collect();
 
-    let piece_size = h.sector_size as u64 / 2;
     let mut manifests = vec![
         make_activation_manifest(
             snos[0],
@@ -298,17 +304,22 @@ fn multiple_notifs_for_piece() {
 #[test]
 fn expired_precommit_dropped_batch() {
     let (h, mut rt) = setup_basic();
-    let precommits1 = precommit_sectors(&mut rt, &h, 1);
+    let piece_size = h.sector_size as u64;
+    let precommits1 = precommit_sectors(&mut rt, &h, &[&[piece_size]]);
     let epoch = *rt.epoch.borrow();
     rt.set_epoch(epoch + 31 * EPOCHS_IN_DAY); // The first precommit expired.
 
-    let precommits2 =
-        precommit_sectors_from(&mut rt, &h, precommits1[0].sector_number + 1, 1, false);
+    let precommits2 = precommit_sectors_from(
+        &mut rt,
+        &h,
+        precommits1[0].sector_number + 1,
+        &[&[piece_size]],
+        false,
+    );
     let precommits = [&precommits1[..], &precommits2[..]].concat();
     let snos: Vec<SectorNumber> =
         precommits.iter().map(|pci: &SectorPreCommitInfo| pci.sector_number).collect();
 
-    let piece_size = h.sector_size as u64;
     let manifests = vec![
         make_activation_manifest(snos[0], &[(piece_size, CLIENT_ID, 1000, 2000)]),
         make_activation_manifest(snos[1], &[(piece_size, CLIENT_ID, 1001, 2001)]),
@@ -335,17 +346,22 @@ fn expired_precommit_dropped_batch() {
 #[test]
 fn expired_precommit_dropped_aggregate() {
     let (h, mut rt) = setup_basic();
-    let precommits1 = precommit_sectors(&mut rt, &h, 1);
+    let piece_size = h.sector_size as u64;
+    let precommits1 = precommit_sectors(&mut rt, &h, &[&[piece_size]]);
     let epoch = *rt.epoch.borrow();
     rt.set_epoch(epoch + 31 * EPOCHS_IN_DAY); // The first precommit expired.
 
-    let precommits2 =
-        precommit_sectors_from(&mut rt, &h, precommits1[0].sector_number + 1, 3, false);
+    let precommits2 = precommit_sectors_from(
+        &mut rt,
+        &h,
+        precommits1[0].sector_number + 1,
+        &[&[piece_size], &[piece_size], &[piece_size]],
+        false,
+    );
     let precommits = [&precommits1[..], &precommits2[..]].concat();
     let snos: Vec<SectorNumber> =
         precommits.iter().map(|pci: &SectorPreCommitInfo| pci.sector_number).collect();
 
-    let piece_size = h.sector_size as u64;
     let manifests = vec![
         make_activation_manifest(snos[0], &[(piece_size, CLIENT_ID, 1000, 2000)]),
         make_activation_manifest(snos[1], &[(piece_size, CLIENT_ID, 1001, 2001)]),
@@ -374,11 +390,11 @@ fn expired_precommit_dropped_aggregate() {
 #[test]
 fn invalid_proof_dropped() {
     let (h, mut rt) = setup_basic();
-    let precommits = precommit_sectors(&mut rt, &h, 2);
+    let piece_size = h.sector_size as u64;
+    let precommits = precommit_sectors(&mut rt, &h, &[&[piece_size], &[piece_size]]);
     let snos: Vec<SectorNumber> =
         precommits.iter().map(|pci: &SectorPreCommitInfo| pci.sector_number).collect();
 
-    let piece_size = h.sector_size as u64;
     let manifests = vec![
         make_activation_manifest(snos[0], &[(piece_size, CLIENT_ID, 1000, 2000)]),
         make_activation_manifest(snos[1], &[(piece_size, CLIENT_ID, 1001, 2001)]),
@@ -400,11 +416,11 @@ fn invalid_proof_dropped() {
 #[test]
 fn invalid_claim_dropped() {
     let (h, mut rt) = setup_basic();
-    let precommits = precommit_sectors(&mut rt, &h, 2);
+    let piece_size = h.sector_size as u64;
+    let precommits = precommit_sectors(&mut rt, &h, &[&[piece_size], &[piece_size]]);
     let snos: Vec<SectorNumber> =
         precommits.iter().map(|pci: &SectorPreCommitInfo| pci.sector_number).collect();
 
-    let piece_size = h.sector_size as u64;
     let manifests = vec![
         make_activation_manifest(snos[0], &[(piece_size, CLIENT_ID, 1000, 2000)]),
         make_activation_manifest(snos[1], &[(piece_size, CLIENT_ID, 1001, 2001)]),
@@ -425,11 +441,11 @@ fn invalid_claim_dropped() {
 #[test]
 fn aborted_notification_dropped() {
     let (h, mut rt) = setup_basic();
-    let precommits = precommit_sectors(&mut rt, &h, 2);
+    let piece_size = h.sector_size as u64;
+    let precommits = precommit_sectors(&mut rt, &h, &[&[piece_size], &[piece_size]]);
     let snos: Vec<SectorNumber> =
         precommits.iter().map(|pci: &SectorPreCommitInfo| pci.sector_number).collect();
 
-    let piece_size = h.sector_size as u64;
     let manifests = vec![
         make_activation_manifest(snos[0], &[(piece_size, CLIENT_ID, 1000, 2000)]),
         make_activation_manifest(snos[1], &[(piece_size, CLIENT_ID, 1001, 2001)]),
@@ -452,11 +468,11 @@ fn aborted_notification_dropped() {
 #[test]
 fn rejected_notification_dropped() {
     let (h, mut rt) = setup_basic();
-    let precommits = precommit_sectors(&mut rt, &h, 2);
+    let piece_size = h.sector_size as u64;
+    let precommits = precommit_sectors(&mut rt, &h, &[&[piece_size], &[piece_size]]);
     let snos: Vec<SectorNumber> =
         precommits.iter().map(|pci: &SectorPreCommitInfo| pci.sector_number).collect();
 
-    let piece_size = h.sector_size as u64;
     let manifests = vec![
         make_activation_manifest(snos[0], &[(piece_size, CLIENT_ID, 1000, 2000)]),
         make_activation_manifest(snos[1], &[(piece_size, CLIENT_ID, 1001, 2001)]),
@@ -484,26 +500,26 @@ fn setup_basic() -> (ActorHarness, MockRuntime) {
 fn precommit_sectors(
     rt: &mut MockRuntime,
     h: &ActorHarness,
-    sector_count: usize,
+    piece_sizes: &[&[u64]],
 ) -> Vec<SectorPreCommitInfo> {
-    precommit_sectors_from(rt, h, FIRST_SECTOR_NUMBER, sector_count, true)
+    precommit_sectors_from(rt, h, FIRST_SECTOR_NUMBER, piece_sizes, true)
 }
 
 fn precommit_sectors_from(
     rt: &mut MockRuntime,
     h: &ActorHarness,
     first_sector_number: SectorNumber,
-    sector_count: usize,
+    piece_sizes: &[&[u64]],
     first_for_miner: bool,
 ) -> Vec<SectorPreCommitInfo> {
     let precommit_epoch = *rt.epoch.borrow();
     let sector_expiry = *rt.epoch.borrow() + DEFAULT_SECTOR_EXPIRATION_DAYS * EPOCHS_IN_DAY;
-    let precommits = make_fake_commd_precommits(
+    let precommits = make_fake_precommits(
         h,
         first_sector_number,
         precommit_epoch - 1,
         sector_expiry,
-        sector_count,
+        piece_sizes,
     );
     h.pre_commit_sector_batch_v2(rt, &precommits, first_for_miner, &TokenAmount::zero()).unwrap();
     rt.set_epoch(precommit_epoch + rt.policy.pre_commit_challenge_delay + 1);

actors/miner/tests/prove_replica_test.rs

@@ -298,7 +298,15 @@ fn cant_update_nonempty_sector() {
 
     // Onboard a non-empty sector
     let sector_expiry = *rt.epoch.borrow() + DEFAULT_SECTOR_EXPIRATION_DAYS * EPOCHS_IN_DAY;
-    let sectors = onboard_nonempty_sectors(&rt, &h, sector_expiry, FIRST_SECTOR_NUMBER, 1);
+    let challenge = *rt.epoch.borrow();
+    let precommits = make_fake_precommits(
+        &h,
+        FIRST_SECTOR_NUMBER,
+        challenge - 1,
+        sector_expiry,
+        &[&[h.sector_size as u64]], // A piece fills the sector.
+    );
+    let sectors = onboard_sectors(&rt, &h, &precommits);
     let snos = sectors.iter().map(|s| s.sector_number).collect::<Vec<_>>();
 
     // Attempt to update
@@ -446,6 +454,43 @@ fn rejected_notification_dropped() {
     h.check_state(&rt);
 }
 
+#[test]
+fn update_to_empty() {
+    let (h, rt, sectors) = setup_empty_sectors(1);
+    let snos = sectors.iter().map(|s| s.sector_number).collect::<Vec<_>>();
+    let st: State = h.get_state(&rt);
+    let store = rt.store();
+
+    let sector_updates = vec![
+        make_update_manifest(&st, store, snos[0], &[]), // No pieces
+    ];
+
+    let cfg = ProveReplicaUpdatesConfig::default();
+    let (result, _, _) =
+        h.prove_replica_updates2_batch(&rt, &sector_updates, true, true, cfg).unwrap();
+    assert_update_result(&vec![ExitCode::OK; sectors.len()], &result);
+    verify_weights(&rt, &h, snos[0], 0, 0);
+
+    // Because data is still empty, it's eligible for update again, this time with data.
+    let piece_size = h.sector_size as u64;
+    let sector_updates = vec![make_update_manifest(&st, store, snos[0], &[(piece_size, 0, 0, 0)])];
+
+    let cfg = ProveReplicaUpdatesConfig::default();
+    let (result, _, _) =
+        h.prove_replica_updates2_batch(&rt, &sector_updates, true, true, cfg).unwrap();
+    assert_update_result(&vec![ExitCode::OK; sectors.len()], &result);
+
+    // But not again now it's non-empty.
+    let cfg = ProveReplicaUpdatesConfig::default();
+    expect_abort_contains_message(
+        ExitCode::USR_ILLEGAL_ARGUMENT,
+        "cannot update sector with non-zero data",
+        h.prove_replica_updates2_batch(&rt, &sector_updates, true, true, cfg),
+    );
+
+    h.check_state(&rt);
+}
+
 fn setup_basic() -> (ActorHarness, MockRuntime) {
     let h = ActorHarness::new_with_options(HarnessOptions::default());
     let rt = h.new_runtime();