Merge pull request #72 from ipfs-force-community/feat/lt/speed_cc_sector

Opt: skip AP for cc sector

venus-worker/src/bin/venus-worker/generator/mod.rs

@@ -0,0 +1,57 @@
+use anyhow::{anyhow, Result};
+use byte_unit::Byte;
+use clap::{value_t, App, AppSettings, Arg, ArgMatches, SubCommand};
+use std::convert::TryFrom;
+use std::path::PathBuf;
+use tracing::{error, info};
+
+use venus_worker::create_tree_d;
+
+use venus_worker::{RegisteredSealProof, SealProof};
+
+pub const SUB_CMD_NAME: &str = "generator";
+
+pub fn subcommand<'a, 'b>() -> App<'a, 'b> {
+    let tree_d_cmd = SubCommand::with_name("tree-d")
+        .arg(
+            Arg::with_name("sector-size")
+                .long("sector-size")
+                .short("s")
+                .takes_value(true)
+                .help("sector size"),
+        )
+        .arg(
+            Arg::with_name("path")
+                .long("path")
+                .short("p")
+                .takes_value(true)
+                .help("path for the staic-tree-d"),
+        );
+
+    SubCommand::with_name(SUB_CMD_NAME)
+        .setting(AppSettings::ArgRequiredElseHelp)
+        .subcommand(tree_d_cmd)
+}
+
+pub(crate) fn submatch<'a>(subargs: &ArgMatches<'a>) -> Result<()> {
+    match subargs.subcommand() {
+        ("tree-d", Some(m)) => {
+            let size_str = value_t!(m, "sector-size", String)?;
+            let size = Byte::from_str(size_str)?;
+            let path = value_t!(m, "path", String)?;
+
+            let proof_type = SealProof::try_from(size.get_bytes() as u64)?;
+            let registered_proof = RegisteredSealProof::from(proof_type);
+            let cache_dir = PathBuf::from(&path);
+
+            match create_tree_d(registered_proof, None, cache_dir) {
+                Ok(_) => info!("generate static tree-d succeed"),
+                Err(e) => error!("generate static tree-d {}", e),
+            }
+
+            Ok(())
+        }
+
+        (other, _) => Err(anyhow!("unexpected subcommand `{}` of generator", other)),
+    }
+}

venus-worker/src/bin/venus-worker/main.rs

@@ -5,6 +5,7 @@ use tokio::runtime::Builder;
 
 use venus_worker::{logging, start_deamon, start_mock};
 
+mod generator;
 mod processor;
 mod store;
 mod worker;
@@ -57,6 +58,7 @@ pub fn main() -> Result<()> {
                 .help("path for the config file"),
         );
 
+    let generator_cmd = generator::subcommand();
     let processor_cmd = processor::subcommand();
     let store_cmd = store::subcommand();
     let worker_cmd = worker::subcommand();
@@ -65,6 +67,7 @@ pub fn main() -> Result<()> {
         .version(env!("CARGO_PKG_VERSION"))
         .setting(AppSettings::ArgRequiredElseHelp)
         .subcommand(daemon_cmd)
+        .subcommand(generator_cmd)
         .subcommand(mock_cmd)
         .subcommand(processor_cmd)
         .subcommand(store_cmd)
@@ -88,6 +91,8 @@ pub fn main() -> Result<()> {
             start_deamon(cfg_path)
         }
 
+        (generator::SUB_CMD_NAME, Some(args)) => generator::submatch(args),
+
         (processor::SUB_CMD_NAME, Some(args)) => processor::submatch(args),
 
         (store::SUB_CMD_NAME, Some(args)) => store::submatch(args),

venus-worker/src/lib.rs

@@ -17,11 +17,13 @@ pub use config::Config;
 pub use infra::objstore;
 pub use run::{start_deamon, start_mock};
 pub use sealing::processor::{
+    create_tree_d,
     external::sub::{run, run_c2, run_pc1, run_pc2, run_tree_d},
-    Input, SnapProveReplicaUpdateInput, SnapReplicaUpdateInput,
+    Input, RegisteredSealProof, SnapProveReplicaUpdateInput, SnapReplicaUpdateInput,
 };
 pub use sealing::store;
 pub use sealing::util as seal_util;
+pub use types::SealProof;
 pub use util::task::block_on;
 pub use watchdog::dones;
 

venus-worker/src/sealing/processor/safe.rs

@@ -105,6 +105,7 @@ pub fn seal_pre_commit_phase2(
     }
 }
 
+/// create tree-d file
 pub fn create_tree_d(
     registered_proof: RegisteredSealProof,
     in_path: Option<PathBuf>,

venus-worker/src/sealing/worker/sealer.rs

@@ -16,8 +16,10 @@ use crate::rpc::sealer::{
 };
 use crate::sealing::processor::{
     clear_cache, seal_commit_phase1, tree_d_path_in_dir, write_and_preprocess, C2Input, PC1Input,
-    PC2Input, PaddedBytesAmount, Stage, TreeDInput, UnpaddedBytesAmount,
+    PC2Input, PaddedBytesAmount, PieceInfo, Stage, TreeDInput, UnpaddedBytesAmount,
 };
+use crate::sealing::util::get_all_zero_commitment;
+
 use crate::store::Store;
 use crate::watchdog::Ctx;
 
@@ -152,10 +154,10 @@ impl<'c> Sealer<'c> {
             .base
             .as_ref()
             .map(|base| base.allocated.id.clone())
-        {
-            Some(sid) => sid,
-            None => return Ok(()),
-        };
+            {
+                Some(sid) => sid,
+                None => return Ok(()),
+            };
 
         let _ = call_rpc! {
             self.ctx.global.rpc,
@@ -464,7 +466,7 @@ impl<'c> Sealer<'c> {
                 return Ok(None);
             }
         }
-        .map(From::from)
+            .map(From::from)
     }
 
     fn handle_empty(&mut self) -> HandleResult {
@@ -599,8 +601,8 @@ impl<'c> Sealer<'c> {
                         &mut staged_file,
                         UnpaddedBytesAmount(unpadded_piece_size.0),
                     )
-                    .with_context(|| format!("write pledge piece, size={}", unpadded_piece_size.0))
-                    .perm()?
+                        .with_context(|| format!("write pledge piece, size={}", unpadded_piece_size.0))
+                        .perm()?
                 } else {
                     let mut piece_reader = piece_store
                         .get(deal.piece.cid.0, deal.payload_size, unpadded_piece_size)
@@ -612,33 +614,35 @@ impl<'c> Sealer<'c> {
                         &mut staged_file,
                         UnpaddedBytesAmount(unpadded_piece_size.0),
                     )
-                    .with_context(|| {
-                        format!(
-                            "write deal piece, cid={}, size={}",
-                            deal.piece.cid.0, unpadded_piece_size.0
-                        )
-                    })
-                    .perm()?
+                        .with_context(|| {
+                            format!(
+                                "write deal piece, cid={}, size={}",
+                                deal.piece.cid.0, unpadded_piece_size.0
+                            )
+                        })
+                        .perm()?
                 };
 
                 pieces.push(piece_info);
             }
         }
 
         if pieces.is_empty() {
-            let unpadded_size: UnpaddedBytesAmount = PaddedBytesAmount(sector_size).into();
+            // skip AP for cc sector
+            staged_file
+                .set_len(sector_size)
+                .context("add zero commitment")
+                .perm()?;
 
-            let mut pledge_piece = io::repeat(0).take(unpadded_size.0);
-            let (piece_info, _) = write_and_preprocess(
-                proof_type.into(),
-                &mut pledge_piece,
-                &mut staged_file,
-                unpadded_size,
-            )
-            .context("write full pledge piece")
-            .perm()?;
+            let commitment = get_all_zero_commitment(sector_size)
+                .context("get zero commitment")
+                .perm()?;
 
-            pieces.push(piece_info);
+            let unpadded_size: UnpaddedBytesAmount = PaddedBytesAmount(sector_size).into();
+            let pi = PieceInfo::new(commitment, unpadded_size)
+                .context("create piece info")
+                .perm()?;
+            pieces.push(pi);
         }
 
         Ok(Event::AddPiece(pieces))
@@ -671,7 +675,7 @@ impl<'c> Sealer<'c> {
         // pledge sector
         if self.sector.deals.as_ref().map(|d| d.len()).unwrap_or(0) == 0 {
             if let Some(static_tree_path) =
-                self.ctx.global.static_tree_d.get(&proof_type.sector_size())
+            self.ctx.global.static_tree_d.get(&proof_type.sector_size())
             {
                 symlink(static_tree_path, tree_d_path_in_dir(&prepared_dir)).crit()?;
                 return Ok(Event::BuildTreeD);
@@ -759,7 +763,7 @@ impl<'c> Sealer<'c> {
             tree_d_path_in_dir(&prepared_dir),
             tree_d_path_in_dir(&cache_dir),
         )
-        .crit()?;
+            .crit()?;
 
         let (prover_id, sector_id) = fetch_cloned_field! {
             self.sector.base,
@@ -922,7 +926,7 @@ impl<'c> Sealer<'c> {
                         "pre commit on-chain info permanent failed: {:?}",
                         state.desc
                     )
-                    .perm())
+                        .perm())
                 }
 
                 OnChainState::Pending | OnChainState::Packed => {}
@@ -1033,7 +1037,7 @@ impl<'c> Sealer<'c> {
             Err(anyhow!(
                 "sector files are persisted but unavailable for sealer"
             ))
-            .perm()
+                .perm()
         }
     }
 
@@ -1115,7 +1119,7 @@ impl<'c> Sealer<'c> {
             p2out,
             piece_infos,
         )
-        .perm()?;
+            .perm()?;
 
         drop(token);
         Ok(Event::C1(out))
@@ -1166,7 +1170,7 @@ impl<'c> Sealer<'c> {
                 self.sector.phases.c2out,
                 proof,
             }?
-            .into(),
+                .into(),
         };
 
         let res = call_rpc! {
@@ -1223,7 +1227,7 @@ impl<'c> Sealer<'c> {
                             "proof on-chain info permanent failed: {:?}",
                             state.desc
                         )
-                        .perm())
+                            .perm())
                     }
 
                     OnChainState::Pending | OnChainState::Packed => {}