From 6b61e8ac1ceff7dbbc1144dcf512eae666219f83 Mon Sep 17 00:00:00 2001 From: Elden Young <59600396+ytqaljn@users.noreply.github.com> Date: Thu, 9 Nov 2023 14:16:51 +0800 Subject: [PATCH] 0.7.5 mark fragment (#258) * refactor: upload file function refactor * refactor: deal info struct refactor --- c-pallets/file-bank/impls/bucket.rs | 0 c-pallets/file-bank/src/functions.rs | 41 ++---- c-pallets/file-bank/src/impls/dealimpl.rs | 37 +++++ c-pallets/file-bank/src/impls/mod.rs | 5 + c-pallets/file-bank/src/impls/receptionist.rs | 82 +++++++++++ c-pallets/file-bank/src/lib.rs | 132 ++---------------- c-pallets/file-bank/src/types.rs | 11 +- 7 files changed, 160 insertions(+), 148 deletions(-) delete mode 100644 c-pallets/file-bank/impls/bucket.rs create mode 100644 c-pallets/file-bank/src/impls/dealimpl.rs create mode 100644 c-pallets/file-bank/src/impls/mod.rs create mode 100644 c-pallets/file-bank/src/impls/receptionist.rs diff --git a/c-pallets/file-bank/impls/bucket.rs b/c-pallets/file-bank/impls/bucket.rs deleted file mode 100644 index e69de29b..00000000 diff --git a/c-pallets/file-bank/src/functions.rs b/c-pallets/file-bank/src/functions.rs index cd89358c..edcaaf91 100644 --- a/c-pallets/file-bank/src/functions.rs +++ b/c-pallets/file-bank/src/functions.rs @@ -16,41 +16,31 @@ impl Pallet { pub fn generate_file( file_hash: &Hash, deal_info: BoundedVec, T::SegmentCount>, - mut miner_task_list: BoundedVec, ConstU32>, + complete_list: BoundedVec, T::FragmentCount>, user_brief: UserBrief, stat: FileState, file_size: u128, ) -> DispatchResult { let mut segment_info_list: BoundedVec, T::SegmentCount> = Default::default(); + ensure!(complete_list.len() == FRAGMENT_COUNT as usize, Error::::Unexpected); for segment in deal_info.iter() { let mut segment_info = SegmentInfo:: { hash: segment.hash, fragment_list: Default::default(), }; - - for miner_task in &mut miner_task_list { - miner_task.fragment_list.sort(); - } - - for frag_hash in segment.fragment_list.iter() { - for miner_task in &mut miner_task_list { - if let Ok(index) = miner_task.fragment_list.binary_search(&frag_hash) { - let miner = miner_task.miner.clone().ok_or(Error::::Unexpected)?; - let frag_info = FragmentInfo:: { - hash: *frag_hash, - avail: true, - miner: miner.clone(), - }; - segment_info.fragment_list.try_push(frag_info).map_err(|_e| Error::::BoundedVecError)?; - miner_task.fragment_list.remove(index); - break; - } - } + for (index, fragment_hash) in segment.fragment_list.iter().enumerate() { + let frag_info = FragmentInfo:: { + hash: *fragment_hash, + avail: true, + miner: complete_list[index as usize].miner.clone(), + }; + + segment_info.fragment_list.try_push(frag_info).map_err(|_e| Error::::BoundedVecError)?; } segment_info_list.try_push(segment_info).map_err(|_e| Error::::BoundedVecError)?; } - + let cur_block = >::block_number(); let file_info = FileInfo:: { @@ -111,7 +101,6 @@ impl Pallet { pub(super) fn generate_deal( file_hash: Hash, file_info: BoundedVec, T::SegmentCount>, - miner_task_list: BoundedVec, ConstU32>, user_brief: UserBrief, file_size: u128, ) -> DispatchResult { @@ -127,7 +116,6 @@ impl Pallet { file_size, segment_list: file_info.clone(), user: user_brief, - miner_task_list: miner_task_list, complete_list: Default::default(), }; @@ -179,11 +167,8 @@ impl Pallet { let needed_space = Self::cal_file_size(deal_info.segment_list.len() as u128); T::StorageHandle::unlock_user_space(&deal_info.user.user, needed_space)?; // unlock mienr space - for miner_task in deal_info.miner_task_list { - if let Some(miner) = miner_task.miner { - let count = miner_task.fragment_list.len() as u128; - T::MinerControl::unlock_space(&miner, FRAGMENT_SIZE * count)?; - } + for complete_info in deal_info.complete_list { + T::MinerControl::unlock_space(&complete_info.miner, FRAGMENT_SIZE * FRAGMENT_COUNT as u128)?; } >::remove(deal_hash); diff --git a/c-pallets/file-bank/src/impls/dealimpl.rs b/c-pallets/file-bank/src/impls/dealimpl.rs new file mode 100644 index 00000000..b3dba3e8 --- /dev/null +++ b/c-pallets/file-bank/src/impls/dealimpl.rs @@ -0,0 +1,37 @@ +use crate::*; + +impl DealInfo { + pub fn complete_part(&mut self, miner: AccountOf, index: u8) -> DispatchResult { + for complete_info in &self.complete_list { + ensure!(index != complete_info.index, Error::::Existed); + ensure!(miner != complete_info.miner, Error::::Existed); + } + + let complete_info = CompleteInfo:: { + index, + miner, + }; + + self.complete_list.try_push(complete_info).map_err(|_| Error::::BoundedVecError)?; + + Ok(()) + } + + pub fn completed_all(&mut self) -> DispatchResult { + self.stage = 2; + for complete_info in self.complete_list.iter() { + >::try_mutate(&complete_info.miner, |pending_space| -> DispatchResult { + let replace_space = FRAGMENT_SIZE + .checked_mul(self.segment_list.len() as u128) + .ok_or(Error::::Overflow)?; + + *pending_space = pending_space + .checked_add(replace_space).ok_or(Error::::Overflow)?; + + Ok(()) + })?; + } + + Ok(()) + } +} \ No newline at end of file diff --git a/c-pallets/file-bank/src/impls/mod.rs b/c-pallets/file-bank/src/impls/mod.rs new file mode 100644 index 00000000..175fcf55 --- /dev/null +++ b/c-pallets/file-bank/src/impls/mod.rs @@ -0,0 +1,5 @@ +pub mod receptionist; +pub use receptionist::*; + +pub mod dealimpl; +pub use dealimpl::*; \ No newline at end of file diff --git a/c-pallets/file-bank/src/impls/receptionist.rs b/c-pallets/file-bank/src/impls/receptionist.rs new file mode 100644 index 00000000..aca74cd4 --- /dev/null +++ b/c-pallets/file-bank/src/impls/receptionist.rs @@ -0,0 +1,82 @@ +use crate::*; + +pub struct Receptionist(PhantomData); + +impl Receptionist { + pub fn fly_upload_file(file_hash: Hash, user_brief: UserBrief, used_space: u128) -> DispatchResult { + T::StorageHandle::update_user_space(&user_brief.user, 1, used_space)?; + + if >::contains_key(&user_brief.user, &user_brief.bucket_name) { + Pallet::::add_file_to_bucket(&user_brief.user, &user_brief.bucket_name, &file_hash)?; + } else { + Pallet::::create_bucket_helper(&user_brief.user, &user_brief.bucket_name, Some(file_hash))?; + } + + Pallet::::add_user_hold_fileslice(&user_brief.user, file_hash, used_space)?; + + >::try_mutate(&file_hash, |file_opt| -> DispatchResult { + let file = file_opt.as_mut().ok_or(Error::::FileNonExistent)?; + file.owner.try_push(user_brief.clone()).map_err(|_e| Error::::BoundedVecError)?; + Ok(()) + })?; + + Ok(()) + } + + pub fn generate_deal( + file_hash: Hash, + deal_info: BoundedVec, T::SegmentCount>, + user_brief: UserBrief, + needed_space: u128, + file_size: u128, + ) -> DispatchResult { + T::StorageHandle::lock_user_space(&user_brief.user, needed_space)?; + // TODO! Replace the file_hash param + Pallet::::generate_deal(file_hash.clone(), deal_info, user_brief.clone(), file_size)?; + + Ok(()) + } + + pub fn qualification_report_processing(sender: AccountOf, deal_hash: Hash, deal_info: &mut DealInfo, index: u8) -> DispatchResult { + deal_info.complete_part(sender.clone(), index)?; + + // If it is the last submitter of the order. + if deal_info.complete_list.len() == FRAGMENT_COUNT as usize { + deal_info.completed_all()?; + Pallet::::generate_file( + &deal_hash, + deal_info.segment_list.clone(), + deal_info.complete_list.clone(), + deal_info.user.clone(), + FileState::Calculate, + deal_info.file_size, + )?; + + let segment_count = deal_info.segment_list.len(); + let needed_space = Pallet::::cal_file_size(segment_count as u128); + T::StorageHandle::unlock_and_used_user_space(&deal_info.user.user, needed_space)?; + T::StorageHandle::sub_total_idle_space(needed_space)?; + T::StorageHandle::add_total_service_space(needed_space)?; + let result = T::FScheduler::cancel_named(deal_hash.0.to_vec()).map_err(|_| Error::::Unexpected); + if let Err(_) = result { + log::info!("transfer report cancel schedule failed: {:?}", deal_hash.clone()); + } + // Calculate the maximum time required for storage nodes to tag files + let max_needed_cal_space = (segment_count as u32).checked_mul(FRAGMENT_SIZE as u32).ok_or(Error::::Overflow)?; + let mut life: u32 = (max_needed_cal_space / TRANSFER_RATE as u32).checked_add(11).ok_or(Error::::Overflow)?; + life = (max_needed_cal_space / CALCULATE_RATE as u32) + .checked_add(30).ok_or(Error::::Overflow)? + .checked_add(life).ok_or(Error::::Overflow)?; + Pallet::::start_second_task(deal_hash.0.to_vec(), deal_hash, life)?; + if >::contains_key(&deal_info.user.user, &deal_info.user.bucket_name) { + Pallet::::add_file_to_bucket(&deal_info.user.user, &deal_info.user.bucket_name, &deal_hash)?; + } else { + Pallet::::create_bucket_helper(&deal_info.user.user, &deal_info.user.bucket_name, Some(deal_hash))?; + } + Pallet::::add_user_hold_fileslice(&deal_info.user.user, deal_hash.clone(), needed_space)?; + Pallet::::deposit_event(Event::::StorageCompleted{ file_hash: deal_hash }); + } + + Ok(()) + } +} \ No newline at end of file diff --git a/c-pallets/file-bank/src/lib.rs b/c-pallets/file-bank/src/lib.rs index 980a141c..397f1fa6 100755 --- a/c-pallets/file-bank/src/lib.rs +++ b/c-pallets/file-bank/src/lib.rs @@ -45,6 +45,9 @@ mod functions; mod constants; use constants::*; +mod impls; +use impls::receptionist::Receptionist; + use codec::{Decode, Encode}; use frame_support::{ // bounded_vec, @@ -423,10 +426,6 @@ pub mod pallet { origin: OriginFor, file_hash: Hash, deal_info: BoundedVec, T::SegmentCount>, - assigned_data: BoundedVec< - BoundedVec, - ConstU32, - >, user_brief: UserBrief, file_size: u128, ) -> DispatchResult { @@ -436,7 +435,7 @@ pub mod pallet { // Check file specifications. ensure!(Self::check_file_spec(&deal_info), Error::::SpecError); // Check whether the user-defined name meets the rules. - + let minimum = T::NameMinLength::get(); ensure!(user_brief.file_name.len() as u32 >= minimum, Error::::SpecError); ensure!(user_brief.bucket_name.len() as u32 >= minimum, Error::::SpecError); @@ -448,38 +447,9 @@ pub mod pallet { ensure!(T::StorageHandle::get_user_avail_space(&user_brief.user)? > needed_space, Error::::InsufficientAvailableSpace); if >::contains_key(&file_hash) { - T::StorageHandle::update_user_space(&user_brief.user, 1, needed_space)?; - - if >::contains_key(&user_brief.user, &user_brief.bucket_name) { - Self::add_file_to_bucket(&user_brief.user, &user_brief.bucket_name, &file_hash)?; - } else { - Self::create_bucket_helper(&user_brief.user, &user_brief.bucket_name, Some(file_hash))?; - } - - Self::add_user_hold_fileslice(&user_brief.user, file_hash, needed_space)?; - - >::try_mutate(&file_hash, |file_opt| -> DispatchResult { - let file = file_opt.as_mut().ok_or(Error::::FileNonExistent)?; - file.owner.try_push(user_brief.clone()).map_err(|_e| Error::::BoundedVecError)?; - Ok(()) - })?; + Receptionist::::fly_upload_file(file_hash, user_brief.clone(), needed_space)?; } else { - ensure!(assigned_data.len() as u32 == ASSIGN_MINER_IDEAL_QUANTITY, Error::::SpecError); - let mut miner_task_list: BoundedVec, ConstU32> = Default::default(); - let mut index = 1; - for fragment_list in assigned_data { - ensure!(fragment_list.len() == deal_info.len(), Error::::SpecError); - let miner_task = MinerTaskList:: { - index, - miner: None, - fragment_list, - }; - miner_task_list.try_push(miner_task).map_err(|_| Error::::BoundedVecError)?; - index += 1; - } - T::StorageHandle::lock_user_space(&user_brief.user, needed_space)?; - // TODO! Replace the file_hash param - Self::generate_deal(file_hash.clone(), deal_info, miner_task_list, user_brief.clone(), file_size)?; + Receptionist::::generate_deal(file_hash, deal_info, user_brief.clone(), needed_space, file_size)?; } Self::deposit_event(Event::::UploadDeclaration { operator: sender, owner: user_brief.user, deal_hash: file_hash }); @@ -619,81 +589,7 @@ pub mod pallet { >::try_mutate(&deal_hash, |deal_info_opt| -> DispatchResult { // can use unwrap because there was a judgment above let deal_info = deal_info_opt.as_mut().unwrap(); - - if !deal_info.complete_list.contains(&index) { - for task_info in &mut deal_info.miner_task_list { - if task_info.index == index { - match task_info.miner { - Some(_) => Err(Error::::Existed)?, - None => { - task_info.miner = Some(sender.clone()); - T::MinerControl::lock_space(&sender, task_info.fragment_list.len() as u128 * FRAGMENT_SIZE)?; - deal_info.complete_list.try_push(index).map_err(|_| Error::::BoundedVecError)?; - }, - }; - } else { - if let Some(miner) = &task_info.miner { - if miner == &sender { - Err(Error::::Existed)?; - } - }; - } - } - - - // If it is the last submitter of the order. - if deal_info.complete_list.len() == deal_info.miner_task_list.len() { - deal_info.stage = 2; - Self::generate_file( - &deal_hash, - deal_info.segment_list.clone(), - deal_info.miner_task_list.clone(), - deal_info.user.clone(), - FileState::Calculate, - deal_info.file_size, - )?; - let mut max_task_count = 0; - for miner_task in deal_info.miner_task_list.iter() { - let count = miner_task.fragment_list.len() as u128; - if count > max_task_count { - max_task_count = count; - } - // Miners need to report the replaced documents themselves. - // If a challenge is triggered before the report is completed temporarily, - // these documents to be replaced also need to be verified - if let Some(miner) = &miner_task.miner { - >::try_mutate(miner, |pending_space| -> DispatchResult { - let replace_space = FRAGMENT_SIZE.checked_mul(count).ok_or(Error::::Overflow)?; - let pending_space_temp = pending_space.checked_add(replace_space).ok_or(Error::::Overflow)?; - *pending_space = pending_space_temp; - Ok(()) - })?; - } - } - let needed_space = Self::cal_file_size(deal_info.segment_list.len() as u128); - T::StorageHandle::unlock_and_used_user_space(&deal_info.user.user, needed_space)?; - T::StorageHandle::sub_total_idle_space(needed_space)?; - T::StorageHandle::add_total_service_space(needed_space)?; - let result = T::FScheduler::cancel_named(deal_hash.0.to_vec()).map_err(|_| Error::::Unexpected); - if let Err(_) = result { - log::info!("transfer report cancel schedule failed: {:?}", deal_hash.clone()); - } - // Calculate the maximum time required for storage nodes to tag files - let max_needed_cal_space = (max_task_count as u32).checked_mul(FRAGMENT_SIZE as u32).ok_or(Error::::Overflow)?; - let mut life: u32 = (max_needed_cal_space / TRANSFER_RATE as u32).checked_add(11).ok_or(Error::::Overflow)?; - life = (max_needed_cal_space / CALCULATE_RATE as u32) - .checked_add(30).ok_or(Error::::Overflow)? - .checked_add(life).ok_or(Error::::Overflow)?; - Self::start_second_task(deal_hash.0.to_vec(), deal_hash, life)?; - if >::contains_key(&deal_info.user.user, &deal_info.user.bucket_name) { - Self::add_file_to_bucket(&deal_info.user.user, &deal_info.user.bucket_name, &deal_hash)?; - } else { - Self::create_bucket_helper(&deal_info.user.user, &deal_info.user.bucket_name, Some(deal_hash))?; - } - Self::add_user_hold_fileslice(&deal_info.user.user, deal_hash.clone(), needed_space)?; - Self::deposit_event(Event::::StorageCompleted{ file_hash: deal_hash }); - } - } + Receptionist::::qualification_report_processing(sender.clone(), deal_hash, deal_info, index)?; Ok(()) })?; @@ -719,18 +615,18 @@ pub mod pallet { let _ = ensure_root(origin)?; let deal_info = >::try_get(&deal_hash).map_err(|_| Error::::NonExistent)?; - for miner_task in deal_info.miner_task_list { - let count = miner_task.fragment_list.len() as u32; - let mut hash_list: Vec> = Default::default(); - for fragment_hash in miner_task.fragment_list { + for (index, complete_info) in deal_info.complete_list.iter().enumerate() { + let count = FRAGMENT_COUNT; + let mut hash_list: Vec> = Default::default(); + for segment in &deal_info.segment_list { + let fragment_hash = segment.fragment_list[index as usize]; let hash_temp = fragment_hash.binary().map_err(|_| Error::::BugInvalid)?; hash_list.push(hash_temp); } // Accumulate the number of fragments stored by each miner let unlock_space = FRAGMENT_SIZE.checked_mul(count as u128).ok_or(Error::::Overflow)?; - let miner = miner_task.miner.ok_or(Error::::Unexpected)?; - T::MinerControl::unlock_space_to_service(&miner, unlock_space)?; - T::MinerControl::insert_service_bloom(&miner, hash_list)?; + T::MinerControl::unlock_space_to_service(&complete_info.miner, unlock_space)?; + T::MinerControl::insert_service_bloom(&complete_info.miner, hash_list)?; } >::try_mutate(&deal_hash, |file_opt| -> DispatchResult { diff --git a/c-pallets/file-bank/src/types.rs b/c-pallets/file-bank/src/types.rs index eb53eab4..320ab863 100755 --- a/c-pallets/file-bank/src/types.rs +++ b/c-pallets/file-bank/src/types.rs @@ -43,8 +43,15 @@ pub struct DealInfo { pub(super) file_size: u128, pub(super) segment_list: BoundedVec, T::SegmentCount>, pub(super) user: UserBrief, - pub(super) miner_task_list: BoundedVec, ConstU32>, - pub(super) complete_list: BoundedVec, + pub(super) complete_list: BoundedVec, T::FragmentCount>, +} + +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, MaxEncodedLen, TypeInfo)] +#[scale_info(skip_type_params(T))] +#[codec(mel_bound())] +pub struct CompleteInfo { + pub(super) index: u8, + pub(super) miner: AccountOf, } //TODO! BoundedVec type -> BTreeMap