diff --git a/ant-cli/src/access/user_data.rs b/ant-cli/src/access/user_data.rs index 57deb85785..3fc20785cd 100644 --- a/ant-cli/src/access/user_data.rs +++ b/ant-cli/src/access/user_data.rs @@ -10,8 +10,7 @@ use std::collections::HashMap; use autonomi::client::{ address::{addr_to_str, str_to_addr}, - archive::ArchiveAddr, - archive_private::PrivateArchiveAccess, + files::{archive::PrivateArchiveAccess, archive_public::ArchiveAddr}, registers::{RegisterAddress, RegisterSecretKey}, vault::UserData, }; diff --git a/ant-cli/src/actions/download.rs b/ant-cli/src/actions/download.rs index ff737ac2c1..6b3bbd380c 100644 --- a/ant-cli/src/actions/download.rs +++ b/ant-cli/src/actions/download.rs @@ -8,7 +8,10 @@ use super::get_progress_bar; use autonomi::{ - client::{address::str_to_addr, archive::ArchiveAddr, archive_private::PrivateArchiveAccess}, + client::{ + address::str_to_addr, + files::{archive::PrivateArchiveAccess, archive_public::ArchiveAddr}, + }, Client, }; use color_eyre::{ @@ -40,7 +43,7 @@ async fn download_private( client: &mut Client, ) -> Result<()> { let archive = client - .private_archive_get(private_address) + .archive_get(private_address) .await .wrap_err("Failed to fetch data from address")?; @@ -48,7 +51,7 @@ async fn download_private( let mut all_errs = vec![]; for (path, access, _meta) in archive.iter() { progress_bar.println(format!("Fetching file: {path:?}...")); - let bytes = match client.private_data_get(access.clone()).await { + let bytes = match client.data_get(access.clone()).await { Ok(bytes) => bytes, Err(e) => { let err = format!("Failed to fetch file {path:?}: {e}"); @@ -86,7 +89,7 @@ async fn download_public( client: &mut Client, ) -> Result<()> { let archive = client - .archive_get(address) + .archive_get_public(address) .await .wrap_err("Failed to fetch data from address")?; @@ -94,7 +97,7 @@ async fn download_public( let mut all_errs = vec![]; for (path, addr, _meta) in archive.iter() { progress_bar.println(format!("Fetching file: {path:?}...")); - let bytes = match client.data_get(*addr).await { + let bytes = match client.data_get_public(*addr).await { Ok(bytes) => bytes, Err(e) => { let err = format!("Failed to fetch file {path:?}: {e}"); diff --git a/ant-cli/src/commands/file.rs b/ant-cli/src/commands/file.rs index 6d3f051015..146133e348 100644 --- a/ant-cli/src/commands/file.rs +++ b/ant-cli/src/commands/file.rs @@ -53,16 +53,17 @@ pub async fn upload(file: &str, public: bool, peers: Vec) -> Result<( let local_addr; let archive = if public { let xor_name = client - .dir_upload(dir_path, &wallet) + .dir_and_archive_upload_public(dir_path, &wallet) .await .wrap_err("Failed to upload file")?; local_addr = addr_to_str(xor_name); local_addr.clone() } else { let private_data_access = client - .private_dir_upload(dir_path, &wallet) + .dir_and_archive_upload(dir_path, &wallet) .await - .wrap_err("Failed to upload file")?; + .wrap_err("Failed to upload dir and archive")?; + local_addr = private_data_access.address(); private_data_access.to_hex() }; diff --git a/ant-node/tests/data_with_churn.rs b/ant-node/tests/data_with_churn.rs index ffe2a879ab..64b3064350 100644 --- a/ant-node/tests/data_with_churn.rs +++ b/ant-node/tests/data_with_churn.rs @@ -338,7 +338,7 @@ fn store_chunks_task( let mut retries = 1; loop { match client - .data_put(random_data.clone(), (&wallet).into()) + .data_put_public(random_data.clone(), (&wallet).into()) .await .inspect_err(|err| { println!("Error to put chunk: {err:?}"); @@ -537,7 +537,7 @@ async fn query_content(client: &Client, net_addr: &NetworkAddress) -> Result<()> Ok(()) } NetworkAddress::ChunkAddress(addr) => { - client.data_get(*addr.xorname()).await?; + client.data_get_public(*addr.xorname()).await?; Ok(()) } _other => Ok(()), // we don't create/store any other type of content in this test yet diff --git a/ant-node/tests/storage_payments.rs b/ant-node/tests/storage_payments.rs index d2aabead94..bfb6d4ae75 100644 --- a/ant-node/tests/storage_payments.rs +++ b/ant-node/tests/storage_payments.rs @@ -205,7 +205,7 @@ // let _upload_stats = uploader.start_upload().await?; // let mut files_download = FilesDownload::new(files_api); -// let _ = files_download.download_file(file_addr, None).await?; +// let _ = files_download.file_download_public(file_addr, None).await?; // Ok(()) // } @@ -252,7 +252,7 @@ // let mut files_download = FilesDownload::new(files_api); // assert!( // matches!( -// files_download.download_file(content_addr, None).await, +// files_download.file_download_public(content_addr, None).await, // Err(ClientError::Network(NetworkError::GetRecordError( // GetRecordError::RecordNotFound // ))) diff --git a/ant-node/tests/verify_data_location.rs b/ant-node/tests/verify_data_location.rs index efdd848df8..0a82634ffe 100644 --- a/ant-node/tests/verify_data_location.rs +++ b/ant-node/tests/verify_data_location.rs @@ -351,7 +351,7 @@ async fn store_chunks( let random_bytes = Bytes::from(random_bytes); - client.data_put(random_bytes, wallet.into()).await?; + client.data_put_public(random_bytes, wallet.into()).await?; uploaded_chunks_count += 1; diff --git a/autonomi/README.md b/autonomi/README.md index c781c46bf9..63235554a1 100644 --- a/autonomi/README.md +++ b/autonomi/README.md @@ -28,14 +28,14 @@ async fn main() -> Result<(), Box> { // Put and fetch data. let data_addr = client - .data_put(Bytes::from("Hello, World"), (&wallet).into()) + .data_put_public(Bytes::from("Hello, World"), (&wallet).into()) .await?; - let _data_fetched = client.data_get(data_addr).await?; + let _data_fetched = client.data_get_public(data_addr).await?; // Put and fetch directory from local file system. - let dir_addr = client.dir_upload("files/to/upload".into(), &wallet).await?; + let dir_addr = client.dir_and_archive_upload_public("files/to/upload".into(), &wallet).await?; client - .dir_download(dir_addr, "files/downloaded".into()) + .dir_download_public(dir_addr, "files/downloaded".into()) .await?; Ok(()) @@ -53,52 +53,44 @@ let wallet = Wallet::new_from_private_key(EvmNetwork::new_custom("", "< ## Running tests -### Using a local EVM testnet +To run the tests, we can run a local network: -1. If you haven't, install Foundry, to be able to run Anvil - nodes: https://book.getfoundry.sh/getting-started/installation -2. Run a local EVM node: +1. Run a local EVM node: + > Note: To run the EVM node, Foundry is required to be installed: https://book.getfoundry.sh/getting-started/installation -```sh -cargo run --bin=evm-testnet -``` - -3. Run a local network with the `local` feature and use the local evm node. - -```sh -cargo run --bin=antctl --features=local -- local run --build --clean --rewards-address= evm-local -``` + ```sh + cargo run --bin evm-testnet + ``` -4. Then run the tests with the `local` feature and pass the EVM params again: +2. Run a local network with the `local` feature and use the local EVM node. + ```sh + cargo run --bin antctl --features local -- local run --build --clean --rewards-address evm-local + ``` -```sh -EVM_NETWORK=local cargo test --package autonomi --features=local -# Or with logs -RUST_LOG=autonomi EVM_NETWORK=local cargo test --package autonomi --features local -- --nocapture -``` +3. Then run the tests with the `local` feature and pass the EVM params again: + ```sh + EVM_NETWORK=local cargo test --features local --package autonomi + ``` ### Using a live testnet or mainnet -Using the hardcoded `Arbitrum One` option as an example, but you can also use the command flags of the steps above and -point it to a live network. +Using the hardcoded `Arbitrum One` option as an example, but you can also use the command flags of the steps above and point it to a live network. 1. Run a local network with the `local` feature: ```sh -cargo run --bin=antctl --features=local -- local run --build --clean --rewards-address= evm-arbitrum-one +cargo run --bin antctl --features local -- local run --build --clean --rewards-address evm-arbitrum-one ``` -2. Then run the tests with the `local` feature. Make sure that the wallet of the private key you pass has enough gas and - payment tokens on the network (in this case Arbitrum One): +2. Then pass the private key of the wallet, and ensure it has enough gas and payment tokens on the network (in this case Arbitrum One): ```sh -EVM_NETWORK=arbitrum-one EVM_PRIVATE_KEY= cargo test --package=autonomi --features=local +EVM_NETWORK=arbitrum-one EVM_PRIVATE_KEY= cargo test --package autonomi --features local ``` ## Using funds from the Deployer Wallet -You can use the `Deployer wallet private key` printed in the EVM node output to -initialise a wallet from with almost infinite gas and payment tokens. Example: +You can use the `Deployer wallet private key` printed in the EVM node output to initialise a wallet from with almost infinite gas and payment tokens. Example: ```rust let rpc_url = "http://localhost:54370/"; @@ -107,9 +99,9 @@ let data_payments_address = "0x8464135c8F25Da09e49BC8782676a84730C318bC"; let private_key = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; let network = Network::Custom(CustomNetwork::new( -rpc_url, -payment_token_address, -data_payments_address, + rpc_url, + payment_token_address, + data_payments_address, )); let deployer_wallet = Wallet::new_from_private_key(network, private_key).unwrap(); @@ -117,15 +109,15 @@ let receiving_wallet = Wallet::new_with_random_wallet(network); // Send 10 payment tokens (atto) let _ = deployer_wallet -.transfer_tokens(receiving_wallet.address(), Amount::from(10)) -.await; + .transfer_tokens(receiving_wallet.address(), Amount::from(10)) + .await; ``` Alternatively, you can provide the wallet address that should own all the gas and payment tokens to the EVM testnet startup command using the `--genesis-wallet` flag: ```sh -cargo run --bin evm-testnet -- --genesis-wallet= +cargo run --bin evm-testnet -- --genesis-wallet ``` ```shell diff --git a/autonomi/README_PYTHON.md b/autonomi/README_PYTHON.md index 9bbb5a79b8..6772ce14a1 100644 --- a/autonomi/README_PYTHON.md +++ b/autonomi/README_PYTHON.md @@ -26,11 +26,11 @@ payment = PaymentOption.wallet(wallet) # Upload data data = b"Hello, Safe Network!" -addr = client.data_put(data, payment) +addr = client.data_put_public(data, payment) print(f"Data uploaded to: {addr}") # Download data -retrieved = client.data_get(addr) +retrieved = client.data_get_public(addr) print(f"Retrieved: {retrieved.decode()}") ``` @@ -40,10 +40,10 @@ print(f"Retrieved: {retrieved.decode()}") - `Client`: Main interface to the Autonomi network - `connect(peers: List[str])`: Connect to network nodes - - `data_put(data: bytes, payment: PaymentOption)`: Upload data - - `data_get(addr: str)`: Download data - - `private_data_put(data: bytes, payment: PaymentOption)`: Store private data - - `private_data_get(access: PrivateDataAccess)`: Retrieve private data + - `data_put_public(data: bytes, payment: PaymentOption)`: Upload data + - `data_get_public(addr: str)`: Download data + - `data_put(data: bytes, payment: PaymentOption)`: Store private data + - `data_get(access: DataMapChunk)`: Retrieve private data - `register_generate_key()`: Generate register key - `Wallet`: Ethereum wallet management @@ -56,16 +56,16 @@ print(f"Retrieved: {retrieved.decode()}") #### Private Data -- `PrivateDataAccess`: Handle private data storage +- `DataMapChunk`: Handle private data storage - `from_hex(hex: str)`: Create from hex string - `to_hex()`: Convert to hex string - `address()`: Get short reference address ```python # Private data example -access = client.private_data_put(secret_data, payment) +access = client.data_put(secret_data, payment) print(f"Private data stored at: {access.to_hex()}") -retrieved = client.private_data_get(access) +retrieved = client.data_get(access) ``` #### Registers @@ -117,15 +117,15 @@ data, content_type = client.fetch_and_decrypt_vault(vault_key) def handle_data_operations(client, payment): # Upload text text_data = b"Hello, Safe Network!" - text_addr = client.data_put(text_data, payment) + text_addr = client.data_put_public(text_data, payment) # Upload binary data with open("image.jpg", "rb") as f: image_data = f.read() - image_addr = client.data_put(image_data, payment) + image_addr = client.data_put_public(image_data, payment) # Download and verify - downloaded = client.data_get(text_addr) + downloaded = client.data_get_public(text_addr) assert downloaded == text_data ``` @@ -138,11 +138,11 @@ def handle_private_data(client, payment): data = json.dumps(secret).encode() # Store privately - access = client.private_data_put(data, payment) + access = client.data_put(data, payment) print(f"Access token: {access.to_hex()}") # Retrieve - retrieved = client.private_data_get(access) + retrieved = client.data_get(access) secret = json.loads(retrieved.decode()) ``` diff --git a/autonomi/examples/autonomi_advanced.py b/autonomi/examples/autonomi_advanced.py index 310766192e..25e6333cb3 100644 --- a/autonomi/examples/autonomi_advanced.py +++ b/autonomi/examples/autonomi_advanced.py @@ -25,7 +25,7 @@ def connect_to_network(peers: list[str]) -> Client: def upload_data(client: Client, data: bytes, payment: PaymentOption) -> str: try: - addr = client.data_put(data, payment) + addr = client.data_put_public(data, payment) print(f"Successfully uploaded data to: {addr}") return addr except Exception as e: @@ -34,7 +34,7 @@ def upload_data(client: Client, data: bytes, payment: PaymentOption) -> str: def download_data(client: Client, addr: str) -> bytes: try: - data = client.data_get(addr) + data = client.data_get_public(addr) print(f"Successfully downloaded {len(data)} bytes") return data except Exception as e: diff --git a/autonomi/examples/autonomi_data_registers.py b/autonomi/examples/autonomi_data_registers.py index a7b8ba42ff..4d258fefa1 100644 --- a/autonomi/examples/autonomi_data_registers.py +++ b/autonomi/examples/autonomi_data_registers.py @@ -7,22 +7,22 @@ def handle_data_operations(client: Client, payment: PaymentOption): # Upload some text data text_data = b"Hello, Safe Network!" - text_addr = client.data_put(text_data, payment) + text_addr = client.data_put_public(text_data, payment) print(f"Text data uploaded to: {text_addr}") # Upload binary data (like an image) with open("example.jpg", "rb") as f: image_data = f.read() - image_addr = client.data_put(image_data, payment) + image_addr = client.data_put_public(image_data, payment) print(f"Image uploaded to: {image_addr}") # Download and verify data - downloaded_text = client.data_get(text_addr) + downloaded_text = client.data_get_public(text_addr) assert downloaded_text == text_data, "Text data verification failed!" print("Text data verified successfully") # Download and save image - downloaded_image = client.data_get(image_addr) + downloaded_image = client.data_get_public(image_addr) with open("downloaded_example.jpg", "wb") as f: f.write(downloaded_image) print("Image downloaded successfully") diff --git a/autonomi/examples/autonomi_example.py b/autonomi/examples/autonomi_example.py index 496446173c..14d6bbfc0e 100644 --- a/autonomi/examples/autonomi_example.py +++ b/autonomi/examples/autonomi_example.py @@ -21,17 +21,17 @@ def main(): # Upload some data data = b"Hello, Safe Network!" - addr = client.data_put(data, payment) + addr = client.data_put_public(data, payment) print(f"Data uploaded to address: {addr}") # Download the data back - downloaded = client.data_get(addr) + downloaded = client.data_get_public(addr) print(f"Downloaded data: {downloaded.decode()}") # You can also upload files with open("example.txt", "rb") as f: file_data = f.read() - file_addr = client.data_put(file_data, payment) + file_addr = client.data_put_public(file_data, payment) print(f"File uploaded to address: {file_addr}") if __name__ == "__main__": diff --git a/autonomi/examples/autonomi_private_data.py b/autonomi/examples/autonomi_private_data.py index 3b0d9327e4..4d68acd3ea 100644 --- a/autonomi/examples/autonomi_private_data.py +++ b/autonomi/examples/autonomi_private_data.py @@ -10,12 +10,12 @@ def __init__(self, client: Client, wallet: Wallet): def store_private_data(self, data: bytes) -> str: """Store data privately and return its address""" - addr = self.client.private_data_put(data, self.payment) + addr = self.client.data_put(data, self.payment) return addr def retrieve_private_data(self, addr: str) -> bytes: """Retrieve privately stored data""" - return self.client.private_data_get(addr) + return self.client.data_get(addr) def create_shared_register(self, name: str, initial_value: bytes, allowed_writers: List[str]) -> str: diff --git a/autonomi/examples/autonomi_private_encryption.py b/autonomi/examples/autonomi_private_encryption.py index 7f71a6b8d6..e95bcc5386 100644 --- a/autonomi/examples/autonomi_private_encryption.py +++ b/autonomi/examples/autonomi_private_encryption.py @@ -1,5 +1,5 @@ from autonomi_client import ( - Client, Wallet, PaymentOption, PrivateDataAccess, + Client, Wallet, PaymentOption, DataMapChunk, encrypt, hash_to_short_string ) import json @@ -16,12 +16,12 @@ def demonstrate_private_data(client: Client, payment: PaymentOption): data_bytes = json.dumps(secret_data).encode() # Store it privately - access = client.private_data_put(data_bytes, payment) + access = client.data_put(data_bytes, payment) print(f"Stored private data, access token: {access.to_hex()}") print(f"Short reference: {access.address()}") # Retrieve it - retrieved_bytes = client.private_data_get(access) + retrieved_bytes = client.data_get(access) retrieved_data = json.loads(retrieved_bytes.decode()) print(f"Retrieved private data: {retrieved_data}") diff --git a/autonomi/examples/basic.py b/autonomi/examples/basic.py index b7d8f21619..4ddaee182c 100644 --- a/autonomi/examples/basic.py +++ b/autonomi/examples/basic.py @@ -22,15 +22,15 @@ def main(): # Upload public data data = b"Hello World!" - addr = client.data_put(data, wallet) + addr = client.data_put_public(data, wallet) print(f"Uploaded public data to: {addr}") - retrieved = client.data_get(addr) + retrieved = client.data_get_public(addr) print(f"Retrieved public data: {retrieved}") # Upload private data - private_access = client.private_data_put(b"Secret message", wallet) + private_access = client.data_put(b"Secret message", wallet) print(f"Private data access: {private_access}") - private_data = client.private_data_get(private_access) + private_data = client.data_get(private_access) print(f"Retrieved private data: {private_data}") # Create register @@ -40,7 +40,7 @@ def main(): print(f"Register values: {reg_values}") # Upload file/directory - file_addr = client.file_upload("./test_data", wallet) + file_addr = client.file_upload_public("./test_data", wallet) print(f"Uploaded files to: {file_addr}") client.file_download(file_addr, "./downloaded_data") print("Downloaded files") @@ -58,9 +58,9 @@ def main(): print(f"Retrieved user data: {retrieved_data}") # Private directory operations - private_dir_access = client.private_dir_upload("./test_data", wallet) + private_dir_access = client.dir_upload("./test_data", wallet) print(f"Uploaded private directory, access: {private_dir_access}") - client.private_dir_download(private_dir_access, "./downloaded_private") + client.dir_download(private_dir_access, "./downloaded_private") print("Downloaded private directory") # External signer example diff --git a/autonomi/examples/put_and_dir_upload.rs b/autonomi/examples/put_and_dir_upload.rs index f90480d101..874ca57980 100644 --- a/autonomi/examples/put_and_dir_upload.rs +++ b/autonomi/examples/put_and_dir_upload.rs @@ -10,14 +10,16 @@ async fn main() -> Result<(), Box> { // Put and fetch data. let data_addr = client - .data_put(Bytes::from("Hello, World"), (&wallet).into()) + .data_put_public(Bytes::from("Hello, World"), (&wallet).into()) .await?; - let _data_fetched = client.data_get(data_addr).await?; + let _data_fetched = client.data_get_public(data_addr).await?; // Put and fetch directory from local file system. - let dir_addr = client.dir_upload("files/to/upload".into(), &wallet).await?; + let dir_addr = client + .dir_and_archive_upload_public("files/to/upload".into(), &wallet) + .await?; client - .dir_download(dir_addr, "files/downloaded".into()) + .dir_download_public(dir_addr, "files/downloaded".into()) .await?; Ok(()) diff --git a/autonomi/python/autonomi_client/__init__.py b/autonomi/python/autonomi_client/__init__.py index 11d550e79d..b1e437b894 100644 --- a/autonomi/python/autonomi_client/__init__.py +++ b/autonomi/python/autonomi_client/__init__.py @@ -1,4 +1,4 @@ -from .autonomi_client import Client, Wallet, PaymentOption, VaultSecretKey, UserData, PrivateDataAccess, encrypt +from .autonomi_client import Client, Wallet, PaymentOption, VaultSecretKey, UserData, DataMapChunk, encrypt __all__ = [ "Client", @@ -6,6 +6,6 @@ "PaymentOption", "VaultSecretKey", "UserData", - "PrivateDataAccess", + "DataMapChunk", "encrypt" ] diff --git a/autonomi/src/client/data/mod.rs b/autonomi/src/client/data/mod.rs new file mode 100644 index 0000000000..b85f54a68e --- /dev/null +++ b/autonomi/src/client/data/mod.rs @@ -0,0 +1,239 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::sync::LazyLock; + +use ant_evm::{Amount, EvmWalletError}; +use ant_networking::NetworkError; +use ant_protocol::storage::Chunk; +use ant_protocol::NetworkAddress; +use bytes::Bytes; +use serde::{Deserialize, Serialize}; +use xor_name::XorName; + +use crate::client::payment::PaymentOption; +use crate::client::{ClientEvent, UploadSummary}; +use crate::{self_encryption::encrypt, Client}; + +pub mod public; + +/// Number of chunks to upload in parallel. +/// +/// Can be overridden by the `CHUNK_UPLOAD_BATCH_SIZE` environment variable. +pub(crate) static CHUNK_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { + let batch_size = std::env::var("CHUNK_UPLOAD_BATCH_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or( + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + * 8, + ); + info!("Chunk upload batch size: {}", batch_size); + batch_size +}); + +/// Number of chunks to download in parallel. +/// +/// Can be overridden by the `CHUNK_DOWNLOAD_BATCH_SIZE` environment variable. +pub static CHUNK_DOWNLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { + let batch_size = std::env::var("CHUNK_DOWNLOAD_BATCH_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or( + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + * 8, + ); + info!("Chunk download batch size: {}", batch_size); + batch_size +}); + +/// Number of retries to upload chunks. +pub(crate) const RETRY_ATTEMPTS: usize = 3; + +/// Raw Data Address (points to a DataMap) +pub type DataAddr = XorName; +/// Raw Chunk Address (points to a [`Chunk`]) +pub type ChunkAddr = XorName; + +/// Errors that can occur during the put operation. +#[derive(Debug, thiserror::Error)] +pub enum PutError { + #[error("Failed to self-encrypt data.")] + SelfEncryption(#[from] crate::self_encryption::Error), + #[error("A network error occurred.")] + Network(#[from] NetworkError), + #[error("Error occurred during cost estimation.")] + CostError(#[from] CostError), + #[error("Error occurred during payment.")] + PayError(#[from] PayError), + #[error("Serialization error: {0}")] + Serialization(String), + #[error("A wallet error occurred.")] + Wallet(#[from] ant_evm::EvmError), + #[error("The vault owner key does not match the client's public key")] + VaultBadOwner, + #[error("Payment unexpectedly invalid for {0:?}")] + PaymentUnexpectedlyInvalid(NetworkAddress), +} + +/// Errors that can occur during the pay operation. +#[derive(Debug, thiserror::Error)] +pub enum PayError { + #[error("Wallet error: {0:?}")] + EvmWalletError(#[from] EvmWalletError), + #[error("Failed to self-encrypt data.")] + SelfEncryption(#[from] crate::self_encryption::Error), + #[error("Cost error: {0:?}")] + Cost(#[from] CostError), +} + +/// Errors that can occur during the get operation. +#[derive(Debug, thiserror::Error)] +pub enum GetError { + #[error("Could not deserialize data map.")] + InvalidDataMap(rmp_serde::decode::Error), + #[error("Failed to decrypt data.")] + Decryption(crate::self_encryption::Error), + #[error("Failed to deserialize")] + Deserialization(#[from] rmp_serde::decode::Error), + #[error("General networking error: {0:?}")] + Network(#[from] NetworkError), + #[error("General protocol error: {0:?}")] + Protocol(#[from] ant_protocol::Error), +} + +/// Errors that can occur during the cost calculation. +#[derive(Debug, thiserror::Error)] +pub enum CostError { + #[error("Failed to self-encrypt data.")] + SelfEncryption(#[from] crate::self_encryption::Error), + #[error("Could not get store quote for: {0:?} after several retries")] + CouldNotGetStoreQuote(XorName), + #[error("Could not get store costs: {0:?}")] + CouldNotGetStoreCosts(NetworkError), + #[error("Failed to serialize {0}")] + Serialization(String), +} + +/// Private data on the network can be accessed with this +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct DataMapChunk(Chunk); + +impl DataMapChunk { + pub fn to_hex(&self) -> String { + hex::encode(self.0.value()) + } + + pub fn from_hex(hex: &str) -> Result { + let data = hex::decode(hex)?; + Ok(Self(Chunk::new(Bytes::from(data)))) + } + + /// Get a private address for [`DataMapChunk`]. Note that this is not a network address, it is only used for refering to private data client side. + pub fn address(&self) -> String { + hash_to_short_string(&self.to_hex()) + } +} + +fn hash_to_short_string(input: &str) -> String { + let mut hasher = DefaultHasher::new(); + input.hash(&mut hasher); + let hash_value = hasher.finish(); + hash_value.to_string() +} + +impl Client { + /// Fetch a blob of (private) data from the network + pub async fn data_get(&self, data_map: DataMapChunk) -> Result { + info!( + "Fetching private data from Data Map {:?}", + data_map.0.address() + ); + let data = self.fetch_from_data_map_chunk(data_map.0.value()).await?; + + debug!("Successfully fetched a blob of private data from the network"); + Ok(data) + } + + /// Upload a piece of private data to the network. This data will be self-encrypted. + /// The [`DataMapChunk`] is not uploaded to the network, keeping the data private. + /// + /// Returns the [`DataMapChunk`] containing the map to the encrypted chunks. + pub async fn data_put( + &self, + data: Bytes, + payment_option: PaymentOption, + ) -> Result { + let now = ant_networking::target_arch::Instant::now(); + let (data_map_chunk, chunks) = encrypt(data)?; + debug!("Encryption took: {:.2?}", now.elapsed()); + + // Pay for all chunks + let xor_names: Vec<_> = chunks.iter().map(|chunk| *chunk.name()).collect(); + info!("Paying for {} addresses", xor_names.len()); + let receipt = self + .pay_for_content_addrs(xor_names.into_iter(), payment_option) + .await + .inspect_err(|err| error!("Error paying for data: {err:?}"))?; + + // Upload the chunks with the payments + debug!("Uploading {} chunks", chunks.len()); + + let mut failed_uploads = self + .upload_chunks_with_retries(chunks.iter().collect(), &receipt) + .await; + + // Return the last chunk upload error + if let Some(last_chunk_fail) = failed_uploads.pop() { + tracing::error!( + "Error uploading chunk ({:?}): {:?}", + last_chunk_fail.0.address(), + last_chunk_fail.1 + ); + return Err(last_chunk_fail.1); + } + + let record_count = chunks.len(); + + // Reporting + if let Some(channel) = self.client_event_sender.as_ref() { + let tokens_spent = receipt + .values() + .map(|proof| proof.quote.cost.as_atto()) + .sum::(); + + let summary = UploadSummary { + record_count, + tokens_spent, + }; + if let Err(err) = channel.send(ClientEvent::UploadComplete(summary)).await { + error!("Failed to send client event: {err:?}"); + } + } + + Ok(DataMapChunk(data_map_chunk)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hex() { + let data_map = DataMapChunk(Chunk::new(Bytes::from_static(b"hello"))); + let hex = data_map.to_hex(); + let data_map2 = DataMapChunk::from_hex(&hex).expect("Failed to decode hex"); + assert_eq!(data_map, data_map2); + } +} diff --git a/autonomi/src/client/data.rs b/autonomi/src/client/data/public.rs similarity index 69% rename from autonomi/src/client/data.rs rename to autonomi/src/client/data/public.rs index e7f5d80a8e..a4ff4e1a40 100644 --- a/autonomi/src/client/data.rs +++ b/autonomi/src/client/data/public.rs @@ -10,124 +10,25 @@ use bytes::Bytes; use libp2p::kad::Quorum; use std::collections::{HashMap, HashSet}; -use std::sync::LazyLock; use xor_name::XorName; use crate::client::payment::PaymentOption; use crate::client::utils::process_tasks_with_max_concurrency; use crate::client::{ClientEvent, UploadSummary}; use crate::{self_encryption::encrypt, Client}; +use ant_evm::ProofOfPayment; use ant_evm::{Amount, AttoTokens}; -use ant_evm::{EvmWalletError, ProofOfPayment}; use ant_networking::{GetRecordCfg, NetworkError}; use ant_protocol::{ storage::{try_deserialize_record, Chunk, ChunkAddress, RecordHeader, RecordKind}, NetworkAddress, }; -/// Number of chunks to upload in parallel. -/// Can be overridden by the `CHUNK_UPLOAD_BATCH_SIZE` environment variable. -pub static CHUNK_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { - let batch_size = std::env::var("CHUNK_UPLOAD_BATCH_SIZE") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or( - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1) - * 8, - ); - info!("Chunk upload batch size: {}", batch_size); - batch_size -}); - -/// Number of retries to upload chunks. -pub const RETRY_ATTEMPTS: usize = 3; - -/// Number of chunks to download in parallel. -/// Can be overridden by the `CHUNK_DOWNLOAD_BATCH_SIZE` environment variable. -pub static CHUNK_DOWNLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { - let batch_size = std::env::var("CHUNK_DOWNLOAD_BATCH_SIZE") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or( - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1) - * 8, - ); - info!("Chunk download batch size: {}", batch_size); - batch_size -}); - -/// Raw Data Address (points to a DataMap) -pub type DataAddr = XorName; -/// Raw Chunk Address (points to a [`Chunk`]) -pub type ChunkAddr = XorName; - -/// Errors that can occur during the put operation. -#[derive(Debug, thiserror::Error)] -pub enum PutError { - #[error("Failed to self-encrypt data.")] - SelfEncryption(#[from] crate::self_encryption::Error), - #[error("A network error occurred.")] - Network(#[from] NetworkError), - #[error("Error occurred during cost estimation.")] - CostError(#[from] CostError), - #[error("Error occurred during payment.")] - PayError(#[from] PayError), - #[error("Serialization error: {0}")] - Serialization(String), - #[error("A wallet error occurred.")] - Wallet(#[from] ant_evm::EvmError), - #[error("The vault owner key does not match the client's public key")] - VaultBadOwner, - #[error("Payment unexpectedly invalid for {0:?}")] - PaymentUnexpectedlyInvalid(NetworkAddress), -} - -/// Errors that can occur during the pay operation. -#[derive(Debug, thiserror::Error)] -pub enum PayError { - #[error("Wallet error: {0:?}")] - EvmWalletError(#[from] EvmWalletError), - #[error("Failed to self-encrypt data.")] - SelfEncryption(#[from] crate::self_encryption::Error), - #[error("Cost error: {0:?}")] - Cost(#[from] CostError), -} - -/// Errors that can occur during the get operation. -#[derive(Debug, thiserror::Error)] -pub enum GetError { - #[error("Could not deserialize data map.")] - InvalidDataMap(rmp_serde::decode::Error), - #[error("Failed to decrypt data.")] - Decryption(crate::self_encryption::Error), - #[error("Failed to deserialize")] - Deserialization(#[from] rmp_serde::decode::Error), - #[error("General networking error: {0:?}")] - Network(#[from] NetworkError), - #[error("General protocol error: {0:?}")] - Protocol(#[from] ant_protocol::Error), -} - -/// Errors that can occur during the cost calculation. -#[derive(Debug, thiserror::Error)] -pub enum CostError { - #[error("Failed to self-encrypt data.")] - SelfEncryption(#[from] crate::self_encryption::Error), - #[error("Could not get store quote for: {0:?} after several retries")] - CouldNotGetStoreQuote(XorName), - #[error("Could not get store costs: {0:?}")] - CouldNotGetStoreCosts(NetworkError), - #[error("Failed to serialize {0}")] - Serialization(String), -} +use super::*; impl Client { /// Fetch a blob of data from the network - pub async fn data_get(&self, addr: DataAddr) -> Result { + pub async fn data_get_public(&self, addr: DataAddr) -> Result { info!("Fetching data from Data Address: {addr:?}"); let data_map_chunk = self.chunk_get(addr).await?; let data = self @@ -141,7 +42,7 @@ impl Client { /// Upload a piece of data to the network. /// Returns the Data Address at which the data was stored. /// This data is publicly accessible. - pub async fn data_put( + pub async fn data_put_public( &self, data: Bytes, payment_option: PaymentOption, diff --git a/autonomi/src/client/data_private.rs b/autonomi/src/client/data_private.rs deleted file mode 100644 index d31a13f437..0000000000 --- a/autonomi/src/client/data_private.rs +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use std::hash::{DefaultHasher, Hash, Hasher}; - -use ant_evm::Amount; -use ant_protocol::storage::Chunk; -use bytes::Bytes; -use serde::{Deserialize, Serialize}; - -use super::data::{GetError, PutError}; -use crate::client::payment::PaymentOption; -use crate::client::{ClientEvent, UploadSummary}; -use crate::{self_encryption::encrypt, Client}; - -/// Private data on the network can be accessed with this -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct PrivateDataAccess(Chunk); - -impl PrivateDataAccess { - pub fn to_hex(&self) -> String { - hex::encode(self.0.value()) - } - - pub fn from_hex(hex: &str) -> Result { - let data = hex::decode(hex)?; - Ok(Self(Chunk::new(Bytes::from(data)))) - } - - /// Get a private address for [`PrivateDataAccess`]. Note that this is not a network address, it is only used for refering to private data client side. - pub fn address(&self) -> String { - hash_to_short_string(&self.to_hex()) - } -} - -fn hash_to_short_string(input: &str) -> String { - let mut hasher = DefaultHasher::new(); - input.hash(&mut hasher); - let hash_value = hasher.finish(); - hash_value.to_string() -} - -impl Client { - /// Fetch a blob of private data from the network - pub async fn private_data_get(&self, data_map: PrivateDataAccess) -> Result { - info!( - "Fetching private data from Data Map {:?}", - data_map.0.address() - ); - let data = self.fetch_from_data_map_chunk(data_map.0.value()).await?; - - debug!("Successfully fetched a blob of private data from the network"); - Ok(data) - } - - /// Upload a piece of private data to the network. This data will be self-encrypted. - /// Returns the [`PrivateDataAccess`] containing the map to the encrypted chunks. - /// This data is private and only accessible with the [`PrivateDataAccess`]. - pub async fn private_data_put( - &self, - data: Bytes, - payment_option: PaymentOption, - ) -> Result { - let now = ant_networking::target_arch::Instant::now(); - let (data_map_chunk, chunks) = encrypt(data)?; - debug!("Encryption took: {:.2?}", now.elapsed()); - - // Pay for all chunks - let xor_names: Vec<_> = chunks.iter().map(|chunk| *chunk.name()).collect(); - info!("Paying for {} addresses", xor_names.len()); - let receipt = self - .pay_for_content_addrs(xor_names.into_iter(), payment_option) - .await - .inspect_err(|err| error!("Error paying for data: {err:?}"))?; - - // Upload the chunks with the payments - debug!("Uploading {} chunks", chunks.len()); - - let mut failed_uploads = self - .upload_chunks_with_retries(chunks.iter().collect(), &receipt) - .await; - - // Return the last chunk upload error - if let Some(last_chunk_fail) = failed_uploads.pop() { - tracing::error!( - "Error uploading chunk ({:?}): {:?}", - last_chunk_fail.0.address(), - last_chunk_fail.1 - ); - return Err(last_chunk_fail.1); - } - - let record_count = chunks.len(); - - // Reporting - if let Some(channel) = self.client_event_sender.as_ref() { - let tokens_spent = receipt - .values() - .map(|proof| proof.quote.cost.as_atto()) - .sum::(); - - let summary = UploadSummary { - record_count, - tokens_spent, - }; - if let Err(err) = channel.send(ClientEvent::UploadComplete(summary)).await { - error!("Failed to send client event: {err:?}"); - } - } - - Ok(PrivateDataAccess(data_map_chunk)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_hex() { - let data_map = PrivateDataAccess(Chunk::new(Bytes::from_static(b"hello"))); - let hex = data_map.to_hex(); - let data_map2 = PrivateDataAccess::from_hex(&hex).expect("Failed to decode hex"); - assert_eq!(data_map, data_map2); - } -} diff --git a/autonomi/src/client/archive_private.rs b/autonomi/src/client/files/archive.rs similarity index 60% rename from autonomi/src/client/archive_private.rs rename to autonomi/src/client/files/archive.rs index ee8705be2a..58f0788059 100644 --- a/autonomi/src/client/archive_private.rs +++ b/autonomi/src/client/files/archive.rs @@ -13,25 +13,62 @@ use std::{ use ant_networking::target_arch::{Duration, SystemTime, UNIX_EPOCH}; -use super::{ - archive::{Metadata, RenameError}, - data::{GetError, PutError}, - data_private::PrivateDataAccess, +use crate::{ + client::{ + data::{DataMapChunk, GetError, PutError}, + payment::PaymentOption, + }, Client, }; -use crate::client::payment::PaymentOption; use bytes::Bytes; use serde::{Deserialize, Serialize}; +use thiserror::Error; -/// The address of a private archive -/// Contains the [`PrivateDataAccess`] leading to the [`PrivateArchive`] data -pub type PrivateArchiveAccess = PrivateDataAccess; +/// Private archive data map, allowing access to the [`PrivateArchive`] data. +pub type PrivateArchiveAccess = DataMapChunk; -/// A private archive of files that containing file paths, their metadata and the files data maps -/// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address. +#[derive(Error, Debug, PartialEq, Eq)] +pub enum RenameError { + #[error("File not found in archive: {0}")] + FileNotFound(PathBuf), +} + +/// Metadata for a file in an archive. Time values are UNIX timestamps. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Metadata { + /// When the file was (last) uploaded to the network. + pub uploaded: u64, + /// File creation time on local file system. See [`std::fs::Metadata::created`] for details per OS. + pub created: u64, + /// Last file modification time taken from local file system. See [`std::fs::Metadata::modified`] for details per OS. + pub modified: u64, + /// File size in bytes + pub size: u64, +} + +impl Metadata { + /// Create a new metadata struct with the current time as uploaded, created and modified. + pub fn new_with_size(size: u64) -> Self { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or(Duration::from_secs(0)) + .as_secs(); + + Self { + uploaded: now, + created: now, + modified: now, + size, + } + } +} + +/// Directory structure mapping filepaths to their data maps and metadata. +/// +/// The data maps are stored within this structure instead of uploading them to the network, keeping the data private. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] pub struct PrivateArchive { - map: HashMap, + map: HashMap, } impl PrivateArchive { @@ -62,7 +99,7 @@ impl PrivateArchive { /// Add a file to a local archive /// Note that this does not upload the archive to the network - pub fn add_file(&mut self, path: PathBuf, data_map: PrivateDataAccess, meta: Metadata) { + pub fn add_file(&mut self, path: PathBuf, data_map: DataMapChunk, meta: Metadata) { self.map.insert(path.clone(), (data_map, meta)); debug!("Added a new file to the archive, path: {:?}", path); } @@ -76,23 +113,24 @@ impl PrivateArchive { } /// List all data addresses of the files in the archive - pub fn addresses(&self) -> Vec { + pub fn addresses(&self) -> Vec { self.map .values() .map(|(data_map, _)| data_map.clone()) .collect() } - /// Iterate over the archive items - /// Returns an iterator over (PathBuf, SecretDataMap, Metadata) - pub fn iter(&self) -> impl Iterator { + /// Iterate over the archive items. + /// + /// Returns an iterator over ([`PathBuf`], [`DataMapChunk`], [`Metadata`]) + pub fn iter(&self) -> impl Iterator { self.map .iter() .map(|(path, (data_map, meta))| (path, data_map, meta)) } /// Get the underlying map - pub fn map(&self) -> &HashMap { + pub fn map(&self) -> &HashMap { &self.map } @@ -113,17 +151,17 @@ impl PrivateArchive { } impl Client { - /// Fetch a private archive from the network - pub async fn private_archive_get( + /// Fetch a [`PrivateArchive`] from the network + pub async fn archive_get( &self, addr: PrivateArchiveAccess, ) -> Result { - let data = self.private_data_get(addr).await?; + let data = self.data_get(addr).await?; Ok(PrivateArchive::from_bytes(data)?) } - /// Upload a private archive to the network - pub async fn private_archive_put( + /// Upload a [`PrivateArchive`] to the network + pub async fn archive_put( &self, archive: PrivateArchive, payment_option: PaymentOption, @@ -131,7 +169,7 @@ impl Client { let bytes = archive .into_bytes() .map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?; - let result = self.private_data_put(bytes, payment_option).await; + let result = self.data_put(bytes, payment_option).await; debug!("Uploaded private archive {archive:?} to the network and address is {result:?}"); result } diff --git a/autonomi/src/client/archive.rs b/autonomi/src/client/files/archive_public.rs similarity index 67% rename from autonomi/src/client/archive.rs rename to autonomi/src/client/files/archive_public.rs index bed341c450..108d220553 100644 --- a/autonomi/src/client/archive.rs +++ b/autonomi/src/client/files/archive_public.rs @@ -13,65 +13,31 @@ use std::{ use ant_networking::target_arch::{Duration, SystemTime, UNIX_EPOCH}; -use super::{ - data::{CostError, DataAddr, GetError, PutError}, - Client, -}; use ant_evm::{AttoTokens, EvmWallet}; use bytes::Bytes; use serde::{Deserialize, Serialize}; use xor_name::XorName; -/// The address of an archive on the network. Points to an [`Archive`]. -pub type ArchiveAddr = XorName; - -use thiserror::Error; +use super::archive::Metadata; +use crate::{ + client::{ + data::{CostError, DataAddr, GetError, PutError}, + files::archive::RenameError, + }, + Client, +}; -#[derive(Error, Debug, PartialEq, Eq)] -pub enum RenameError { - #[error("File not found in archive: {0}")] - FileNotFound(PathBuf), -} +/// The address of a public archive on the network. Points to an [`PublicArchive`]. +pub type ArchiveAddr = XorName; -/// An archive of files that containing file paths, their metadata and the files data addresses -/// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address. -/// Archives are public meaning anyone can read the data in the archive. For private archives use [`crate::client::archive_private::PrivateArchive`]. +/// Public variant of [`crate::client::files::archive::PrivateArchive`]. Differs in that data maps of files are uploaded +/// to the network, of which the addresses are stored in this archive. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] -pub struct Archive { +pub struct PublicArchive { map: HashMap, } -/// Metadata for a file in an archive. Time values are UNIX timestamps. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct Metadata { - /// When the file was (last) uploaded to the network. - pub uploaded: u64, - /// File creation time on local file system. See [`std::fs::Metadata::created`] for details per OS. - pub created: u64, - /// Last file modification time taken from local file system. See [`std::fs::Metadata::modified`] for details per OS. - pub modified: u64, - /// File size in bytes - pub size: u64, -} - -impl Metadata { - /// Create a new metadata struct with the current time as uploaded, created and modified. - pub fn new_with_size(size: u64) -> Self { - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or(Duration::from_secs(0)) - .as_secs(); - - Self { - uploaded: now, - created: now, - modified: now, - size, - } - } -} - -impl Archive { +impl PublicArchive { /// Create a new emtpy local archive /// Note that this does not upload the archive to the network pub fn new() -> Self { @@ -131,8 +97,8 @@ impl Archive { } /// Deserialize from bytes. - pub fn from_bytes(data: Bytes) -> Result { - let root: Archive = rmp_serde::from_slice(&data[..])?; + pub fn from_bytes(data: Bytes) -> Result { + let root: PublicArchive = rmp_serde::from_slice(&data[..])?; Ok(root) } @@ -157,13 +123,13 @@ impl Client { /// # async fn main() -> Result<(), Box> { /// # let peers = ["/ip4/127.0.0.1/udp/1234/quic-v1".parse()?]; /// let client = Client::connect(&peers).await?; - /// let archive = client.archive_get(ArchiveAddr::random(&mut rand::thread_rng())).await?; + /// let archive = client.archive_get_public(ArchiveAddr::random(&mut rand::thread_rng())).await?; /// # Ok(()) /// # } /// ``` - pub async fn archive_get(&self, addr: ArchiveAddr) -> Result { - let data = self.data_get(addr).await?; - Ok(Archive::from_bytes(data)?) + pub async fn archive_get_public(&self, addr: ArchiveAddr) -> Result { + let data = self.data_get_public(addr).await?; + Ok(PublicArchive::from_bytes(data)?) } /// Upload an archive to the network @@ -173,34 +139,34 @@ impl Client { /// Create simple archive containing `file.txt` pointing to random XOR name. /// /// ```no_run - /// # use autonomi::client::{Client, data::DataAddr, archive::{Archive, ArchiveAddr, Metadata}}; + /// # use autonomi::client::{Client, data::DataAddr, archive::{PublicArchive, ArchiveAddr, Metadata}}; /// # use std::path::PathBuf; /// # #[tokio::main] /// # async fn main() -> Result<(), Box> { /// # let peers = ["/ip4/127.0.0.1/udp/1234/quic-v1".parse()?]; /// # let client = Client::connect(&peers).await?; /// # let wallet = todo!(); - /// let mut archive = Archive::new(); + /// let mut archive = PublicArchive::new(); /// archive.add_file(PathBuf::from("file.txt"), DataAddr::random(&mut rand::thread_rng()), Metadata::new_with_size(0)); - /// let address = client.archive_put(archive, &wallet).await?; + /// let address = client.archive_put_public(archive, &wallet).await?; /// # Ok(()) /// # } /// ``` - pub async fn archive_put( + pub async fn archive_put_public( &self, - archive: Archive, + archive: PublicArchive, wallet: &EvmWallet, ) -> Result { let bytes = archive .into_bytes() .map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?; - let result = self.data_put(bytes, wallet.into()).await; + let result = self.data_put_public(bytes, wallet.into()).await; debug!("Uploaded archive {archive:?} to the network and the address is {result:?}"); result } /// Get the cost to upload an archive - pub async fn archive_cost(&self, archive: Archive) -> Result { + pub async fn archive_cost(&self, archive: PublicArchive) -> Result { let bytes = archive .into_bytes() .map_err(|e| CostError::Serialization(format!("Failed to serialize archive: {e:?}")))?; diff --git a/autonomi/src/client/fs_private.rs b/autonomi/src/client/files/fs.rs similarity index 52% rename from autonomi/src/client/fs_private.rs rename to autonomi/src/client/files/fs.rs index 654fd4cef3..37df1aa84f 100644 --- a/autonomi/src/client/fs_private.rs +++ b/autonomi/src/client/files/fs.rs @@ -14,26 +14,81 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. +use crate::client::data::{CostError, DataMapChunk, GetError, PutError}; use crate::client::utils::process_tasks_with_max_concurrency; use crate::client::Client; use ant_evm::EvmWallet; use bytes::Bytes; -use std::path::PathBuf; +use std::{path::PathBuf, sync::LazyLock}; -use super::archive_private::{PrivateArchive, PrivateArchiveAccess}; -use super::data_private::PrivateDataAccess; -use super::fs::{DownloadError, UploadError}; +use super::archive::{PrivateArchive, PrivateArchiveAccess}; -use super::fs::FILE_UPLOAD_BATCH_SIZE; +/// Number of files to upload in parallel. +/// +/// Can be overridden by the `FILE_UPLOAD_BATCH_SIZE` environment variable. +pub static FILE_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { + let batch_size = std::env::var("FILE_UPLOAD_BATCH_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or( + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + * 8, + ); + info!("File upload batch size: {}", batch_size); + batch_size +}); + +/// Errors that can occur during the file upload operation. +#[derive(Debug, thiserror::Error)] +pub enum UploadError { + #[error("Failed to recursively traverse directory")] + WalkDir(#[from] walkdir::Error), + #[error("Input/output failure")] + IoError(#[from] std::io::Error), + #[error("Failed to upload file")] + PutError(#[from] PutError), + #[error("Failed to fetch file")] + GetError(#[from] GetError), + #[error("Failed to serialize")] + Serialization(#[from] rmp_serde::encode::Error), + #[error("Failed to deserialize")] + Deserialization(#[from] rmp_serde::decode::Error), +} + +/// Errors that can occur during the download operation. +#[derive(Debug, thiserror::Error)] +pub enum DownloadError { + #[error("Failed to download file")] + GetError(#[from] GetError), + #[error("IO failure")] + IoError(#[from] std::io::Error), +} + +/// Errors that can occur during the file cost calculation. +#[derive(Debug, thiserror::Error)] +pub enum FileCostError { + #[error("Cost error: {0}")] + Cost(#[from] CostError), + #[error("IO failure")] + IoError(#[from] std::io::Error), + #[error("Serialization error")] + Serialization(#[from] rmp_serde::encode::Error), + #[error("Self encryption error")] + SelfEncryption(#[from] crate::self_encryption::Error), + #[error("Walkdir error")] + WalkDir(#[from] walkdir::Error), +} impl Client { /// Download a private file from network to local file system - pub async fn private_file_download( + pub async fn file_download( &self, - data_access: PrivateDataAccess, + data_access: DataMapChunk, to_dest: PathBuf, ) -> Result<(), DownloadError> { - let data = self.private_data_get(data_access).await?; + let data = self.data_get(data_access).await?; if let Some(parent) = to_dest.parent() { tokio::fs::create_dir_all(parent).await?; debug!("Created parent directories for {to_dest:?}"); @@ -44,27 +99,26 @@ impl Client { } /// Download a private directory from network to local file system - pub async fn private_dir_download( + pub async fn dir_download( &self, archive_access: PrivateArchiveAccess, to_dest: PathBuf, ) -> Result<(), DownloadError> { - let archive = self.private_archive_get(archive_access).await?; + let archive = self.archive_get(archive_access).await?; for (path, addr, _meta) in archive.iter() { - self.private_file_download(addr.clone(), to_dest.join(path)) - .await?; + self.file_download(addr.clone(), to_dest.join(path)).await?; } debug!("Downloaded directory to {to_dest:?}"); Ok(()) } - /// Upload a private directory to the network. The directory is recursively walked. - /// Reads all files, splits into chunks, uploads chunks, uploads private archive, returns [`PrivateArchiveAccess`] (pointing to the private archive) - pub async fn private_dir_upload( + /// Upload a directory to the network. The directory is recursively walked and each file is uploaded to the network. + /// The data maps of these (private) files are not uploaded but returned within the [`PrivateArchive`] return type. + pub async fn dir_upload( &self, dir_path: PathBuf, wallet: &EvmWallet, - ) -> Result { + ) -> Result { info!("Uploading directory as private: {dir_path:?}"); let start = tokio::time::Instant::now(); @@ -76,10 +130,10 @@ impl Client { continue; } - let metadata = super::fs::metadata_from_entry(&entry); + let metadata = super::fs_public::metadata_from_entry(&entry); let path = entry.path().to_path_buf(); upload_tasks.push(async move { - let file = self.private_file_upload(path.clone(), wallet).await; + let file = self.file_upload(path.clone(), wallet).await; (path, metadata, file) }); } @@ -103,35 +157,38 @@ impl Client { } } - // upload archive - let archive_serialized = archive.into_bytes()?; - let arch_addr = self - .private_data_put(archive_serialized, wallet.into()) - .await?; - - info!( - "Complete private archive upload completed in {:?}", - start.elapsed() - ); #[cfg(feature = "loud")] println!("Upload completed in {:?}", start.elapsed()); - Ok(arch_addr) + Ok(archive) + } + + /// Same as [`Client::dir_upload`] but also uploads the archive (privately) to the network. + /// + /// Returns the [`PrivateArchiveAccess`] allowing the private archive to be downloaded from the network. + pub async fn dir_and_archive_upload( + &self, + dir_path: PathBuf, + wallet: &EvmWallet, + ) -> Result { + let archive = self.dir_upload(dir_path, wallet).await?; + let archive_addr = self.archive_put(archive, wallet.into()).await?; + Ok(archive_addr) } /// Upload a private file to the network. - /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns [`PrivateDataAccess`] (pointing to the datamap) - async fn private_file_upload( + /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns [`DataMapChunk`] (pointing to the datamap) + async fn file_upload( &self, path: PathBuf, wallet: &EvmWallet, - ) -> Result { + ) -> Result { info!("Uploading file: {path:?}"); #[cfg(feature = "loud")] println!("Uploading file: {path:?}"); let data = tokio::fs::read(path).await?; let data = Bytes::from(data); - let addr = self.private_data_put(data, wallet.into()).await?; + let addr = self.data_put(data, wallet.into()).await?; debug!("Uploaded file successfully in the privateAchive: {addr:?}"); Ok(addr) } diff --git a/autonomi/src/client/fs.rs b/autonomi/src/client/files/fs_public.rs similarity index 69% rename from autonomi/src/client/fs.rs rename to autonomi/src/client/files/fs_public.rs index 3eaf49b212..fd9cad51ba 100644 --- a/autonomi/src/client/fs.rs +++ b/autonomi/src/client/files/fs_public.rs @@ -6,84 +6,26 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::client::archive::Metadata; -use crate::client::data::CostError; +use crate::client::data::DataAddr; +use crate::client::files::archive::Metadata; use crate::client::utils::process_tasks_with_max_concurrency; use crate::client::Client; use ant_evm::EvmWallet; use ant_networking::target_arch::{Duration, SystemTime}; use bytes::Bytes; use std::path::PathBuf; -use std::sync::LazyLock; -use super::archive::{Archive, ArchiveAddr}; -use super::data::{DataAddr, GetError, PutError}; - -/// Number of files to upload in parallel. -/// Can be overridden by the `FILE_UPLOAD_BATCH_SIZE` environment variable. -pub static FILE_UPLOAD_BATCH_SIZE: LazyLock = LazyLock::new(|| { - let batch_size = std::env::var("FILE_UPLOAD_BATCH_SIZE") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or( - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1) - * 8, - ); - info!("File upload batch size: {}", batch_size); - batch_size -}); - -/// Errors that can occur during the file upload operation. -#[derive(Debug, thiserror::Error)] -pub enum UploadError { - #[error("Failed to recursively traverse directory")] - WalkDir(#[from] walkdir::Error), - #[error("Input/output failure")] - IoError(#[from] std::io::Error), - #[error("Failed to upload file")] - PutError(#[from] PutError), - #[error("Failed to fetch file")] - GetError(#[from] GetError), - #[error("Failed to serialize")] - Serialization(#[from] rmp_serde::encode::Error), - #[error("Failed to deserialize")] - Deserialization(#[from] rmp_serde::decode::Error), -} - -/// Errors that can occur during the download operation. -#[derive(Debug, thiserror::Error)] -pub enum DownloadError { - #[error("Failed to download file")] - GetError(#[from] GetError), - #[error("IO failure")] - IoError(#[from] std::io::Error), -} - -/// Errors that can occur during the file cost calculation. -#[derive(Debug, thiserror::Error)] -pub enum FileCostError { - #[error("Cost error: {0}")] - Cost(#[from] CostError), - #[error("IO failure")] - IoError(#[from] std::io::Error), - #[error("Serialization error")] - Serialization(#[from] rmp_serde::encode::Error), - #[error("Self encryption error")] - SelfEncryption(#[from] crate::self_encryption::Error), - #[error("Walkdir error")] - WalkDir(#[from] walkdir::Error), -} +use super::archive_public::{ArchiveAddr, PublicArchive}; +use super::fs::*; impl Client { /// Download file from network to local file system - pub async fn file_download( + pub async fn file_download_public( &self, data_addr: DataAddr, to_dest: PathBuf, ) -> Result<(), DownloadError> { - let data = self.data_get(data_addr).await?; + let data = self.data_get_public(data_addr).await?; if let Some(parent) = to_dest.parent() { tokio::fs::create_dir_all(parent).await?; debug!("Created parent directories {parent:?} for {to_dest:?}"); @@ -94,15 +36,15 @@ impl Client { } /// Download directory from network to local file system - pub async fn dir_download( + pub async fn dir_download_public( &self, archive_addr: ArchiveAddr, to_dest: PathBuf, ) -> Result<(), DownloadError> { - let archive = self.archive_get(archive_addr).await?; + let archive = self.archive_get_public(archive_addr).await?; debug!("Downloaded archive for the directory from the network at {archive_addr:?}"); for (path, addr, _meta) in archive.iter() { - self.file_download(*addr, to_dest.join(path)).await?; + self.file_download_public(*addr, to_dest.join(path)).await?; } debug!( "All files in the directory downloaded to {:?} from the network address {:?}", @@ -112,13 +54,16 @@ impl Client { Ok(()) } - /// Upload a directory to the network. The directory is recursively walked. - /// Reads all files, splits into chunks, uploads chunks, uploads datamaps, uploads archive, returns ArchiveAddr (pointing to the archive) - pub async fn dir_upload( + /// Upload a directory to the network. The directory is recursively walked and each file is uploaded to the network. + /// + /// The data maps of these files are uploaded on the network, making the individual files publicly available. + /// + /// This returns, but does not upload (!),the [`PublicArchive`] containing the data maps of the uploaded files. + pub async fn dir_upload_public( &self, dir_path: PathBuf, wallet: &EvmWallet, - ) -> Result { + ) -> Result { info!("Uploading directory: {dir_path:?}"); let start = tokio::time::Instant::now(); @@ -133,7 +78,7 @@ impl Client { let metadata = metadata_from_entry(&entry); let path = entry.path().to_path_buf(); upload_tasks.push(async move { - let file = self.file_upload(path.clone(), wallet).await; + let file = self.file_upload_public(path.clone(), wallet).await; (path, metadata, file) }); } @@ -146,7 +91,7 @@ impl Client { uploads.len(), start.elapsed() ); - let mut archive = Archive::new(); + let mut archive = PublicArchive::new(); for (path, metadata, maybe_file) in uploads.into_iter() { match maybe_file { Ok(file) => archive.add_file(path, file, metadata), @@ -157,20 +102,27 @@ impl Client { } } - // upload archive - let archive_serialized = archive.into_bytes()?; - let arch_addr = self.data_put(archive_serialized, wallet.into()).await?; - - info!("Complete archive upload completed in {:?}", start.elapsed()); #[cfg(feature = "loud")] println!("Upload completed in {:?}", start.elapsed()); - debug!("Directory uploaded to the network at {arch_addr:?}"); - Ok(arch_addr) + Ok(archive) + } + + /// Same as [`Client::dir_upload_public`] but also uploads the archive to the network. + /// + /// Returns the [`ArchiveAddr`] of the uploaded archive. + pub async fn dir_and_archive_upload_public( + &self, + dir_path: PathBuf, + wallet: &EvmWallet, + ) -> Result { + let archive = self.dir_upload_public(dir_path, wallet).await?; + let archive_addr = self.archive_put_public(archive, wallet).await?; + Ok(archive_addr) } /// Upload a file to the network. /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns DataAddr (pointing to the datamap) - async fn file_upload( + async fn file_upload_public( &self, path: PathBuf, wallet: &EvmWallet, @@ -181,7 +133,7 @@ impl Client { let data = tokio::fs::read(path.clone()).await?; let data = Bytes::from(data); - let addr = self.data_put(data, wallet.into()).await?; + let addr = self.data_put_public(data, wallet.into()).await?; debug!("File {path:?} uploaded to the network at {addr:?}"); Ok(addr) } @@ -189,7 +141,7 @@ impl Client { /// Get the cost to upload a file/dir to the network. /// quick and dirty implementation, please refactor once files are cleanly implemented pub async fn file_cost(&self, path: &PathBuf) -> Result { - let mut archive = Archive::new(); + let mut archive = PublicArchive::new(); let mut total_cost = ant_evm::Amount::ZERO; for entry in walkdir::WalkDir::new(path) { diff --git a/autonomi/src/client/files/mod.rs b/autonomi/src/client/files/mod.rs new file mode 100644 index 0000000000..981c1d472c --- /dev/null +++ b/autonomi/src/client/files/mod.rs @@ -0,0 +1,8 @@ +pub mod archive; +pub mod archive_public; +#[cfg(feature = "fs")] +#[cfg_attr(docsrs, doc(cfg(feature = "fs")))] +pub mod fs; +#[cfg(feature = "fs")] +#[cfg_attr(docsrs, doc(cfg(feature = "fs")))] +pub mod fs_public; diff --git a/autonomi/src/client/mod.rs b/autonomi/src/client/mod.rs index 05003d1b19..914b01478b 100644 --- a/autonomi/src/client/mod.rs +++ b/autonomi/src/client/mod.rs @@ -12,19 +12,11 @@ pub mod address; pub mod payment; -pub mod archive; -pub mod archive_private; pub mod data; -pub mod data_private; #[cfg(feature = "external-signer")] #[cfg_attr(docsrs, doc(cfg(feature = "external-signer")))] pub mod external_signer; -#[cfg(feature = "fs")] -#[cfg_attr(docsrs, doc(cfg(feature = "fs")))] -pub mod fs; -#[cfg(feature = "fs")] -#[cfg_attr(docsrs, doc(cfg(feature = "fs")))] -pub mod fs_private; +pub mod files; #[cfg(feature = "registers")] #[cfg_attr(docsrs, doc(cfg(feature = "registers")))] pub mod registers; diff --git a/autonomi/src/client/vault/user_data.rs b/autonomi/src/client/vault/user_data.rs index d9bff46f6f..a0b4069534 100644 --- a/autonomi/src/client/vault/user_data.rs +++ b/autonomi/src/client/vault/user_data.rs @@ -8,10 +8,10 @@ use std::collections::HashMap; -use crate::client::archive::ArchiveAddr; -use crate::client::archive_private::PrivateArchiveAccess; use crate::client::data::GetError; use crate::client::data::PutError; +use crate::client::files::archive::PrivateArchiveAccess; +use crate::client::files::archive_public::ArchiveAddr; use crate::client::payment::PaymentOption; use crate::client::registers::RegisterAddress; use crate::client::vault::VaultError; diff --git a/autonomi/src/client/wasm.rs b/autonomi/src/client/wasm.rs index fac5ec6343..0f9a2ea802 100644 --- a/autonomi/src/client/wasm.rs +++ b/autonomi/src/client/wasm.rs @@ -1,7 +1,7 @@ use super::address::{addr_to_str, str_to_addr}; #[cfg(feature = "vault")] use super::vault::UserData; -use crate::client::data_private::PrivateDataAccess; +use crate::client::data::DataMapChunk; use crate::client::payment::Receipt; use ant_protocol::storage::Chunk; use libp2p::Multiaddr; @@ -100,14 +100,14 @@ impl JsClient { #[wasm_bindgen(js_name = putData)] pub async fn put_data(&self, data: Vec, wallet: &JsWallet) -> Result { let data = crate::Bytes::from(data); - let xorname = self.0.data_put(data, (&wallet.0).into()).await?; + let xorname = self.0.data_put_public(data, (&wallet.0).into()).await?; Ok(addr_to_str(xorname)) } /// Upload private data to the network. /// - /// Returns the `PrivateDataAccess` chunk of the data. + /// Returns the `DataMapChunk` chunk of the data. #[wasm_bindgen(js_name = putPrivateData)] pub async fn put_private_data( &self, @@ -115,7 +115,7 @@ impl JsClient { wallet: &JsWallet, ) -> Result { let data = crate::Bytes::from(data); - let private_data_access = self.0.private_data_put(data, (&wallet.0).into()).await?; + let private_data_access = self.0.data_put(data, (&wallet.0).into()).await?; let js_value = serde_wasm_bindgen::to_value(&private_data_access)?; Ok(js_value) @@ -124,7 +124,7 @@ impl JsClient { /// Upload private data to the network. /// Uses a `Receipt` as payment. /// - /// Returns the `PrivateDataAccess` chunk of the data. + /// Returns the `DataMapChunk` chunk of the data. #[wasm_bindgen(js_name = putPrivateDataWithReceipt)] pub async fn put_private_data_with_receipt( &self, @@ -133,7 +133,7 @@ impl JsClient { ) -> Result { let data = crate::Bytes::from(data); let receipt: Receipt = serde_wasm_bindgen::from_value(receipt)?; - let private_data_access = self.0.private_data_put(data, receipt.into()).await?; + let private_data_access = self.0.data_put(data, receipt.into()).await?; let js_value = serde_wasm_bindgen::to_value(&private_data_access)?; Ok(js_value) @@ -143,7 +143,7 @@ impl JsClient { #[wasm_bindgen(js_name = getData)] pub async fn get_data(&self, addr: String) -> Result, JsError> { let addr = str_to_addr(&addr)?; - let data = self.0.data_get(addr).await?; + let data = self.0.data_get_public(addr).await?; Ok(data.to_vec()) } @@ -151,9 +151,9 @@ impl JsClient { /// Fetch the data from the network. #[wasm_bindgen(js_name = getPrivateData)] pub async fn get_private_data(&self, private_data_access: JsValue) -> Result, JsError> { - let private_data_access: PrivateDataAccess = + let private_data_access: DataMapChunk = serde_wasm_bindgen::from_value(private_data_access)?; - let data = self.0.private_data_get(private_data_access).await?; + let data = self.0.data_get(private_data_access).await?; Ok(data.to_vec()) } @@ -171,15 +171,14 @@ impl JsClient { mod archive { use super::*; use crate::client::{ - address::str_to_addr, - archive::{Archive, Metadata}, + address::str_to_addr, files::archive::Metadata, files::archive_public::PublicArchive, }; use std::path::PathBuf; use wasm_bindgen::JsError; /// Structure mapping paths to data addresses. #[wasm_bindgen(js_name = Archive)] - pub struct JsArchive(Archive); + pub struct JsArchive(PublicArchive); /// Create new metadata with the current time as uploaded, created and modified. /// @@ -201,7 +200,7 @@ mod archive { /// Create a new archive. #[wasm_bindgen(constructor)] pub fn new() -> Self { - Self(Archive::new()) + Self(PublicArchive::new()) } /// Add a new file to the archive. @@ -249,7 +248,7 @@ mod archive { #[wasm_bindgen(js_name = getArchive)] pub async fn get_archive(&self, addr: String) -> Result { let addr = str_to_addr(&addr)?; - let archive = self.0.archive_get(addr).await?; + let archive = self.0.archive_get_public(addr).await?; let archive = JsArchive(archive); Ok(archive) @@ -264,7 +263,10 @@ mod archive { archive: &JsArchive, wallet: &JsWallet, ) -> Result { - let addr = self.0.archive_put(archive.0.clone(), &wallet.0).await?; + let addr = self + .0 + .archive_put_public(archive.0.clone(), &wallet.0) + .await?; Ok(addr_to_str(addr)) } @@ -273,9 +275,8 @@ mod archive { mod archive_private { use super::*; - use crate::client::archive::Metadata; - use crate::client::archive_private::{PrivateArchive, PrivateArchiveAccess}; - use crate::client::data_private::PrivateDataAccess; + use crate::client::data::DataMapChunk; + use crate::client::files::archive::{Metadata, PrivateArchive, PrivateArchiveAccess}; use crate::client::payment::Receipt; use std::path::PathBuf; use wasm_bindgen::{JsError, JsValue}; @@ -301,7 +302,7 @@ mod archive_private { metadata: JsValue, ) -> Result<(), JsError> { let path = PathBuf::from(path); - let data_map: PrivateDataAccess = serde_wasm_bindgen::from_value(data_map)?; + let data_map: DataMapChunk = serde_wasm_bindgen::from_value(data_map)?; let metadata: Metadata = serde_wasm_bindgen::from_value(metadata)?; self.0.add_file(path, data_map, metadata); @@ -332,7 +333,7 @@ mod archive_private { ) -> Result { let private_archive_access: PrivateArchiveAccess = serde_wasm_bindgen::from_value(private_archive_access)?; - let archive = self.0.private_archive_get(private_archive_access).await?; + let archive = self.0.archive_get(private_archive_access).await?; let archive = JsPrivateArchive(archive); Ok(archive) @@ -349,7 +350,7 @@ mod archive_private { ) -> Result { let private_archive_access = self .0 - .private_archive_put(archive.0.clone(), (&wallet.0).into()) + .archive_put(archive.0.clone(), (&wallet.0).into()) .await?; let js_value = serde_wasm_bindgen::to_value(&private_archive_access)?; @@ -371,7 +372,7 @@ mod archive_private { let private_archive_access = self .0 - .private_archive_put(archive.0.clone(), receipt.into()) + .archive_put(archive.0.clone(), receipt.into()) .await?; let js_value = serde_wasm_bindgen::to_value(&private_archive_access)?; @@ -385,7 +386,7 @@ mod archive_private { mod vault { use super::*; use crate::client::address::addr_to_str; - use crate::client::archive_private::PrivateArchiveAccess; + use crate::client::files::archive::PrivateArchiveAccess; use crate::client::payment::Receipt; use crate::client::vault::key::blst_to_blsttc; use crate::client::vault::key::derive_secret_key_from_seed; @@ -691,7 +692,7 @@ mod external_signer { ) -> Result { let data = crate::Bytes::from(data); let receipt: Receipt = serde_wasm_bindgen::from_value(receipt)?; - let xorname = self.0.data_put(data, receipt.into()).await?; + let xorname = self.0.data_put_public(data, receipt.into()).await?; Ok(addr_to_str(xorname)) } } diff --git a/autonomi/src/lib.rs b/autonomi/src/lib.rs index 4f219ea116..7f200df9cc 100644 --- a/autonomi/src/lib.rs +++ b/autonomi/src/lib.rs @@ -22,12 +22,12 @@ //! let wallet = Wallet::new_from_private_key(Default::default(), key)?; //! //! // Put and fetch data. -//! let data_addr = client.data_put(Bytes::from("Hello, World"), (&wallet).into()).await?; -//! let _data_fetched = client.data_get(data_addr).await?; +//! let data_addr = client.data_put_public(Bytes::from("Hello, World"), (&wallet).into()).await?; +//! let _data_fetched = client.data_get_public(data_addr).await?; //! //! // Put and fetch directory from local file system. -//! let dir_addr = client.dir_upload("files/to/upload".into(), &wallet).await?; -//! client.dir_download(dir_addr, "files/downloaded".into()).await?; +//! let dir_addr = client.dir_upload_public("files/to/upload".into(), &wallet).await?; +//! client.dir_download_public(dir_addr, "files/downloaded".into()).await?; //! //! Ok(()) //! } @@ -79,7 +79,7 @@ pub use bytes::Bytes; pub use libp2p::Multiaddr; #[doc(inline)] -pub use client::Client; +pub use client::{files::archive::PrivateArchive, Client}; #[cfg(feature = "extension-module")] mod python; diff --git a/autonomi/src/python.rs b/autonomi/src/python.rs index 2106327347..0c28401b55 100644 --- a/autonomi/src/python.rs +++ b/autonomi/src/python.rs @@ -2,9 +2,8 @@ #![allow(non_local_definitions)] use crate::client::{ - archive::ArchiveAddr, - archive_private::PrivateArchiveAccess, - data_private::PrivateDataAccess, + data::DataMapChunk, + files::{archive::PrivateArchiveAccess, archive_public::ArchiveAddr}, payment::PaymentOption as RustPaymentOption, vault::{UserData, VaultSecretKey}, Client as RustClient, @@ -39,40 +38,36 @@ impl PyClient { Ok(Self { inner: client }) } - fn private_data_put( - &self, - data: Vec, - payment: &PyPaymentOption, - ) -> PyResult { + fn data_put(&self, data: Vec, payment: &PyPaymentOption) -> PyResult { let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); let access = rt .block_on( self.inner - .private_data_put(Bytes::from(data), payment.inner.clone()), + .data_put(Bytes::from(data), payment.inner.clone()), ) .map_err(|e| { pyo3::exceptions::PyValueError::new_err(format!("Failed to put private data: {e}")) })?; - Ok(PyPrivateDataAccess { inner: access }) + Ok(PyDataMapChunk { inner: access }) } - fn private_data_get(&self, access: &PyPrivateDataAccess) -> PyResult> { + fn data_get(&self, access: &PyDataMapChunk) -> PyResult> { let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); let data = rt - .block_on(self.inner.private_data_get(access.inner.clone())) + .block_on(self.inner.data_get(access.inner.clone())) .map_err(|e| { pyo3::exceptions::PyValueError::new_err(format!("Failed to get private data: {e}")) })?; Ok(data.to_vec()) } - fn data_put(&self, data: Vec, payment: &PyPaymentOption) -> PyResult { + fn data_put_public(&self, data: Vec, payment: &PyPaymentOption) -> PyResult { let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); let addr = rt .block_on( self.inner - .data_put(bytes::Bytes::from(data), payment.inner.clone()), + .data_put_public(bytes::Bytes::from(data), payment.inner.clone()), ) .map_err(|e| { pyo3::exceptions::PyValueError::new_err(format!("Failed to put data: {e}")) @@ -81,13 +76,13 @@ impl PyClient { Ok(crate::client::address::addr_to_str(addr)) } - fn data_get(&self, addr: &str) -> PyResult> { + fn data_get_public(&self, addr: &str) -> PyResult> { let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); let addr = crate::client::address::str_to_addr(addr).map_err(|e| { pyo3::exceptions::PyValueError::new_err(format!("Invalid address: {e}")) })?; - let data = rt.block_on(self.inner.data_get(addr)).map_err(|e| { + let data = rt.block_on(self.inner.data_get_public(addr)).map_err(|e| { pyo3::exceptions::PyValueError::new_err(format!("Failed to get data: {e}")) })?; @@ -299,17 +294,17 @@ impl PyUserData { } } -#[pyclass(name = "PrivateDataAccess")] +#[pyclass(name = "DataMapChunk")] #[derive(Clone)] -pub(crate) struct PyPrivateDataAccess { - inner: PrivateDataAccess, +pub(crate) struct PyDataMapChunk { + inner: DataMapChunk, } #[pymethods] -impl PyPrivateDataAccess { +impl PyDataMapChunk { #[staticmethod] fn from_hex(hex: &str) -> PyResult { - PrivateDataAccess::from_hex(hex) + DataMapChunk::from_hex(hex) .map(|access| Self { inner: access }) .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid hex: {e}"))) } @@ -347,7 +342,7 @@ fn autonomi_client_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; m.add_function(wrap_pyfunction!(encrypt, m)?)?; Ok(()) } diff --git a/autonomi/tests/external_signer.rs b/autonomi/tests/external_signer.rs index a9755400a4..58722c5d45 100644 --- a/autonomi/tests/external_signer.rs +++ b/autonomi/tests/external_signer.rs @@ -4,9 +4,8 @@ use alloy::network::TransactionBuilder; use alloy::providers::Provider; use ant_evm::{QuoteHash, TxHash}; use ant_logging::LogBuilder; -use autonomi::client::archive::Metadata; -use autonomi::client::archive_private::PrivateArchive; use autonomi::client::external_signer::encrypt_data; +use autonomi::client::files::archive::{Metadata, PrivateArchive}; use autonomi::client::payment::Receipt; use autonomi::client::vault::user_data::USER_DATA_VAULT_CONTENT_IDENTIFIER; use autonomi::client::vault::VaultSecretKey; @@ -111,9 +110,7 @@ async fn external_signer_put() -> eyre::Result<()> { sleep(Duration::from_secs(5)).await; - let private_data_access = client - .private_data_put(data.clone(), receipt.into()) - .await?; + let private_data_access = client.data_put(data.clone(), receipt.into()).await?; let mut private_archive = PrivateArchive::new(); private_archive.add_file( @@ -128,9 +125,7 @@ async fn external_signer_put() -> eyre::Result<()> { sleep(Duration::from_secs(5)).await; - let private_archive_access = client - .private_archive_put(private_archive, receipt.into()) - .await?; + let private_archive_access = client.archive_put(private_archive, receipt.into()).await?; let vault_key = VaultSecretKey::random(); @@ -174,9 +169,7 @@ async fn external_signer_put() -> eyre::Result<()> { .expect("No private archive present in the UserData") .clone(); - let fetched_private_archive = client - .private_archive_get(fetched_private_archive_access) - .await?; + let fetched_private_archive = client.archive_get(fetched_private_archive_access).await?; let (_, (fetched_private_file_access, _)) = fetched_private_archive .map() @@ -184,9 +177,7 @@ async fn external_signer_put() -> eyre::Result<()> { .next() .expect("No file present in private archive"); - let fetched_private_file = client - .private_data_get(fetched_private_file_access.clone()) - .await?; + let fetched_private_file = client.data_get(fetched_private_file_access.clone()).await?; assert_eq!( fetched_private_file, data, diff --git a/autonomi/tests/fs.rs b/autonomi/tests/fs.rs index 274fc447f2..1b8b59f801 100644 --- a/autonomi/tests/fs.rs +++ b/autonomi/tests/fs.rs @@ -30,13 +30,13 @@ async fn dir_upload_download() -> Result<()> { let wallet = get_funded_wallet(); let addr = client - .dir_upload("tests/file/test_dir".into(), &wallet) + .dir_and_archive_upload_public("tests/file/test_dir".into(), &wallet) .await?; sleep(Duration::from_secs(10)).await; client - .dir_download(addr, "tests/file/test_dir_fetched".into()) + .dir_download_public(addr, "tests/file/test_dir_fetched".into()) .await?; // compare the two directories @@ -86,11 +86,11 @@ async fn file_into_vault() -> Result<()> { let client_sk = bls::SecretKey::random(); let addr = client - .dir_upload("tests/file/test_dir".into(), &wallet) + .dir_and_archive_upload_public("tests/file/test_dir".into(), &wallet) .await?; sleep(Duration::from_secs(2)).await; - let archive = client.archive_get(addr).await?; + let archive = client.archive_get_public(addr).await?; let set_version = 0; client .write_bytes_to_vault( @@ -106,7 +106,8 @@ async fn file_into_vault() -> Result<()> { let (ap, got_version) = new_client.fetch_and_decrypt_vault(&client_sk).await?; assert_eq!(set_version, got_version); - let ap_archive_fetched = autonomi::client::archive::Archive::from_bytes(ap)?; + let ap_archive_fetched = + autonomi::client::files::archive_public::PublicArchive::from_bytes(ap)?; assert_eq!( archive, ap_archive_fetched, diff --git a/autonomi/tests/put.rs b/autonomi/tests/put.rs index 401b5d3356..f5d411e691 100644 --- a/autonomi/tests/put.rs +++ b/autonomi/tests/put.rs @@ -21,11 +21,11 @@ async fn put() -> Result<()> { let wallet = get_funded_wallet(); let data = gen_random_data(1024 * 1024 * 10); - let addr = client.data_put(data.clone(), wallet.into()).await?; + let addr = client.data_put_public(data.clone(), wallet.into()).await?; sleep(Duration::from_secs(10)).await; - let data_fetched = client.data_get(addr).await?; + let data_fetched = client.data_get_public(addr).await?; assert_eq!(data, data_fetched, "data fetched should match data put"); Ok(()) diff --git a/autonomi/tests/wasm.rs b/autonomi/tests/wasm.rs index 980682765c..efdc8d179e 100644 --- a/autonomi/tests/wasm.rs +++ b/autonomi/tests/wasm.rs @@ -25,11 +25,11 @@ async fn put() -> Result<(), Box> { let wallet = get_funded_wallet(); let data = gen_random_data(1024 * 1024 * 10); - let addr = client.data_put(data.clone(), wallet.into()).await?; + let addr = client.data_put_public(data.clone(), wallet.into()).await?; sleep(Duration::from_secs(10)).await; - let data_fetched = client.data_get(addr).await?; + let data_fetched = client.data_get_public(addr).await?; assert_eq!(data, data_fetched, "data fetched should match data put"); Ok(())