Merge pull request #87 from xelis-project/dev

v1.13.4

API.md

@@ -3361,12 +3361,18 @@ NOTE: If `as_hex` is `false`, the response result will contains a field named `b
 #### Make Integrated Address
 Create an integrated address using a wallet address and data to include.
 
+NOTE: Integrated data can be useful for services like Exchanges to identify a user transaction
+by integrating an ID (or anything else) in the address (like PaymentID for Monero).
+
+It is not mandatory and support any data formatted in JSON up to 1 KB in serialized format.
+
 ##### Method `make_integrated_address`
 
 ##### Parameters
-|   Name  |   Type  | Required |                                 Note                                 |
-|:-------:|:-------:|:--------:|:--------------------------------------------------------------------:|
-| address | Address | Required | Address to split in two parts: original address, and integrated data |
+|       Name      |   Type  | Required |                       Note                       |
+|:---------------:|:-------:|:--------:|:------------------------------------------------:|
+|     address     | Address | Required | Wallet address to use for the integrated address |
+| integrated_data |   JSON  | Required |           Any data type in JSON format           |
 
 ##### Request
 ```json

CHANGELOG.md

@@ -4,6 +4,22 @@ This  file  contains  all  the  changelogs  to  ensure  that  changes  can  be
 
 To see the full history and exact changes, please refer to the commits history directly.
 
+## v1.13.4
+
+Bug fixes for nodes with less than 2GB of ram, and for nodes having reorg issues.
+
+All:
+- Update CLI description/about text.
+
+Daemon:
+- Fix rewind / `pop_blocks` function to:
+    - not delete TXs included in more than one block.
+    - rewind on sync blocks only
+- Reduce Sled cache size from 1 GB to 16 MB to support low devices
+- use `--log-level` by default if `--file-log-level` is not provided
+- Add option `--internal-cache-size` to config (in bytes) the inner DB cache size
+- Add option `--internal-db-mode` to select the DB mode (`slow-space` or `high-throughput`)
+
 ## v1.13.3
 
 Bug fixes, add new features, new RPC methods for easier integration including documentation update.

xelis_common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "xelis_common"
-version = "1.13.3"
+version = "1.13.4"
 edition = "2021"
 authors = ["Slixe <slixeprivate@gmail.com>"]
 build = "build.rs"

xelis_daemon/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "xelis_daemon"
-version = "1.13.3"
+version = "1.13.4"
 edition = "2021"
 authors = ["Slixe <slixeprivate@gmail.com>"]
 

xelis_daemon/src/core/storage/mod.rs

@@ -2,7 +2,7 @@ mod providers;
 mod sled;
 
 pub use self::{
-    sled::SledStorage,
+    sled::{SledStorage, StorageMode},
     providers::*,
 };
 

xelis_daemon/src/core/storage/sled.rs

@@ -18,8 +18,12 @@ use xelis_common::{
 use std::{
     collections::HashSet,
     hash::Hash as StdHash,
-    sync::{Arc, atomic::{AtomicU64, Ordering}},
-    num::NonZeroUsize
+    num::NonZeroUsize,
+    str::FromStr,
+    sync::{
+        atomic::{AtomicU64, Ordering},
+        Arc
+    }
 };
 use tokio::sync::Mutex;
 use lru::LruCache;
@@ -152,9 +156,47 @@ macro_rules! init_cache {
     }};
 }
 
+#[derive(Clone, Copy, clap::ValueEnum)]
+pub enum StorageMode {
+    HighThroughput,
+    LowSpace
+}
+
+impl FromStr for StorageMode {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(match s {
+            "high_throughput" => Self::HighThroughput,
+            "low_space" => Self::LowSpace,
+            _ => return Err("Invalid storage mode".into())
+        })
+    }
+}
+
+impl Into<sled::Mode> for StorageMode {
+    fn into(self) -> sled::Mode {
+        match self {
+            Self::HighThroughput => sled::Mode::HighThroughput,
+            Self::LowSpace => sled::Mode::LowSpace
+        }
+    }
+}
+
+// Default cache size
+const DEFAULT_DB_CACHE_CAPACITY: u64 = 16 * 1024 * 1024; // 16 MB
+
 impl SledStorage {
-    pub fn new(dir_path: String, cache_size: Option<usize>, network: Network) -> Result<Self, BlockchainError> {
-        let sled = sled::open(format!("{}{}", dir_path, network.to_string().to_lowercase()))?;
+    pub fn new(dir_path: String, cache_size: Option<usize>, network: Network, internal_cache_size: Option<u64>, mode: StorageMode) -> Result<Self, BlockchainError> {
+        let path = format!("{}{}", dir_path, network.to_string().to_lowercase());
+        let config = sled::Config::new()
+            .temporary(false)
+            .path(path)
+            .cache_capacity(internal_cache_size.unwrap_or(DEFAULT_DB_CACHE_CAPACITY))
+            .mode(mode.into());
+
+        let sled = config.open()?;
+
         let mut storage = Self {
             network,
             transactions: sled.open_tree("transactions")?,
@@ -494,10 +536,13 @@ impl Storage for SledStorage {
 
         let mut txs = Vec::new();
         for tx_hash in block.get_transactions() {
+            // Should we delete the tx too or only unlink it
+            let mut should_delete = true;
             if self.has_tx_blocks(tx_hash)? {
                 let mut blocks: Tips = self.delete_cacheable_data(&self.tx_blocks, &None, tx_hash).await?;
                 let blocks_len =  blocks.len();
                 blocks.remove(&hash);
+                should_delete = blocks.is_empty();
                 self.set_blocks_for_tx(tx_hash, &blocks)?;
                 trace!("Tx was included in {}, blocks left: {}", blocks_len, blocks.into_iter().map(|b| b.to_string()).collect::<Vec<String>>().join(", "));
             }
@@ -509,7 +554,7 @@ impl Storage for SledStorage {
 
             // We have to check first as we may have already deleted it because of client protocol
             // which allow multiple time the same txs in differents blocks
-            if self.contains_data(&self.transactions, &self.transactions_cache, tx_hash).await? {
+            if should_delete && self.contains_data(&self.transactions, &self.transactions_cache, tx_hash).await? {
                 trace!("Deleting TX {} in block {}", tx_hash, hash);
                 let tx: Arc<Transaction> = self.delete_data(&self.transactions, &self.transactions_cache, tx_hash).await?;
                 txs.push((tx_hash.clone(), tx));
@@ -797,6 +842,23 @@ impl Storage for SledStorage {
         trace!("Lowest topoheight for rewind: {}", lowest_topo);
 
         let pruned_topoheight = self.get_pruned_topoheight().await?.unwrap_or(0);
+
+        // we must check that we are stopping a sync block
+        // easy way for this: check the block at topo is currently alone at height
+        while lowest_topo > pruned_topoheight {
+            let hash = self.get_hash_at_topo_height(lowest_topo).await?;
+            let block_height = self.get_height_for_block_hash(&hash).await?;
+            let blocks_at_height = self.get_blocks_at_height(block_height).await?;
+            info!("blocks at height: {}", blocks_at_height.len());
+            if blocks_at_height.len() == 1 {
+                info!("Sync block found at topoheight {}", lowest_topo);
+                break;
+            } else {
+                warn!("No sync block found at topoheight {} we must go lower if possible", lowest_topo);
+                lowest_topo -= 1;
+            }
+        }
+
         if pruned_topoheight != 0 {
             let safety_pruned_topoheight = pruned_topoheight + PRUNE_SAFETY_LIMIT;
             if lowest_topo <= safety_pruned_topoheight && stable_topo_height != 0 {
@@ -907,6 +969,7 @@ impl Storage for SledStorage {
 
                 // find the first version which is under topoheight
                 let pkey = PublicKey::from_bytes(&key)?;
+                trace!("Highest topoheight for {} nonce is {}, above {}", pkey.as_address(self.is_mainnet()), highest_topoheight, topoheight);
                 let mut version = self.get_nonce_at_exact_topoheight(&pkey, highest_topoheight).await
                     .context(format!("Error while retrieving nonce at exact topoheight {highest_topoheight}"))?;
 

xelis_daemon/src/main.rs

@@ -70,7 +70,8 @@ use core::{
     hard_fork::{
         get_pow_algorithm_for_version,
         get_version_at_height
-    }
+    },
+    storage::StorageMode
 };
 use std::{
     fs::File,
@@ -86,7 +87,7 @@ use anyhow::{
 };
 
 #[derive(Parser)]
-#[clap(version = VERSION, about = "XELIS: An innovate cryptocurrency with BlockDAG and Homomorphic Encryption enabling Smart Contracts")]
+#[clap(version = VERSION, about = "XELIS is an innovative cryptocurrency built from scratch with BlockDAG, Homomorphic Encryption, Zero-Knowledge Proofs, and Smart Contracts.")]
 #[command(styles = xelis_common::get_cli_styles())]
 pub struct NodeConfig {
     #[structopt(flatten)]
@@ -95,8 +96,9 @@ pub struct NodeConfig {
     #[clap(long, value_enum, default_value_t = LogLevel::Info)]
     log_level: LogLevel,
     /// Set file log level
-    #[clap(long, value_enum, default_value_t = LogLevel::Info)]
-    file_log_level: LogLevel,
+    /// By default, it will be the same as log level
+    #[clap(long, value_enum)]
+    file_log_level: Option<LogLevel>,
     /// Disable the log file
     #[clap(long)]
     disable_file_logging: bool,
@@ -129,7 +131,13 @@ pub struct NodeConfig {
     logs_modules: Vec<ModuleConfig>,
     /// Network selected for chain
     #[clap(long, value_enum, default_value_t = Network::Mainnet)]
-    network: Network
+    network: Network,
+    /// DB cache size in bytes
+    #[clap(long)]
+    internal_cache_size: Option<u64>,
+    /// Internal DB mode to use
+    #[clap(long, value_enum, default_value_t = StorageMode::LowSpace)]
+    internal_db_mode: StorageMode
 }
 
 const BLOCK_TIME: Difficulty = Difficulty::from_u64(BLOCK_TIME_MILLIS / MILLIS_PER_SECOND);
@@ -138,7 +146,7 @@ const BLOCK_TIME: Difficulty = Difficulty::from_u64(BLOCK_TIME_MILLIS / MILLIS_P
 async fn main() -> Result<()> {
     let mut config: NodeConfig = NodeConfig::parse();
 
-    let prompt = Prompt::new(config.log_level, &config.logs_path, &config.filename_log, config.disable_file_logging, config.disable_file_log_date_based, config.disable_log_color, !config.disable_interactive_mode, config.logs_modules, config.file_log_level)?;
+    let prompt = Prompt::new(config.log_level, &config.logs_path, &config.filename_log, config.disable_file_logging, config.disable_file_log_date_based, config.disable_log_color, !config.disable_interactive_mode, config.logs_modules, config.file_log_level.unwrap_or(config.log_level))?;
     info!("XELIS Blockchain running version: {}", VERSION);
     info!("----------------------------------------------");
 
@@ -162,7 +170,7 @@ async fn main() -> Result<()> {
         };
 
         let dir_path = blockchain_config.dir_path.clone().unwrap_or_default();
-        SledStorage::new(dir_path, use_cache, config.network)?
+        SledStorage::new(dir_path, use_cache, config.network, config.internal_cache_size, config.internal_db_mode)?
     };
 
     let blockchain = Blockchain::new(blockchain_config, config.network, storage).await?;

xelis_daemon/src/rpc/rpc.rs

@@ -184,7 +184,7 @@ pub async fn get_block_response_for_hash<S: Storage>(blockchain: &Blockchain<S>,
         // calculate total size in bytes
         let mut total_size_in_bytes = header.size();
         for tx_hash in header.get_txs_hashes() {
-            total_size_in_bytes += storage.get_transaction_size(tx_hash).await.context(format!("Error while retrieving transaction {hash} size"))?;
+            total_size_in_bytes += storage.get_transaction_size(tx_hash).await.context(format!("Error while retrieving transaction {tx_hash} size"))?;
         }
 
         let mainnet = blockchain.get_network().is_mainnet();

xelis_miner/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "xelis_miner"
-version = "1.13.3"
+version = "1.13.4"
 edition = "2021"
 authors = ["Slixe <slixeprivate@gmail.com>"]
 

xelis_miner/src/main.rs

@@ -88,7 +88,7 @@ use anyhow::{
 use lazy_static::lazy_static;
 
 #[derive(Parser)]
-#[clap(version = VERSION, about = "XELIS: An innovate cryptocurrency with BlockDAG and Homomorphic Encryption enabling Smart Contracts")]
+#[clap(version = VERSION, about = "XELIS is an innovative cryptocurrency built from scratch with BlockDAG, Homomorphic Encryption, Zero-Knowledge Proofs, and Smart Contracts.")]
 #[command(styles = xelis_common::get_cli_styles())]
 pub struct MinerConfig {
     /// Wallet address to mine and receive block rewards on
@@ -105,8 +105,9 @@ pub struct MinerConfig {
     #[clap(long, value_enum, default_value_t = LogLevel::Info)]
     log_level: LogLevel,
     /// Set file log level
-    #[clap(long, value_enum, default_value_t = LogLevel::Info)]
-    file_log_level: LogLevel,
+    /// By default, it will be the same as log level
+    #[clap(long, value_enum)]
+    file_log_level: Option<LogLevel>,
     /// Enable the benchmark mode with the specified algorithm
     #[clap(long)]
     benchmark: Option<Algorithm>,
@@ -187,7 +188,7 @@ const UPDATE_EVERY_NONCE: u64 = 10;
 #[tokio::main(flavor = "current_thread")]
 async fn main() -> Result<()> {
     let config: MinerConfig = MinerConfig::parse();
-    let prompt = Prompt::new(config.log_level, &config.logs_path, &config.filename_log, config.disable_file_logging, config.disable_file_log_date_based, config.disable_log_color, !config.disable_interactive_mode, config.logs_modules, config.file_log_level)?;
+    let prompt = Prompt::new(config.log_level, &config.logs_path, &config.filename_log, config.disable_file_logging, config.disable_file_log_date_based, config.disable_log_color, !config.disable_interactive_mode, config.logs_modules, config.file_log_level.unwrap_or(config.log_level))?;
 
     let detected_threads = match thread::available_parallelism() {
         Ok(value) => value.get() as u16,

xelis_wallet/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "xelis_wallet"
-version = "1.13.3"
+version = "1.13.4"
 edition = "2021"
 authors = ["Slixe <slixeprivate@gmail.com>"]
 

xelis_wallet/src/main.rs

@@ -97,7 +97,7 @@ pub struct RPCConfig {
 }
 
 #[derive(Parser)]
-#[clap(version = VERSION, about = "XELIS: An innovate cryptocurrency with BlockDAG and Homomorphic Encryption enabling Smart Contracts")]
+#[clap(version = VERSION, about = "XELIS is an innovative cryptocurrency built from scratch with BlockDAG, Homomorphic Encryption, Zero-Knowledge Proofs, and Smart Contracts.")]
 #[command(styles = xelis_common::get_cli_styles())]
 pub struct Config {
     /// Daemon address to use
@@ -112,8 +112,9 @@ pub struct Config {
     #[clap(long, value_enum, default_value_t = LogLevel::Info)]
     log_level: LogLevel,
     /// Set file log level
-    #[clap(long, value_enum, default_value_t = LogLevel::Info)]
-    file_log_level: LogLevel,
+    /// By default, it will be the same as log level
+    #[clap(long, value_enum)]
+    file_log_level: Option<LogLevel>,
     /// Disable the log file
     #[clap(long)]
     disable_file_logging: bool,
@@ -194,7 +195,7 @@ impl ecdlp::ProgressTableGenerationReportFunction for LogProgressTableGeneration
 #[tokio::main]
 async fn main() -> Result<()> {
     let config: Config = Config::parse();
-    let prompt = Prompt::new(config.log_level, &config.logs_path, &config.filename_log, config.disable_file_logging, config.disable_file_log_date_based, config.disable_log_color, !config.disable_interactive_mode, config.logs_modules, config.file_log_level)?;
+    let prompt = Prompt::new(config.log_level, &config.logs_path, &config.filename_log, config.disable_file_logging, config.disable_file_log_date_based, config.disable_log_color, !config.disable_interactive_mode, config.logs_modules, config.file_log_level.unwrap_or(config.log_level))?;
 
     #[cfg(feature = "api_server")]
     {