refactor: better caching, optional auth check

.gitignore

@@ -4,6 +4,7 @@
 event-logs
 faucet/kv
 keyfile.json
+config.json
 .deploy/monitoring/data
 .deploy/monitoring/prometheus
 .deploy/monitoring/alertmanager

faucet/Cargo.toml

@@ -32,6 +32,9 @@ async-trait = "0.1.40"
 futures = "0.3.5"
 git-version = "0.3.4"
 
+reqwest = "0.11.11"
+url = "2.2.2"
+
 # Workspace dependencies
 runtime = { path = "../runtime" }
 service = { path = "../service" }

faucet/src/error.rs

@@ -1,13 +1,15 @@
 #![allow(clippy::enum_variant_names)]
 
-use chrono::ParseError;
+use chrono::ParseError as ChronoParseError;
 use jsonrpc_http_server::jsonrpc_core::Error as JsonRpcError;
 use kv::Error as KvError;
 use parity_scale_codec::Error as CodecError;
+use reqwest::Error as ReqwestError;
 use runtime::Error as RuntimeError;
 use serde_json::Error as SerdeJsonError;
 use std::{io::Error as IoError, net::AddrParseError};
 use thiserror::Error;
+use url::ParseError as UrlParseError;
 
 #[derive(Error, Debug)]
 pub enum Error {
@@ -21,16 +23,23 @@ pub enum Error {
     AddrParseError(#[from] AddrParseError),
     #[error("Kv store error: {0}")]
     KvError(#[from] KvError),
+    #[error("ReqwestError: {0}")]
+    ReqwestError(#[from] ReqwestError),
+    #[error("UrlParseError: {0}")]
+    UrlParseError(#[from] UrlParseError),
     #[error("Error parsing datetime string: {0}")]
-    DatetimeParsingError(#[from] ParseError),
+    DatetimeParsingError(#[from] ChronoParseError),
+    #[error("IoError: {0}")]
+    IoError(#[from] IoError),
+    #[error("SerdeJsonError: {0}")]
+    SerdeJsonError(#[from] SerdeJsonError),
+
     #[error("Requester balance already sufficient")]
     AccountBalanceExceedsMaximum,
     #[error("Requester was recently funded")]
     AccountAlreadyFunded,
     #[error("Mathematical operation error")]
     MathError,
-    #[error("IoError: {0}")]
-    IoError(#[from] IoError),
-    #[error("SerdeJsonError: {0}")]
-    SerdeJsonError(#[from] SerdeJsonError),
+    #[error("Terms and conditions not signed")]
+    SignatureMissing,
 }

faucet/src/http.rs

@@ -7,18 +7,19 @@ use jsonrpc_http_server::{
 };
 use kv::*;
 use parity_scale_codec::{Decode, Encode};
+use reqwest::Url;
 use runtime::{
     AccountId, CollateralBalancesPallet, CurrencyId, Error as RuntimeError, InterBtcParachain, RuntimeCurrencyInfo,
-    TryFromSymbol, VaultRegistryPallet,
+    Ss58Codec, TryFromSymbol, VaultRegistryPallet, SS58_PREFIX,
 };
-use serde::{Deserialize, Deserializer};
+use serde::{Deserialize, Deserializer, Serialize};
 use std::{net::SocketAddr, time::Duration};
 use tokio::time::timeout;
 
 const HEALTH_DURATION: Duration = Duration::from_millis(5000);
 const KV_STORE_NAME: &str = "store";
 
-#[derive(serde::Serialize, serde::Deserialize, PartialEq)]
+#[derive(Serialize, Deserialize, PartialEq)]
 struct FaucetRequest {
     datetime: String,
     account_type: FundingRequestAccountType,
@@ -137,14 +138,14 @@ fn has_request_expired(
 
 async fn ensure_funding_allowed(
     parachain_rpc: &InterBtcParachain,
-    account_id: AccountId,
-    allowance_config: AllowanceConfig,
+    account_id: &AccountId,
+    allowance_config: &AllowanceConfig,
     last_request_json: Option<Json<FaucetRequest>>,
     account_type: FundingRequestAccountType,
 ) -> Result<(), Error> {
     let account_allowances = match account_type {
-        FundingRequestAccountType::User => allowance_config.user_allowances,
-        FundingRequestAccountType::Vault => allowance_config.vault_allowances,
+        FundingRequestAccountType::User => &allowance_config.user_allowances,
+        FundingRequestAccountType::Vault => &allowance_config.vault_allowances,
     };
     let currency_ids: Result<Vec<_>, _> = account_allowances
         .iter()
@@ -195,6 +196,21 @@ async fn ensure_funding_allowed(
     }
 }
 
+#[derive(Deserialize)]
+struct GetSignatureData {
+    exists: bool,
+}
+
+async fn ensure_signature_exists(auth_url: &str, account_id: &AccountId) -> Result<(), Error> {
+    reqwest::get(Url::parse(auth_url)?.join(&account_id.to_ss58check_with_version(SS58_PREFIX.into()))?)
+        .await?
+        .json::<GetSignatureData>()
+        .await?
+        .exists
+        .then(|| ())
+        .ok_or(Error::SignatureMissing)
+}
+
 async fn atomic_faucet_funding(
     parachain_rpc: &InterBtcParachain,
     kv: Bucket<'_, String, Json<FaucetRequest>>,
@@ -211,35 +227,46 @@ async fn atomic_faucet_funding(
 
     ensure_funding_allowed(
         parachain_rpc,
-        account_id.clone(),
-        allowance_config,
+        &account_id,
+        &allowance_config,
         last_request_json,
         account_type.clone(),
     )
     .await?;
 
-    let mut transfers = vec![];
-    for AllowanceAmount { symbol, amount } in amounts.iter() {
-        let currency_id = CurrencyId::try_from_symbol(symbol.clone())?;
-        log::info!(
-            "AccountId: {}, Currency: {:?} Type: {:?}, Amount: {}",
-            account_id,
-            currency_id.symbol().unwrap_or_default(),
-            account_type,
-            amount
-        );
-        transfers.push(parachain_rpc.transfer_to(&account_id, *amount, currency_id));
+    if let Some(auth_url) = allowance_config.auth_url {
+        ensure_signature_exists(&auth_url, &account_id).await?;
     }
 
-    let result = futures::future::join_all(transfers).await;
+    // replace the previous (expired) claim datetime with the datetime of the current claim
+    update_kv_store(&kv, account_str.clone(), Utc::now().to_rfc2822(), account_type.clone())?;
+
+    let transfers = amounts
+        .into_iter()
+        .map(|AllowanceAmount { symbol, amount }| {
+            let currency_id = CurrencyId::try_from_symbol(symbol.clone())?;
+            log::info!(
+                "AccountId: {}, Currency: {:?} Type: {:?}, Amount: {}",
+                account_id,
+                currency_id.symbol().unwrap_or_default(),
+                account_type,
+                amount
+            );
+            Ok((amount, currency_id))
+        })
+        .collect::<Result<Vec<_>, Error>>()?;
+
+    if let Err(err) = parachain_rpc.transfer_to(&account_id, transfers).await {
+        log::error!("Failed to fund {}", account_str);
+        if err.is_any_module_err() || err.is_invalid_transaction().is_some() {
+            log::info!("Removing previous claim");
+            // transfer failed, reset the db so this can be called again
+            kv.remove(account_str)?;
+        }
 
-    if let Some(err) = result.into_iter().find_map(|x| x.err()) {
         return Err(err.into());
     }
 
-    // Replace the previous (expired) claim datetime with the datetime of the current claim, only update
-    // this after successfully transferring funds to ensure that this can be called again on error
-    update_kv_store(&kv, account_str, Utc::now().to_rfc2822(), account_type.clone())?;
     Ok(())
 }
 
@@ -262,7 +289,7 @@ pub async fn start_http(
     allowance_config: AllowanceConfig,
 ) -> jsonrpc_http_server::CloseHandle {
     let mut io = IoHandler::default();
-    let store = Store::new(Config::new("./kv")).expect("Unable to open kv store");
+    let store = Store::new(Config::new("./kv").flush_every_ms(100)).expect("Unable to open kv store");
     let user_allowances_clone = allowance_config.user_allowances.clone();
     let vault_allowances_clone = allowance_config.vault_allowances.clone();
     io.add_sync_method("user_allowance", move |_| handle_resp(Ok(&user_allowances_clone)));
@@ -397,7 +424,7 @@ mod tests {
         join_all(balance.iter().map(|(amount, currency)| {
             let leftover = leftover_units * 10u128.pow(currency.decimals().unwrap());
             let amount_to_transfer = if *amount > leftover { amount - leftover } else { 0 };
-            provider.transfer_to(&drain_account_id, amount_to_transfer, currency.clone())
+            provider.transfer_to(&drain_account_id, vec![(amount_to_transfer, currency.clone())])
         }))
         .await
         .into_iter()

faucet/src/lib.rs

@@ -1,10 +1,12 @@
 use parity_scale_codec::{Decode, Encode};
 use serde::Deserialize;
+
 #[derive(Deserialize, Debug, Clone, Encode, Decode)]
 pub struct AllowanceAmount {
     pub symbol: String,
     pub amount: u128,
 }
+
 impl AllowanceAmount {
     pub fn new(symbol: String, amount: u128) -> Self {
         Self { symbol, amount }

faucet/src/main.rs

@@ -40,6 +40,7 @@ pub struct AllowanceConfig {
     pub faucet_cooldown_hours: i64,
     pub user_allowances: Allowance,
     pub vault_allowances: Allowance,
+    pub auth_url: Option<String>,
 }
 
 impl AllowanceConfig {
@@ -54,6 +55,7 @@ impl AllowanceConfig {
             faucet_cooldown_hours,
             user_allowances,
             vault_allowances,
+            auth_url: None,
         }
     }
 }

runtime/src/error.rs

@@ -108,6 +108,13 @@ impl From<module_bitcoin::Error> for Error {
 }
 
 impl Error {
+    pub fn is_any_module_err(&self) -> bool {
+        matches!(
+            self,
+            Error::SubxtRuntimeError(SubxtError::Runtime(DispatchError::Module(_))),
+        )
+    }
+
     fn is_module_err(&self, pallet_name: &str, error_name: &str) -> bool {
         matches!(
             self,

runtime/src/rpc.rs

@@ -836,7 +836,7 @@ pub trait CollateralBalancesPallet {
 
     async fn get_reserved_balance_for_id(&self, id: AccountId, currency_id: CurrencyId) -> Result<Balance, Error>;
 
-    async fn transfer_to(&self, recipient: &AccountId, amount: u128, currency_id: CurrencyId) -> Result<(), Error>;
+    async fn transfer_to(&self, recipient: &AccountId, amounts: Vec<(u128, CurrencyId)>) -> Result<(), Error>;
 }
 
 #[async_trait]
@@ -859,10 +859,20 @@ impl CollateralBalancesPallet for InterBtcParachain {
         Ok(self.query_finalized_or_default(storage_key).await?.reserved)
     }
 
-    async fn transfer_to(&self, recipient: &AccountId, amount: u128, currency_id: CurrencyId) -> Result<(), Error> {
-        self.with_unique_signer(metadata::tx().tokens().transfer(recipient.clone(), currency_id, amount))
-            .await?;
-        Ok(())
+    async fn transfer_to(&self, recipient: &AccountId, amounts: Vec<(u128, CurrencyId)>) -> Result<(), Error> {
+        self.batch(
+            amounts
+                .into_iter()
+                .map(|(amount, currency_id)| {
+                    EncodedCall::Tokens(metadata::runtime_types::orml_tokens::module::Call::transfer {
+                        dest: recipient.clone(),
+                        currency_id,
+                        amount,
+                    })
+                })
+                .collect(),
+        )
+        .await
     }
 }
 

vault/src/metrics.rs

@@ -748,7 +748,7 @@ mod tests {
             async fn get_free_balance_for_id(&self, id: AccountId, currency_id: CurrencyId) -> Result<Balance, RuntimeError>;
             async fn get_reserved_balance(&self, currency_id: CurrencyId) -> Result<Balance, RuntimeError>;
             async fn get_reserved_balance_for_id(&self, id: AccountId, currency_id: CurrencyId) -> Result<Balance, RuntimeError>;
-            async fn transfer_to(&self, recipient: &AccountId, amount: u128, currency_id: CurrencyId) -> Result<(), RuntimeError>;
+            async fn transfer_to(&self, recipient: &AccountId, amounts: Vec<(u128, CurrencyId)>) -> Result<(), RuntimeError>;
         }
 
         #[async_trait]

vault/src/replace.rs

@@ -344,7 +344,7 @@ mod tests {
         async fn get_free_balance_for_id(&self, id: AccountId, currency_id: CurrencyId) -> Result<Balance, RuntimeError>;
         async fn get_reserved_balance(&self, currency_id: CurrencyId) -> Result<Balance, RuntimeError>;
         async fn get_reserved_balance_for_id(&self, id: AccountId, currency_id: CurrencyId) -> Result<Balance, RuntimeError>;
-        async fn transfer_to(&self, recipient: &AccountId, amount: u128, currency_id: CurrencyId) -> Result<(), RuntimeError>;         }
+        async fn transfer_to(&self, recipient: &AccountId, amounts: Vec<(u128, CurrencyId)>) -> Result<(), RuntimeError>;         }
     }
 
     impl Clone for MockProvider {