support httpsig pk registry

Cargo.toml

@@ -28,7 +28,7 @@ edition = "2021"
 publish = false
 
 [workspace]
-members = ["modoh-bin", "modoh-lib", "httpsig-wire-proto"]
+members = ["modoh-bin", "modoh-lib", "httpsig-wire-proto", "httpsig-registry"]
 exclude = ["submodules/hyper-tls"]
 resolver = "2"
 

httpsig-registry/Cargo.toml

@@ -0,0 +1,37 @@
+[package]
+name = "httpsig-registry"
+description = "Handler for endpoints that serves wire-formatted HTTPSig public keys"
+version.workspace = true
+authors.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+readme.workspace = true
+categories.workspace = true
+keywords.workspace = true
+edition.workspace = true
+publish.workspace = true
+
+[dependencies]
+anyhow = { version = "1.0.81" }
+thiserror = { version = "1.0.58" }
+pulldown-cmark = { version = "0.10.0", default-features = false }
+http = { version = "1.1.0" }
+indexmap = { version = "2.2.6" }
+minisign-verify = { version = "0.2.1" }
+reqwest = { version = "0.12.1", default-features = false, features = [
+  "rustls-tls",
+  "http2",
+  "hickory-dns",
+] }
+futures = { version = "0.3.30", default-features = false, features = [
+  "std",
+  "async-await",
+] }
+tokio = { version = "1.36.0", features = [
+  "net",
+  "rt-multi-thread",
+  "time",
+  "sync",
+  "macros",
+] }

httpsig-registry/src/constants.rs

@@ -0,0 +1 @@
+pub const HTTPSIG_CONFIGS_PATH: &str = "/.well-known/httpsigconfigs";

httpsig-registry/src/error.rs

@@ -0,0 +1,18 @@
+use thiserror::Error;
+
+/// Describes things that can go wrong in registry handling
+#[derive(Debug, Error)]
+pub enum ModohRegistryError {
+  /// Url parse error
+  #[error("Url parse error")]
+  FailToParseUrl,
+  /// IO error
+  #[error("IO error")]
+  Io(#[from] std::io::Error),
+  /// Reqwest error
+  #[error("Reqwest error")]
+  Reqwest(#[from] reqwest::Error),
+  /// Minisign error
+  #[error("Minisign error")]
+  Minisign(#[from] minisign_verify::Error),
+}

httpsig-registry/src/lib.rs

@@ -0,0 +1,117 @@
+mod constants;
+mod error;
+mod http_client;
+mod parse_md;
+
+use crate::{constants::HTTPSIG_CONFIGS_PATH, error::ModohRegistryError};
+use minisign_verify::{PublicKey, Signature};
+use std::{borrow::Cow, str::FromStr};
+
+/* ------------------------------------------------ */
+#[derive(Clone, Debug)]
+/// HTTP message signatures enabled domain information
+pub struct HttpSigDomainInfo {
+  /// Configs endpoint
+  pub configs_endpoint_uri: http::Uri,
+  /// Domain name
+  pub dh_signing_target_domain: String,
+}
+impl HttpSigDomainInfo {
+  /// Create a new HttpSigDomainInfo
+  pub fn new<'a, T: Into<Cow<'a, str>>>(configs_endpoint_domain: T, dh_signing_target_domain: Option<String>) -> Self {
+    let configs_endpoint_uri: http::Uri = format!("https://{}{}", configs_endpoint_domain.into(), HTTPSIG_CONFIGS_PATH)
+      .parse()
+      .unwrap();
+    let dh_signing_target_domain =
+      dh_signing_target_domain.unwrap_or_else(|| configs_endpoint_uri.authority().unwrap().to_string());
+    Self {
+      configs_endpoint_uri,
+      dh_signing_target_domain,
+    }
+  }
+
+  /// Create a new HttpSigDomainInfo by fetching endpoint list in markdown format from `file://<abs_path>` or `https://<domain>/<path>`
+  pub async fn new_from_registry_md<'a, T1, T2>(registry_uri: T1, minisign_base64_pk: T2) -> Result<Vec<Self>, ModohRegistryError>
+  where
+    T1: Into<Cow<'a, str>>,
+    T2: Into<Cow<'a, str>>,
+  {
+    // let registry_uri = registry_uri.into();
+    let reqwest_uri = reqwest::Url::from_str(&registry_uri.into()).map_err(|_| ModohRegistryError::FailToParseUrl)?;
+    if !reqwest_uri.path().ends_with(".md") {
+      return Err(ModohRegistryError::FailToParseUrl);
+    }
+    let (markdown_input, markdown_minisig_input) = match reqwest_uri.scheme() {
+      "file" => {
+        let markdown_path = reqwest_uri.to_file_path().map_err(|_| ModohRegistryError::FailToParseUrl)?;
+        let markdown_sig_path = markdown_path.with_extension("md.minisig");
+        let markdown_input = std::fs::read_to_string(markdown_path)?;
+        let markdown_minisig_input = std::fs::read_to_string(markdown_sig_path)?;
+        (markdown_input, markdown_minisig_input)
+      }
+      "https" => {
+        let mut reqwest_minisig_uri = reqwest_uri.clone();
+        reqwest_minisig_uri.set_path(&format!("{}.minisig", reqwest_uri.path()));
+        let client = reqwest::Client::new();
+        let futs = vec![client.get(reqwest_uri).send(), client.get(reqwest_minisig_uri).send()];
+        let res = futures::future::join_all(futs)
+          .await
+          .into_iter()
+          .collect::<Result<Vec<_>, _>>()?;
+        let texts = futures::future::join_all(res.into_iter().map(|v| v.text()))
+          .await
+          .into_iter()
+          .collect::<Result<Vec<_>, _>>()?;
+        (texts[0].clone(), texts[1].clone())
+      }
+      _ => return Err(ModohRegistryError::FailToParseUrl),
+    };
+
+    let minisign_pk = minisign_base64_pk.into();
+    let pk = PublicKey::from_base64(&minisign_pk)?;
+    let sig = Signature::decode(&markdown_minisig_input)?;
+    pk.verify(markdown_input.as_bytes(), &sig, false)?;
+
+    let parsed = parse_md::parse_md(markdown_input);
+    Ok(parsed)
+  }
+}
+
+/* ------------------------------------------------ */
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn it_works() {
+    let minisign_pk = "RWQm8wdk0lJP8AyGtShi96d72ZzkZnGX9gxR0F5EIWmMW2N25SDfzbrt";
+    let file_path = std::path::PathBuf::from("../.private/registry/httpsig-endpoints.md");
+    let file_path_minisig = std::path::PathBuf::from("../.private/registry/httpsig-endpoints.md.minisig");
+    let markdown_input = std::fs::read_to_string(file_path).unwrap();
+    let markdown_minisig_input = std::fs::read_to_string(file_path_minisig).unwrap();
+    let pk = PublicKey::from_base64(minisign_pk).unwrap();
+    let sig = Signature::decode(&markdown_minisig_input).unwrap();
+    let res = pk.verify(markdown_input.as_bytes(), &sig, false);
+    assert!(res.is_ok());
+
+    let parsed = parse_md::parse_md(markdown_input);
+    println!("{:#?}", parsed);
+  }
+
+  #[tokio::test]
+  async fn test_from_uri() {
+    let minisign_pk = "RWQm8wdk0lJP8AyGtShi96d72ZzkZnGX9gxR0F5EIWmMW2N25SDfzbrt";
+
+    let abs_path = std::path::PathBuf::from("../.private/registry/httpsig-endpoints.md")
+      .canonicalize()
+      .unwrap();
+    let string_path = format!("file://{}", abs_path.to_str().unwrap());
+    let res = HttpSigDomainInfo::new_from_registry_md(string_path, minisign_pk).await;
+    println!("from file:\n{:#?}", res);
+
+    let https_path = "https://filedn.com/lVEKDQEKcCIhnH516GYdXu0/modoh_httpsig_dev/httpsig-endpoints.md";
+    let res = HttpSigDomainInfo::new_from_registry_md(https_path, minisign_pk).await;
+    println!("from https:\n{:#?}", res);
+    assert!(res.is_ok());
+  }
+}

httpsig-registry/src/parse_md.rs

@@ -0,0 +1,84 @@
+use crate::HttpSigDomainInfo;
+use indexmap::IndexMap;
+use pulldown_cmark::{Event, HeadingLevel, Parser, Tag, TagEnd};
+use std::borrow::Cow;
+
+/// Parse the markdown and convert it to a vector of HttpSigDomainInfo
+pub(crate) fn parse_md<'a, T: Into<Cow<'a, str>>>(markdown_input: T) -> Vec<HttpSigDomainInfo> {
+  let markdown_input = markdown_input.into();
+  let parser = Parser::new(markdown_input.as_ref());
+
+  type VisitingState = Option<u8>;
+  const VISITING_H2: u8 = 0;
+  const VISITING_LIST: u8 = 1;
+  const VISITING_ITEM: u8 = 2;
+  let mut visiting_state: VisitingState = None;
+  let mut visiting_domain: Option<String> = None;
+  let mut domain_map: IndexMap<String, Vec<String>> = IndexMap::new();
+  let mut textbuf = String::new();
+  for event in parser {
+    match event {
+      Event::Start(Tag::Heading {
+        level: HeadingLevel::H2, ..
+      }) => {
+        visiting_state = Some(VISITING_H2);
+        visiting_domain = None;
+        textbuf.clear();
+      }
+      Event::End(TagEnd::Heading(HeadingLevel::H2)) => {
+        visiting_state = None;
+      }
+      Event::Start(Tag::List(_)) => {
+        visiting_state = Some(VISITING_LIST);
+        textbuf.clear();
+      }
+      Event::End(TagEnd::List(_)) => {
+        visiting_state = None;
+        visiting_domain = None;
+        textbuf.clear();
+      }
+      Event::Start(Tag::Item) => {
+        if matches!(visiting_state, Some(VISITING_LIST)) {
+          visiting_state = Some(VISITING_ITEM);
+          textbuf.clear();
+        }
+      }
+      Event::End(TagEnd::Item) => {
+        if matches!(visiting_state, Some(VISITING_ITEM)) {
+          visiting_state = Some(VISITING_LIST);
+          textbuf.clear();
+        }
+      }
+      Event::Text(text) => match visiting_state {
+        Some(VISITING_H2) => {
+          visiting_domain = Some(text.to_string());
+          domain_map.entry(text.trim().to_string()).or_default();
+          textbuf.clear();
+        }
+        Some(VISITING_ITEM) => {
+          if let Some(domain) = &visiting_domain {
+            textbuf.push_str(&text);
+            let text = text.trim().to_string();
+            if text != "*" {
+              domain_map.get_mut(domain).unwrap().push(textbuf.clone());
+              textbuf.clear();
+            }
+          }
+        }
+        _ => (),
+      },
+      _ => (),
+    }
+  }
+  let domain_info_vec = domain_map
+    .iter_mut()
+    .map(|(k, v)| if v.is_empty() { (k, vec![k.clone()]) } else { (k, v.clone()) })
+    .flat_map(|(k, v)| {
+      v.iter()
+        .map(move |dh_target| HttpSigDomainInfo::new(k.clone(), Some(dh_target.clone())))
+        .collect::<Vec<_>>()
+    })
+    .collect::<Vec<_>>();
+
+  domain_info_vec
+}

httpsig-wire-proto/Cargo.toml

@@ -17,7 +17,7 @@ anyhow = "1.0.81"
 thiserror = "1.0.58"
 rand = "0.8.5"
 hpke = "0.11.0"
-bytes = "1.5.0"
+bytes = "1.6.0"
 byteorder = "1.5.0"
 p256 = { version = "0.13.2" }
 elliptic-curve = { version = "0.13.8", features = ["ecdh"] }

httpsig-wire-proto/src/dh.rs

@@ -268,7 +268,7 @@ mod tests {
 
   #[test]
   fn test_derive_secret() {
-    let dh_types = vec![HttpSigDhTypes::Hs256DhP256HkdfSha256, HttpSigDhTypes::Hs256X25519HkdfSha256];
+    let dh_types = [HttpSigDhTypes::Hs256DhP256HkdfSha256, HttpSigDhTypes::Hs256X25519HkdfSha256];
     dh_types.iter().for_each(|t| {
       let alice_kp = t.generate_key_pair(&mut thread_rng());
       let bob_kp = t.generate_key_pair(&mut thread_rng());
@@ -289,7 +289,7 @@ mod tests {
 
   #[test]
   fn test_serialize_dh_config() {
-    let dh_types = vec![HttpSigDhTypes::Hs256DhP256HkdfSha256, HttpSigDhTypes::Hs256X25519HkdfSha256];
+    let dh_types = [HttpSigDhTypes::Hs256DhP256HkdfSha256, HttpSigDhTypes::Hs256X25519HkdfSha256];
     dh_types.iter().for_each(|t| {
       let kp = t.generate_key_pair(&mut thread_rng());
       let mut serialized_config = Vec::new();

modoh-bin/Cargo.toml

@@ -58,12 +58,12 @@ tokio = { version = "1.36.0", default-features = false, features = [
   "sync",
   "macros",
 ] }
-async-trait = "0.1.77"
+async-trait = "0.1.79"
 url = "2.5.0"
 
 # config
-clap = { version = "4.5.2", features = ["std", "cargo", "wrap_help"] }
-toml = { version = "0.8.11", default-features = false, features = ["parse"] }
+clap = { version = "4.5.3", features = ["std", "cargo", "wrap_help"] }
+toml = { version = "0.8.12", default-features = false, features = ["parse"] }
 hot_reload = "0.1.5"
 
 # tracing and metrics
@@ -79,7 +79,7 @@ opentelemetry-otlp = { version = "0.15.0", optional = true }
 opentelemetry-semantic-conventions = { version = "0.14.0", optional = true }
 
 # add random otel service id whenever restarting
-uuid = { version = "1.7.0", default-features = false, features = [
+uuid = { version = "1.8.0", default-features = false, features = [
   "v4",
   "fast-rng",
 ], optional = true }

modoh-bin/src/config/target_config.rs

@@ -3,7 +3,9 @@ use crate::{error::*, trace::*};
 use async_trait::async_trait;
 use hot_reload::{Reload, ReloaderError};
 use ipnet::IpNet;
-use modoh_server_lib::{AccessConfig, HttpSigConfig, HttpSigDomainInfo, ServiceConfig, ValidationConfig, ValidationConfigInner};
+use modoh_server_lib::{
+  AccessConfig, HttpSigConfig, HttpSigDomain, HttpSigRegistry, ServiceConfig, ValidationConfig, ValidationConfigInner,
+};
 use std::{
   fs::read_to_string,
   net::{IpAddr, SocketAddr},
@@ -218,17 +220,28 @@ impl TryInto<ServiceConfig> for &TargetConfig {
         if let Some(enabled_domains) = &httpsig.enabled_domains {
           let enabled_domains = enabled_domains
             .iter()
-            .map(|domain| {
-              HttpSigDomainInfo::new(
-                domain.configs_endpoint_domain.clone(),
-                domain.dh_signing_target_domain.clone(),
-              )
-            })
+            .map(|domain| HttpSigDomain::new(&domain.configs_endpoint_domain, domain.dh_signing_target_domain.as_deref()))
             .collect();
           httpsig_config.enabled_domains = enabled_domains;
         }
         info!("Set HttpSig-enabled targeted domains: {:#?}", httpsig_config.enabled_domains);
 
+        if let Some(enabled_domains_registry) = &httpsig.enabled_domains_registry {
+          let enabled_domains_registry = enabled_domains_registry
+            .iter()
+            .map(|registry| HttpSigRegistry::new(&registry.md_url, &registry.public_key))
+            .collect();
+          httpsig_config.enabled_domains_registry = enabled_domains_registry;
+        }
+        info!(
+          "Set HttpSig-enabled targeted domains registry: {:#?}",
+          httpsig_config
+            .enabled_domains_registry
+            .iter()
+            .map(|r| r.md_url.as_str())
+            .collect::<Vec<_>>()
+        );
+
         if let Some(false) = httpsig.accept_previous_dh_public_keys {
           httpsig_config.previous_dh_public_keys_gen = 0;
         }

modoh-bin/src/config/toml.rs

@@ -98,6 +98,8 @@ pub struct HttpSig {
   pub key_rotation_period: Option<u64>,
   /// List of HTTP message signatures enabled domains, which exposes public keys for Diffie-Hellman key exchange or directly for signature verification.
   pub enabled_domains: Option<Vec<HttpSigEnabledDomains>>,
+  /// Domain registry urls and public keys
+  pub enabled_domains_registry: Option<Vec<HttpSigEnabledDomainsRegistry>>,
   /// Accept previous DH public keys to fill the gap of the key rotation period.
   pub accept_previous_dh_public_keys: Option<bool>,
   /// Force httpsig verification for all requests regardless of the source ip validation result.
@@ -116,6 +118,14 @@ pub struct HttpSigEnabledDomains {
   /// DH signing target domain
   pub dh_signing_target_domain: Option<String>,
 }
+#[derive(Deserialize, Debug, PartialEq, Eq, Clone)]
+/// Object with url and public_key
+pub struct HttpSigEnabledDomainsRegistry {
+  /// URL
+  pub md_url: String,
+  /// Public key
+  pub public_key: String,
+}
 
 impl ConfigToml {
   pub(super) fn new(config_file: &str) -> anyhow::Result<Self> {