0.2.0

.github/workflows/release_docker.yml

@@ -3,6 +3,7 @@ name: Nightly and release build
 on:
   push:
     branches:
+      - "feat/*"
       - "develop"
   pull_request:
     types: [closed]
@@ -75,13 +76,27 @@ jobs:
           cache-to: type=gha,mode=max,scope=modoh-server-nightly
           labels: ${{ steps.meta.outputs.labels }}
 
+      - name: Unstable build and push x86_64 for 'feat/*' branches (for development purposes)
+        if: ${{ startsWith(github.ref_name, 'feat/') && (github.event_name == 'push') }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.GHCR }}/${{ env.GHCR_IMAGE_NAME }}:unstable
+            ${{ env.DH_REGISTRY_NAME }}:unstable
+          file: ./docker/Dockerfile
+          cache-from: type=gha,scope=modoh-server-unstable
+          cache-to: type=gha,mode=max,scope=modoh-server-unstable
+          labels: ${{ steps.meta.outputs.labels }}
+
   dispatch_release:
     runs-on: ubuntu-latest
     if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref == 'develop' && github.event.pull_request.base.ref == 'main' && github.event.pull_request.merged == true }}
     needs: docker_build_and_push
     steps:
       - name: check pull_request title
-        uses: kaisugi/action-regex-match@v1.0.0
+        uses: kaisugi/action-regex-match@v1.0.1
         id: regex-match
         with:
           text: ${{ github.event.pull_request.title }}
@@ -102,7 +117,7 @@ jobs:
 
       - name: release
         if: ${{ steps.regex-match.outputs.match != ''}}
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           files: /tmp/modoh-server-x86_64-unknown-linux-gnu.tar.gz
           name: ${{ github.event.pull_request.title }}

Cargo.toml

@@ -1,6 +1,34 @@
-[workspace]
+[workspace.package]
+version = "0.2.0"
+authors = ["Jun Kurihara"]
+homepage = "https://github.com/junkurihara/modoh-server"
+repository = "https://github.com/junkurihara/modoh-server"
+license = "MIT"
+readme = "./README.md"
+categories = [
+  "asynchronous",
+  "network-programming",
+  "command-line-utilities",
+  "web-programming::http-server",
+]
+keywords = [
+  "dns",
+  "https",
+  "dns-over-https",
+  "doh",
+  "oblivious-dns-over-https",
+  "odoh",
+  "mutualized-oblivious-dns",
+  "mutualized-odoh",
+  "modoh",
+  "relay",
+  "authorization",
+]
+edition = "2021"
+publish = false
 
-members = ["modoh-bin", "modoh-lib"]
+[workspace]
+members = ["modoh-bin", "modoh-lib", "httpsig-wire-proto", "httpsig-registry"]
 exclude = ["submodules/hyper-tls"]
 resolver = "2"
 

docker-otel/README.md

@@ -0,0 +1,15 @@
+# Example Construction of Opentelemetry and its Related Servers for Observability
+
+Here is an example of docker containers for the observability of `modoh-server`, conjunctively configured with its container in [`../docker/`](../docker/) directory.
+
+![Example for Observability](../assets/observability.jpg)
+
+This example consists of the following containers:
+
+- [`opentelemetry-collector`](https://github.com/open-telemetry/opentelemetry-collector): Receives and aggregates OTLP gRPC messages from `modoh-server`. In this example, it is collocated with `modoh-server` in a virtual local network. You need to update [`./otel-config.yml`](./otel-config.yml) as your setting.
+- [`Jaeger`](https://www.jaegertracing.io/): Receives trace information via OTLP gRPC from `opentelemetry-collector`, and visualize the trace on the web ([`http://localhost:16686`](http://localhost:16686)). Currently `Jaeger` in our setting uses a non-persistent in-memory storage.
+- [`Grafana mimir`](https://github.com/grafana/mimir): Receives metrics information via Prometheus Remote Write protocol from `opentelemetry-collector`. This is responsible to aggregate and store the metric information in a long-term storage like object storage services. Update [`./mimir.yml`](./mimir.yml) and [`./mimir-alertmanager-fallback.yml`](./mimir-alertmanager-fallback.yml) if needed.
+- [`Grafana`](https://grafana.com/): Retrieves Prometheus metrics from `Grafana mimir` and visualize the metrics information on the web ([`http://localhost:3000`](http://localhost:3000)).
+- [`Rclone`](https://rclone.org/): Serves an S3 compatible object storage backed by a certain cloud storage service like Dropbox. This is connected from `Grafana mimir` to store the metrics data. `rclone.conf` requires to be configured. (See the [Rclone docs](https://rclone.org/docs/))
+
+For the detailed configurations for these containers, please refer to [`./docker-compose.yml`](./docker-compose.yml) and its mounting configuration files. Of course, the [`./docker-compose.yml`](./docker-compose.yml) itself should be updated according to your environment.

docker-otel/docker-compose.yml

@@ -31,6 +31,7 @@ services:
     image: jaegertracing/all-in-one:latest
     container_name: jaeger
     restart: unless-stopped
+    command: ["--memory.max-traces", "4096"] # limits the memory used by the in-memory storage
     ports:
       - 127.0.0.1:16686:16686 # frontend
     expose:

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04 AS base
+FROM ubuntu:24.04 AS base
 
 SHELL ["/bin/sh", "-x", "-c"]
 ENV SERIAL 2

docker/README.md

@@ -0,0 +1,26 @@
+# Docker container of `modoh-server`
+
+## Environment Variables
+
+We have several container-specific environment variables, which doesn't relates the behavior of `modoh-server`.
+
+- `HOST_USER` (default: `user`): User name executing `rpxy` inside the container.
+- `HOST_UID` (default: `900`): `UID` of `HOST_USER`.
+- `HOST_GID` (default: `900`): `GID` of `HOST_USER`
+- `LOG_LEVEL=debug|info|warn|error` (default: `info`): Log level
+- `LOG_TO_FILE=true|false` (default: `false`): Enable logging to the log file `/modoh/log/modoh-server.log` using `logrotate`. You should mount `/modoh/log` via docker volume option if enabled. The log dir and file will be owned by the `HOST_USER` with `HOST_UID:HOST_GID` on the host machine. Hence, `HOST_USER`, `HOST_UID` and `HOST_GID` should be the same as ones of the user who executes the `modoh-server` container on the host.
+- `DISABLE_OTEL`: If explicitly set to `true`, `--trace` and `--metrics` are disabled in the execute option. (default: `false`)
+- `OTEL_ENDPOINT`: Set the gRPC endpoint of `opentelemetry-collector`. (default: `http://localhost:4317` but no collector is contained in the `modoh-server` docker container.)
+
+See [`./docker-compose.yml`](./docker-compose.yml) for the detailed configuration of the above environment variables.
+
+## Volumes
+
+At least, the configured `config.toml` file (or its contained directory) must be mounted:
+
+- case 1: only `config.toml` is directly mounted as `/etc/modoh-server.toml`. Then, the *hot-reload function is disabled* due to the limitation of docker.
+- case 2: `path/to/config_dir/` containing `config.toml` is mounted as `/modoh/config`. Then `modoh-server` can tracks changes of the configuration file.
+
+In addition to the configuration file/directory, the list of up-to-date CDN IP addresses should be mounted and set in the `config.toml`. See [`../modoh-server.toml`](../modoh-server.toml) and [`./docker-compose.yml`](./docker-compose.yml).
+
+You may also need to mount the log directory (`/modoh/log`).

docker/docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3.9"
 services:
   modoh-server:
     image: jqtype/modoh-server:latest # ghcr.io/junkurihara/modoh-server:latest also works
@@ -20,7 +19,8 @@ services:
       - HOST_UID=501
       - HOST_GID=501
       # - WATCH=true
-      - OTLP_ENDPOINT=http://otel-collector:4317 # opentelemetry is enabled if specified the OTLP_ENDPOINT
+      # - DISABLE_OTEL=true # opentelemetry is disabled if DISABLE_OTEL=true (default: false)
+      - OTLP_ENDPOINT=http://otel-collector:4317 # opentelemetry endpoint (default: http://localhost:4317)
     tty: false
     privileged: true
     volumes:

httpsig-registry/Cargo.toml

@@ -0,0 +1,37 @@
+[package]
+name = "httpsig-registry"
+description = "Handler for endpoints that serves wire-formatted HTTPSig public keys"
+version.workspace = true
+authors.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+readme.workspace = true
+categories.workspace = true
+keywords.workspace = true
+edition.workspace = true
+publish.workspace = true
+
+[dependencies]
+anyhow = { version = "1.0.83" }
+thiserror = { version = "1.0.60" }
+pulldown-cmark = { version = "0.10.3", default-features = false }
+http = { version = "1.1.0" }
+indexmap = { version = "2.2.6" }
+minisign-verify = { version = "0.2.1" }
+reqwest = { version = "0.12.4", default-features = false, features = [
+  "rustls-tls",
+  "http2",
+  "hickory-dns",
+] }
+futures = { version = "0.3.30", default-features = false, features = [
+  "std",
+  "async-await",
+] }
+tokio = { version = "1.37.0", features = [
+  "net",
+  "rt-multi-thread",
+  "time",
+  "sync",
+  "macros",
+] }

httpsig-registry/registry-sample/httpsig-endpoints.md

@@ -0,0 +1,13 @@
+# List of endpoints serving wire-formatted HTTPSig public keys for MODoH
+
+## modoh01.typeq.org
+
+Below is the list of target domain names that are handled under the public keys of the above endpoint (For DHKex). Wildcard for prefix, i.e., `*.example.com` is acceptable. If not specified, it is identical to the endpoint domain.
+
+- modoh01.typeq.org
+
+## modoh02.typeq.org
+
+## modoh03.typeq.org
+
+## dnsauth.typeq.org

httpsig-registry/registry-sample/httpsig-endpoints.md.minisig

@@ -0,0 +1,4 @@
+untrusted comment: signature from rsign secret key
+RUQm8wdk0lJP8EHUuBcr5PqdYYiBqC7XVePhn4VByqvpWUHcEQ8RQ4DUA6WS11deeQmVqj6nX4IwDSpol6YaP4wxN1CyAbMu4QI=
+trusted comment: timestamp:1711525202	file:httpsig-endpoints.md	prehashed
+l+/B9UT3aI65PbBVCl/ptq6jSbDj0bd/aT3C0F0dFfdoj32N/bjw7T9Hj8v2UKzb6lfx6mvUPM23j3pbtur9BA==

httpsig-registry/src/constants.rs

@@ -0,0 +1 @@
+pub const HTTPSIG_CONFIGS_PATH: &str = "/.well-known/httpsigconfigs";

httpsig-registry/src/error.rs

@@ -0,0 +1,18 @@
+use thiserror::Error;
+
+/// Describes things that can go wrong in registry handling
+#[derive(Debug, Error)]
+pub enum ModohRegistryError {
+  /// Url parse error
+  #[error("Url parse error")]
+  FailToParseUrl,
+  /// IO error
+  #[error("IO error")]
+  Io(#[from] std::io::Error),
+  /// Reqwest error
+  #[error("Reqwest error")]
+  Reqwest(#[from] reqwest::Error),
+  /// Minisign error
+  #[error("Minisign error")]
+  Minisign(#[from] minisign_verify::Error),
+}

httpsig-registry/src/lib.rs

@@ -0,0 +1,117 @@
+mod constants;
+mod error;
+mod http_client;
+mod parse_md;
+
+use crate::{constants::HTTPSIG_CONFIGS_PATH, error::ModohRegistryError};
+use minisign_verify::{PublicKey, Signature};
+use std::{borrow::Cow, str::FromStr};
+
+/* ------------------------------------------------ */
+#[derive(Clone, Debug)]
+/// HTTP message signatures enabled domain information
+pub struct HttpSigDomainInfo {
+  /// Configs endpoint
+  pub configs_endpoint_uri: http::Uri,
+  /// Domain name
+  pub dh_signing_target_domain: String,
+}
+impl HttpSigDomainInfo {
+  /// Create a new HttpSigDomainInfo
+  pub fn new<'a, T: Into<Cow<'a, str>>>(configs_endpoint_domain: T, dh_signing_target_domain: Option<String>) -> Self {
+    let configs_endpoint_uri: http::Uri = format!("https://{}{}", configs_endpoint_domain.into(), HTTPSIG_CONFIGS_PATH)
+      .parse()
+      .unwrap();
+    let dh_signing_target_domain =
+      dh_signing_target_domain.unwrap_or_else(|| configs_endpoint_uri.authority().unwrap().to_string());
+    Self {
+      configs_endpoint_uri,
+      dh_signing_target_domain,
+    }
+  }
+
+  /// Create a new HttpSigDomainInfo by fetching endpoint list in markdown format from `file://<abs_path>` or `https://<domain>/<path>`
+  pub async fn new_from_registry_md<'a, T1, T2>(registry_uri: T1, minisign_base64_pk: T2) -> Result<Vec<Self>, ModohRegistryError>
+  where
+    T1: Into<Cow<'a, str>>,
+    T2: Into<Cow<'a, str>>,
+  {
+    // let registry_uri = registry_uri.into();
+    let reqwest_uri = reqwest::Url::from_str(&registry_uri.into()).map_err(|_| ModohRegistryError::FailToParseUrl)?;
+    if !reqwest_uri.path().ends_with(".md") {
+      return Err(ModohRegistryError::FailToParseUrl);
+    }
+    let (markdown_input, markdown_minisig_input) = match reqwest_uri.scheme() {
+      "file" => {
+        let markdown_path = reqwest_uri.to_file_path().map_err(|_| ModohRegistryError::FailToParseUrl)?;
+        let markdown_sig_path = markdown_path.with_extension("md.minisig");
+        let markdown_input = std::fs::read_to_string(markdown_path)?;
+        let markdown_minisig_input = std::fs::read_to_string(markdown_sig_path)?;
+        (markdown_input, markdown_minisig_input)
+      }
+      "https" => {
+        let mut reqwest_minisig_uri = reqwest_uri.clone();
+        reqwest_minisig_uri.set_path(&format!("{}.minisig", reqwest_uri.path()));
+        let client = reqwest::Client::new();
+        let futs = vec![client.get(reqwest_uri).send(), client.get(reqwest_minisig_uri).send()];
+        let res = futures::future::join_all(futs)
+          .await
+          .into_iter()
+          .collect::<Result<Vec<_>, _>>()?;
+        let texts = futures::future::join_all(res.into_iter().map(|v| v.text()))
+          .await
+          .into_iter()
+          .collect::<Result<Vec<_>, _>>()?;
+        (texts[0].clone(), texts[1].clone())
+      }
+      _ => return Err(ModohRegistryError::FailToParseUrl),
+    };
+
+    let minisign_pk = minisign_base64_pk.into();
+    let pk = PublicKey::from_base64(&minisign_pk)?;
+    let sig = Signature::decode(&markdown_minisig_input)?;
+    pk.verify(markdown_input.as_bytes(), &sig, false)?;
+
+    let parsed = parse_md::parse_md(markdown_input);
+    Ok(parsed)
+  }
+}
+
+/* ------------------------------------------------ */
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn it_works() {
+    let minisign_pk = "RWQm8wdk0lJP8AyGtShi96d72ZzkZnGX9gxR0F5EIWmMW2N25SDfzbrt";
+    let file_path = std::path::PathBuf::from("./registry-sample/httpsig-endpoints.md");
+    let file_path_minisig = std::path::PathBuf::from("./registry-sample/httpsig-endpoints.md.minisig");
+    let markdown_input = std::fs::read_to_string(file_path).unwrap();
+    let markdown_minisig_input = std::fs::read_to_string(file_path_minisig).unwrap();
+    let pk = PublicKey::from_base64(minisign_pk).unwrap();
+    let sig = Signature::decode(&markdown_minisig_input).unwrap();
+    let res = pk.verify(markdown_input.as_bytes(), &sig, false);
+    assert!(res.is_ok());
+
+    let parsed = parse_md::parse_md(markdown_input);
+    println!("{:#?}", parsed);
+  }
+
+  #[tokio::test]
+  async fn test_from_uri() {
+    let minisign_pk = "RWQm8wdk0lJP8AyGtShi96d72ZzkZnGX9gxR0F5EIWmMW2N25SDfzbrt";
+
+    let abs_path = std::path::PathBuf::from("./registry-sample/httpsig-endpoints.md")
+      .canonicalize()
+      .unwrap();
+    let string_path = format!("file://{}", abs_path.to_str().unwrap());
+    let res = HttpSigDomainInfo::new_from_registry_md(string_path, minisign_pk).await;
+    println!("from file:\n{:#?}", res);
+
+    let https_path = "https://filedn.com/lVEKDQEKcCIhnH516GYdXu0/modoh_httpsig_dev/httpsig-endpoints.md";
+    let res = HttpSigDomainInfo::new_from_registry_md(https_path, minisign_pk).await;
+    println!("from https:\n{:#?}", res);
+    assert!(res.is_ok());
+  }
+}