From 63a2913bf2259af4a61743e9b860d572186c647b Mon Sep 17 00:00:00 2001
From: Daniel Porteous <danielporteous1@gmail.com>
Date: Thu, 8 Dec 2022 20:59:40 +0000
Subject: [PATCH] [NHC] Add Fetchers, overhaul Evaluator inputs

---
 .../framework/aptos-framework/doc/voting.md   |   7 +-
 .../node-checker/src/checker/minimum_peers.rs | 213 ++++++++++
 ecosystem/node-checker/src/checker/mod.rs     |  40 ++
 ecosystem/node-checker/src/checker/traits.rs  |  48 +++
 ecosystem/node-checker/src/checker/types.rs   |  69 ++++
 ecosystem/node-checker/src/common/clients.rs  | 101 +++++
 .../src/{ => common}/common_args.rs           |   0
 ecosystem/node-checker/src/common/mod.rs      |   8 +
 .../node-checker/src/configuration/common.rs  |  22 +-
 .../node-checker/src/configuration/create.rs  |  39 --
 .../node-checker/src/configuration/mod.rs     |  13 +-
 .../node-checker/src/configuration/types.rs   | 371 +++++-------------
 .../src/configuration/validate.rs             |   9 +-
 .../node-checker/src/evaluator/traits.rs      | 123 +-----
 ecosystem/node-checker/src/evaluator/types.rs |  32 +-
 ecosystem/node-checker/src/fetcher/metrics.rs | 135 +++++++
 ecosystem/node-checker/src/fetcher/mod.rs     |   7 +
 ecosystem/node-checker/src/fetcher/traits.rs  |  47 +++
 ecosystem/node-checker/src/handler/mod.rs     |   1 +
 .../src/handler/synchronous_handler.rs        |   9 +
 ecosystem/node-checker/src/lib.rs             |  13 +-
 .../reqwest_metric_collector.rs               |   3 +-
 .../node-checker/src/provider/helpers.rs      |  43 ++
 .../node-checker/src/provider/metrics.rs      | 150 +++++++
 ecosystem/node-checker/src/provider/mod.rs    |  13 +
 .../src/provider/provider_collection.rs       |  41 ++
 ecosystem/node-checker/src/provider/traits.rs |  46 +++
 .../src/runner/blocking_runner.rs             | 274 -------------
 ecosystem/node-checker/src/runner/mod.rs      |   4 +-
 .../node-checker/src/runner/sync_runner.rs    | 122 ++++++
 ecosystem/node-checker/src/runner/traits.rs   |  37 +-
 ecosystem/node-checker/src/server/api.rs      | 237 +++--------
 ecosystem/node-checker/src/server/build.rs    |  96 +++++
 ecosystem/node-checker/src/server/common.rs   |   4 +-
 .../src/server/configurations_manager.rs      |  88 -----
 ecosystem/node-checker/src/server/errors.rs   |   1 +
 ecosystem/node-checker/src/server/mod.rs      |   3 +-
 ecosystem/node-checker/src/server/run.rs      |  49 +--
 38 files changed, 1416 insertions(+), 1102 deletions(-)
 create mode 100644 ecosystem/node-checker/src/checker/minimum_peers.rs
 create mode 100644 ecosystem/node-checker/src/checker/mod.rs
 create mode 100644 ecosystem/node-checker/src/checker/traits.rs
 create mode 100644 ecosystem/node-checker/src/checker/types.rs
 create mode 100644 ecosystem/node-checker/src/common/clients.rs
 rename ecosystem/node-checker/src/{ => common}/common_args.rs (100%)
 create mode 100644 ecosystem/node-checker/src/common/mod.rs
 delete mode 100644 ecosystem/node-checker/src/configuration/create.rs
 create mode 100644 ecosystem/node-checker/src/fetcher/metrics.rs
 create mode 100644 ecosystem/node-checker/src/fetcher/mod.rs
 create mode 100644 ecosystem/node-checker/src/fetcher/traits.rs
 create mode 100644 ecosystem/node-checker/src/handler/mod.rs
 create mode 100644 ecosystem/node-checker/src/handler/synchronous_handler.rs
 create mode 100644 ecosystem/node-checker/src/provider/helpers.rs
 create mode 100644 ecosystem/node-checker/src/provider/metrics.rs
 create mode 100644 ecosystem/node-checker/src/provider/mod.rs
 create mode 100644 ecosystem/node-checker/src/provider/provider_collection.rs
 create mode 100644 ecosystem/node-checker/src/provider/traits.rs
 delete mode 100644 ecosystem/node-checker/src/runner/blocking_runner.rs
 create mode 100644 ecosystem/node-checker/src/runner/sync_runner.rs
 create mode 100644 ecosystem/node-checker/src/server/build.rs
 delete mode 100644 ecosystem/node-checker/src/server/configurations_manager.rs
 create mode 100644 ecosystem/node-checker/src/server/errors.rs

diff --git a/aptos-move/framework/aptos-framework/doc/voting.md b/aptos-move/framework/aptos-framework/doc/voting.md
index 74cdfaf2021c0..f92be497e9c45 100644
--- a/aptos-move/framework/aptos-framework/doc/voting.md
+++ b/aptos-move/framework/aptos-framework/doc/voting.md
@@ -107,8 +107,9 @@ Extra metadata (e.g. description, code url) can be part of the ProposalType stru
  Currently, we have three attributes that are used by the voting flow.
  1. RESOLVABLE_TIME_METADATA_KEY: this is uesed to record the resolvable time to ensure that resolution has to be done non-atomically.
  2. IS_MULTI_STEP_PROPOSAL_KEY: this is used to track if a proposal is single-step or multi-step.
- 3. IS_MULTI_STEP_PROPOSAL_IN_EXECUTION_KEY: this attribute only exists for and applies to multi-step proposals. The value is used to
- indicate if a multi-step proposal is in execution. If yes, we will disable further voting for this multi-step proposal.
+ 3. IS_MULTI_STEP_PROPOSAL_IN_EXECUTION_KEY: this attribute only applies to multi-step proposals. A single-step proposal will not have
+ this field in its metadata map. The value is used to indicate if a multi-step proposal is in execution. If yes, we will disable further
+ voting for this multi-step proposal.
 </dd>
 <dt>
 <code>creation_time_secs: u64</code>
@@ -889,6 +890,7 @@ This guarantees that voting eligibility and voting power are controlled by the r
 
 ## Function `is_proposal_resolvable`
 
+Common checks on if a proposal is resolvable, regardless if the proposal is single-step or multi-step.
 
 
 <pre><code><b>fun</b> <a href="voting.md#0x1_voting_is_proposal_resolvable">is_proposal_resolvable</a>&lt;ProposalType: store&gt;(voting_forum_address: <b>address</b>, proposal_id: u64)
@@ -1019,6 +1021,7 @@ there are more yes votes than no. If either of these conditions is not met, this
     <b>let</b> voting_forum = <b>borrow_global_mut</b>&lt;<a href="voting.md#0x1_voting_VotingForum">VotingForum</a>&lt;ProposalType&gt;&gt;(voting_forum_address);
     <b>let</b> proposal = <a href="../../aptos-stdlib/doc/table.md#0x1_table_borrow_mut">table::borrow_mut</a>(&<b>mut</b> voting_forum.proposals, proposal_id);
 
+    // Update the <a href="voting.md#0x1_voting_IS_MULTI_STEP_PROPOSAL_IN_EXECUTION_KEY">IS_MULTI_STEP_PROPOSAL_IN_EXECUTION_KEY</a> key <b>to</b> indicate that the multi-step proposal is in execution.
     <b>let</b> multi_step_in_execution_key = utf8(<a href="voting.md#0x1_voting_IS_MULTI_STEP_PROPOSAL_IN_EXECUTION_KEY">IS_MULTI_STEP_PROPOSAL_IN_EXECUTION_KEY</a>);
     <b>if</b> (<a href="../../aptos-stdlib/doc/simple_map.md#0x1_simple_map_contains_key">simple_map::contains_key</a>(&proposal.metadata, &multi_step_in_execution_key)) {
         <b>let</b> is_multi_step_proposal_in_execution_value = <a href="../../aptos-stdlib/doc/simple_map.md#0x1_simple_map_borrow_mut">simple_map::borrow_mut</a>(&<b>mut</b> proposal.metadata, &multi_step_in_execution_key);
diff --git a/ecosystem/node-checker/src/checker/minimum_peers.rs b/ecosystem/node-checker/src/checker/minimum_peers.rs
new file mode 100644
index 0000000000000..0cd110914e91a
--- /dev/null
+++ b/ecosystem/node-checker/src/checker/minimum_peers.rs
@@ -0,0 +1,213 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+/// These evaluators are only valuable in certain contexts. For example, this is
+/// not a useful evaluator for node registration for the AITs, since each node
+/// is running in their own isolated network, where no consensus is occurring.
+/// This is useful for the AIT itself though, where the nodes are participating
+/// in a real network.
+use crate::{
+    checker::{CheckResult, Checker},
+    get_provider,
+    provider::{
+        metrics::{get_metric, GetMetricResult, Label, MetricsProvider},
+        Provider, ProviderCollection,
+    },
+};
+use anyhow::Result;
+use once_cell::sync::Lazy;
+use prometheus_parse::Scrape;
+use serde::{Deserialize, Serialize};
+
+use super::traits::CheckerError;
+
+/// Evaluator for minimum number of peers.
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct MinimumPeersCheckerConfig {
+    #[serde(default)]
+    pub required: bool,
+
+    /// The minimum number of inbound connections required to be able to pass.
+    /// For fullnodes, it only matters that this is greater than zero if the
+    /// node operator wants to seed data to other nodes.
+    #[serde(default = "MinimumPeersCheckerConfig::default_minimum_peers_inbound")]
+    pub minimum_peers_inbound: u64,
+
+    /// The minimum number of outbound connections required to be able to pass.
+    /// This must be greater than zero for the node to be able to synchronize.
+    #[serde(default = "MinimumPeersCheckerConfig::default_minimum_peers_outbound")]
+    pub minimum_peers_outbound: u64,
+}
+
+impl MinimumPeersCheckerConfig {
+    pub fn default_minimum_peers_inbound() -> u64 {
+        0
+    }
+
+    pub fn default_minimum_peers_outbound() -> u64 {
+        1
+    }
+}
+
+#[derive(Debug)]
+pub struct MinimumPeersChecker {
+    config: MinimumPeersCheckerConfig,
+}
+
+impl MinimumPeersChecker {
+    pub fn new(config: MinimumPeersCheckerConfig) -> Self {
+        Self { config }
+    }
+
+    #[allow(clippy::comparison_chain)]
+    fn build_evaluation(
+        &self,
+        connections: u64,
+        minimum: u64,
+        connection_type: &ConnectionType,
+    ) -> CheckResult {
+        let name = connection_type.get_name();
+        let particle = connection_type.get_particle();
+        let opposite_particle = connection_type.get_opposite_particle();
+        let explanation = format!(
+            "There are {} {} connections {} other nodes {} the target node (the minimum is {}).",
+            connections, name, particle, opposite_particle, minimum
+        );
+        if connections >= minimum {
+            CheckResult::new(
+                format!(
+                    "There are sufficient {} connections {} the target node",
+                    name, particle
+                ),
+                100,
+                explanation,
+            )
+        } else {
+            CheckResult::new(
+                format!(
+                    "There are not enough {} connections {} the target node",
+                    name, particle
+                ),
+                50,
+                format!("{} Try setting explicit peers.", explanation),
+            )
+            .links(vec![
+                "https://aptos.dev/nodes/full-node/troubleshooting-fullnode-setup".to_string(),
+            ])
+        }
+    }
+
+    fn default_minimum_inbound() -> u64 {
+        0
+    }
+
+    fn default_minimum_outbound() -> u64 {
+        1
+    }
+}
+
+#[async_trait::async_trait]
+impl Checker for MinimumPeersChecker {
+    async fn check(&self, input: &ProviderCollection) -> Result<Vec<CheckResult>, CheckerError> {
+        let target_metrics_provider = get_provider!(
+            input.target_metrics_provider,
+            self.config.required,
+            MetricsProvider
+        );
+        let scrape = target_metrics_provider.provide().await?;
+        let (inbound_connections, outbound_connections) = match get_metrics(&scrape) {
+            Ok((inbound_connections, outbound_connections)) => {
+                (inbound_connections, outbound_connections)
+            }
+            Err(evaluation_results) => return Ok(evaluation_results),
+        };
+
+        Ok(vec![
+            self.build_evaluation(
+                inbound_connections,
+                self.config.minimum_peers_inbound,
+                &ConnectionType::Inbound,
+            ),
+            self.build_evaluation(
+                outbound_connections,
+                self.config.minimum_peers_outbound,
+                &ConnectionType::Outbound,
+            ),
+        ])
+    }
+}
+
+//////////////////////////////////////////////////////////////////////////////
+// Helpers.
+//////////////////////////////////////////////////////////////////////////////
+
+const METRIC: &str = "aptos_connections";
+
+static INBOUND_LABEL: Lazy<Label> = Lazy::new(|| Label {
+    key: "direction",
+    value: "inbound",
+});
+static OUTBOUND_LABEL: Lazy<Label> = Lazy::new(|| Label {
+    key: "direction",
+    value: "outbound",
+});
+
+enum ConnectionType {
+    Inbound,
+    Outbound,
+}
+
+impl ConnectionType {
+    fn get_name(&self) -> &'static str {
+        match &self {
+            ConnectionType::Inbound => "inbound",
+            ConnectionType::Outbound => "outbound",
+        }
+    }
+
+    fn get_particle(&self) -> &'static str {
+        match &self {
+            ConnectionType::Inbound => "from",
+            ConnectionType::Outbound => "to",
+        }
+    }
+
+    fn get_opposite_particle(&self) -> &'static str {
+        match &self {
+            ConnectionType::Inbound => "to",
+            ConnectionType::Outbound => "from",
+        }
+    }
+}
+
+/// Given a Scrape, pull the metrics telling us the number of inbound and
+/// outbound connections.
+fn get_metrics(metrics: &Scrape) -> Result<(u64, u64), Vec<CheckResult>> {
+    let result_on_missing_fn = || {
+        CheckResult::new(
+            "Missing metric".to_string(),
+            0,
+            format!(
+                "The metrics from the node are missing the metric: {}",
+                METRIC
+            ),
+        )
+    };
+    let (inbound, outbound) = (
+        get_metric(metrics, METRIC, Some(&INBOUND_LABEL), result_on_missing_fn),
+        get_metric(metrics, METRIC, Some(&OUTBOUND_LABEL), result_on_missing_fn),
+    );
+    if let (GetMetricResult::Present(inbound), GetMetricResult::Present(outbound)) =
+        (&inbound, &outbound)
+    {
+        return Ok((*inbound, *outbound));
+    }
+    let mut evaluation_results = vec![];
+    if let GetMetricResult::Missing(evaluation_result) = inbound {
+        evaluation_results.push(evaluation_result);
+    }
+    if let GetMetricResult::Missing(evaluation_result) = outbound {
+        evaluation_results.push(evaluation_result);
+    }
+    Err(evaluation_results)
+}
diff --git a/ecosystem/node-checker/src/checker/mod.rs b/ecosystem/node-checker/src/checker/mod.rs
new file mode 100644
index 0000000000000..56b24ef11f3bb
--- /dev/null
+++ b/ecosystem/node-checker/src/checker/mod.rs
@@ -0,0 +1,40 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+mod minimum_peers;
+mod traits;
+mod types;
+
+use serde::{Deserialize, Serialize};
+pub use traits::{Checker, CheckerError};
+pub use types::{CheckResult, CheckSummary};
+
+use self::minimum_peers::{MinimumPeersChecker, MinimumPeersCheckerConfig};
+
+/// This enum lets us represent all the different Bypassers in a config.
+/// This should only be used at config reading time.
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(tag = "type")]
+pub enum CheckerConfig {
+    MinimumPeers(MinimumPeersCheckerConfig),
+}
+
+impl CheckerConfig {
+    pub fn try_into_boxed_checker(self) -> Result<Box<dyn Checker>, anyhow::Error> {
+        match self {
+            Self::MinimumPeers(config) => Ok(Box::new(MinimumPeersChecker::new(config))),
+        }
+    }
+}
+
+pub fn build_checkers(checkers: &[CheckerConfig]) -> Result<Vec<Box<dyn Checker>>, anyhow::Error> {
+    checkers
+        .iter()
+        .map(|checker| checker.try_into_boxed_checker())
+        .collect()
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct CommonCheckerArgs {
+    pub required: bool,
+}
diff --git a/ecosystem/node-checker/src/checker/traits.rs b/ecosystem/node-checker/src/checker/traits.rs
new file mode 100644
index 0000000000000..5c7926c7edd20
--- /dev/null
+++ b/ecosystem/node-checker/src/checker/traits.rs
@@ -0,0 +1,48 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+use std::fmt::Debug;
+use thiserror::Error;
+
+use crate::provider::{ProviderCollection, ProviderError};
+
+use super::CheckResult;
+
+/// A Checker is a component of NHC that is responsible for checking a
+/// particular aspect of the node under investigation, be that metrics,
+/// system information, API checks, load tests, etc.
+#[async_trait::async_trait]
+pub trait Checker: Debug + Sync + Send {
+    /// This function is expected to take in a ProviderCollection
+    /// and return a vec of evaluation results. It should only return
+    /// errors when there is something wrong with NHC itself or the
+    /// baseline node. If something is unexpected with the target,
+    /// we expect this function to return an EvaluationResult indicating
+    /// as such.
+    async fn check(
+        &self,
+        input: &ProviderCollection,
+    ) -> anyhow::Result<Vec<CheckResult>, CheckerError>;
+}
+
+#[derive(Error, Debug)]
+pub enum CheckerError {
+    #[error("Provider failed to return data: {0:#}")]
+    ProviderError(#[from] ProviderError),
+
+    #[error("Something went wrong hitting endpoint {0}: {1:#}")]
+    RetryableEndpointError(&'static str, #[source] anyhow::Error),
+
+    #[error("Something went wrong hitting endpoint {0}: {1:#}")]
+    NonRetryableEndpointError(&'static str, #[source] anyhow::Error),
+}
+
+impl CheckerError {
+    pub fn is_retryable(&self) -> bool {
+        match self {
+            CheckerError::ProviderError(error) => error.is_retryable(),
+            CheckerError::RetryableEndpointError(_, __) => true,
+            CheckerError::NonRetryableEndpointError(_, _) => false,
+        }
+    }
+}
diff --git a/ecosystem/node-checker/src/checker/types.rs b/ecosystem/node-checker/src/checker/types.rs
new file mode 100644
index 0000000000000..4e4f3c857f6f6
--- /dev/null
+++ b/ecosystem/node-checker/src/checker/types.rs
@@ -0,0 +1,69 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+use poem_openapi::Object;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Deserialize, Object, Serialize)]
+pub struct CheckResult {
+    /// Headline of the result, e.g. "Healthy!" or "Metrics missing!".
+    pub headline: String,
+
+    /// Score out of 100.
+    pub score: u8,
+
+    /// Explanation of the result.
+    pub explanation: String,
+
+    /// Links that might help the user fix a potential problem.
+    pub links: Vec<String>,
+}
+
+impl CheckResult {
+    pub fn new(headline: String, score: u8, explanation: String) -> Self {
+        Self {
+            headline,
+            score,
+            explanation,
+            links: Vec::new(),
+        }
+    }
+
+    pub fn links(mut self, links: Vec<String>) -> Self {
+        self.links = links;
+        self
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Object, Serialize)]
+pub struct CheckSummary {
+    /// Results from all the Checkers NHC ran.
+    pub check_results: Vec<CheckResult>,
+
+    /// An aggeregated summary (method TBA).
+    pub summary_score: u8,
+
+    /// An overall explanation of the results.
+    pub summary_explanation: String,
+}
+
+impl From<Vec<CheckResult>> for CheckSummary {
+    // Very basic for now, we likely want a trait for this.
+    fn from(check_results: Vec<CheckResult>) -> Self {
+        let summary_score = match check_results.len() {
+            0 => 0,
+            len => (check_results.iter().map(|e| e.score as u32).sum::<u32>() / len as u32) as u8,
+        };
+        let summary_explanation = match summary_score {
+            summary_score if summary_score > 95 => format!("{}: Awesome!", summary_score),
+            summary_score if summary_score > 80 => format!("{}: Good!", summary_score),
+            summary_score if summary_score > 50 => format!("{}: Getting there!", summary_score),
+            wildcard => format!("{}: Improvement necessary", wildcard),
+        };
+        CheckSummary {
+            check_results,
+            summary_score,
+            summary_explanation,
+        }
+    }
+}
diff --git a/ecosystem/node-checker/src/common/clients.rs b/ecosystem/node-checker/src/common/clients.rs
new file mode 100644
index 0000000000000..dd37a6590178b
--- /dev/null
+++ b/ecosystem/node-checker/src/common/clients.rs
@@ -0,0 +1,101 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::checker::CheckResult;
+use anyhow::{bail, Context, Result};
+use aptos_crypto::x25519;
+use aptos_rest_client::{aptos_api_types::IndexResponse, Client as AptosRestClient};
+use aptos_sdk::types::network_address::NetworkAddress;
+use reqwest::cookie::Jar;
+use std::sync::Arc;
+use std::time::Duration;
+use url::Url;
+
+pub fn build_url(mut url: Url, path: &str, port: u16) -> Url {
+    url.set_path(path);
+    url.set_port(Some(port)).unwrap();
+    url
+}
+
+pub fn build_client(timeout: Duration, cookie_store: Arc<Jar>) -> reqwest::Client {
+    reqwest::ClientBuilder::new()
+        .timeout(timeout)
+        .cookie_provider(cookie_store.clone())
+        .build()
+        .unwrap()
+}
+
+pub fn build_api_client(
+    base_url: Url,
+    api_port: u16,
+    timeout: Duration,
+    cookie_store: Arc<Jar>,
+) -> AptosRestClient {
+    let client = reqwest::ClientBuilder::new()
+        .timeout(timeout)
+        .cookie_provider(cookie_store.clone())
+        .build()
+        .unwrap();
+
+    AptosRestClient::from((client, build_url(base_url, "", api_port)))
+}
+
+/// Gets the NodeAddress as a NetworkAddress. If the URL is a domain name,
+/// it will automatically perform DNS resolution. This method returns an
+/// error if `public_key` is None.
+pub fn build_noise_network_address(
+    url: Url,
+    noise_port: u16,
+    public_key: x25519::PublicKey,
+) -> Result<NetworkAddress> {
+    // Ensure we can get socket addrs from the URL. If the URL is a domain
+    // name, it will automatically perform DNS resolution.
+    let socket_addrs = url
+        .socket_addrs(|| None)
+        .with_context(|| format!("Failed to get SocketAddrs from address {}", url))?;
+
+    // Ensure this results in exactly one SocketAddr.
+    if socket_addrs.is_empty() {
+        bail!(
+            "NodeAddress {} did not resolve to any SocketAddrs. If DNS, ensure domain name is valid",
+            url
+        );
+    }
+    if socket_addrs.len() > 1 {
+        aptos_logger::warn!(
+            "NodeAddress {} resolved to multiple SocketAddrs, but we're only checking the first one: {:?}",
+            url,
+            socket_addrs,
+        );
+    }
+
+    // Configure the SocketAddr with the provided noise port.
+    let mut socket_addr = socket_addrs[0];
+    socket_addr.set_port(noise_port);
+
+    // Build a network address, including the public key and protocol.
+    Ok(NetworkAddress::from(socket_addr).append_prod_protos(public_key, 0))
+}
+
+pub async fn get_index_response(api_client: AptosRestClient) -> Result<IndexResponse> {
+    Ok(api_client.get_index().await?.into_inner())
+}
+
+/// todo this should probably be a Fetcher instead.
+pub async fn get_index_response_or_evaluation_result(
+    api_client: AptosRestClient,
+    api_port: u16,
+) -> Result<IndexResponse, CheckResult> {
+    match get_index_response(api_client).await {
+        Ok(index_response) => Ok(index_response),
+        Err(error) => Err(CheckResult::new(
+            "Failed to read response from / on API".to_string(),
+            0,
+            format!(
+                "We received an error response hitting / (the index) of the API of \
+                your node, make sure your API port ({}) is publicly accessible: {}.",
+                api_port, error,
+            ),
+        )),
+    }
+}
diff --git a/ecosystem/node-checker/src/common_args.rs b/ecosystem/node-checker/src/common/common_args.rs
similarity index 100%
rename from ecosystem/node-checker/src/common_args.rs
rename to ecosystem/node-checker/src/common/common_args.rs
diff --git a/ecosystem/node-checker/src/common/mod.rs b/ecosystem/node-checker/src/common/mod.rs
new file mode 100644
index 0000000000000..e3035c9178bc2
--- /dev/null
+++ b/ecosystem/node-checker/src/common/mod.rs
@@ -0,0 +1,8 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+mod clients;
+mod common_args;
+
+pub use clients::*;
+pub use common_args::{OutputArgs, OutputFormat};
diff --git a/ecosystem/node-checker/src/configuration/common.rs b/ecosystem/node-checker/src/configuration/common.rs
index bd207e66fc011..def25ed9c8a78 100644
--- a/ecosystem/node-checker/src/configuration/common.rs
+++ b/ecosystem/node-checker/src/configuration/common.rs
@@ -1,8 +1,7 @@
 // Copyright (c) Aptos
 // SPDX-License-Identifier: Apache-2.0
 
-use super::types::NodeConfiguration;
-use crate::evaluators::build_evaluators;
+use super::types::BaselineConfiguration;
 use anyhow::{anyhow, Context, Result};
 use std::{
     convert::{TryFrom, TryInto},
@@ -36,20 +35,20 @@ impl TryFrom<PathBuf> for FileType {
     }
 }
 
-impl TryInto<NodeConfiguration> for FileType {
+impl TryInto<BaselineConfiguration> for FileType {
     type Error = anyhow::Error;
 
-    fn try_into(self) -> Result<NodeConfiguration> {
+    fn try_into(self) -> Result<BaselineConfiguration> {
         match self {
             FileType::Yaml(path) => {
                 let file = File::open(&path)?;
-                let node_configuration: NodeConfiguration = serde_yaml::from_reader(file)
+                let node_configuration: BaselineConfiguration = serde_yaml::from_reader(file)
                     .with_context(|| format!("{} was not valid YAML", path.display()))?;
                 Ok(node_configuration)
             }
             FileType::Json(path) => {
                 let file = File::open(&path)?;
-                let node_configuration: NodeConfiguration = serde_json::from_reader(file)
+                let node_configuration: BaselineConfiguration = serde_json::from_reader(file)
                     .with_context(|| format!("{} was not valid JSON", path.display()))?;
                 Ok(node_configuration)
             }
@@ -57,16 +56,7 @@ impl TryInto<NodeConfiguration> for FileType {
     }
 }
 
-pub fn read_configuration_from_file(path: PathBuf) -> Result<NodeConfiguration> {
+pub fn read_configuration_from_file(path: PathBuf) -> Result<BaselineConfiguration> {
     let file_type = FileType::try_from(path)?;
     file_type.try_into()
 }
-
-pub fn validate_configuration(node_configuration: &NodeConfiguration) -> Result<()> {
-    build_evaluators(
-        &node_configuration.evaluators,
-        &node_configuration.evaluator_args,
-    )
-    .context("Failed to build evaluators")?;
-    Ok(())
-}
diff --git a/ecosystem/node-checker/src/configuration/create.rs b/ecosystem/node-checker/src/configuration/create.rs
deleted file mode 100644
index 20726cc9ec3ea..0000000000000
--- a/ecosystem/node-checker/src/configuration/create.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright (c) Aptos
-// SPDX-License-Identifier: Apache-2.0
-
-use super::{common::validate_configuration, NodeConfiguration};
-use crate::common_args::{OutputArgs, OutputFormat};
-use anyhow::{Context, Result};
-use clap::Parser;
-
-#[derive(Clone, Debug, Parser)]
-pub struct Create {
-    #[clap(flatten)]
-    node_configuration: NodeConfiguration,
-
-    #[clap(flatten)]
-    output_args: OutputArgs,
-
-    // If set, skip config validation. Use with great care.
-    #[clap(long)]
-    skip_validation: bool,
-}
-
-pub async fn create(args: Create) -> Result<()> {
-    if !args.skip_validation {
-        validate_configuration(&args.node_configuration)
-            .context("Configuration failed validation")?;
-    }
-    let output = match args.output_args.format {
-        OutputFormat::Json => serde_json::to_string_pretty(&args.node_configuration)?,
-        OutputFormat::Yaml => {
-            let mut output = format!(
-                "# Base config generated with: {}\n",
-                std::env::args().collect::<Vec<_>>().join(" ")
-            );
-            output.push_str(&serde_yaml::to_string(&args.node_configuration)?);
-            output
-        }
-    };
-    args.output_args.write(&output)
-}
diff --git a/ecosystem/node-checker/src/configuration/mod.rs b/ecosystem/node-checker/src/configuration/mod.rs
index ec3980dc072fe..c4900354df055 100644
--- a/ecosystem/node-checker/src/configuration/mod.rs
+++ b/ecosystem/node-checker/src/configuration/mod.rs
@@ -2,21 +2,16 @@
 // SPDX-License-Identifier: Apache-2.0
 
 mod common;
-mod create;
 mod types;
 mod validate;
 
 use anyhow::Result;
 use clap::{Parser, Subcommand};
 
-use create::{create, Create};
 use validate::{validate, Validate};
 
 pub use common::read_configuration_from_file;
-pub use types::{
-    EvaluatorArgs, NodeAddress, NodeConfiguration, DEFAULT_API_PORT, DEFAULT_API_PORT_STR,
-    DEFAULT_METRICS_PORT, DEFAULT_METRICS_PORT_STR, DEFAULT_NOISE_PORT, DEFAULT_NOISE_PORT_STR,
-};
+pub use types::{BaselineConfiguration, NodeAddress};
 
 #[derive(Clone, Debug, Parser)]
 pub struct Configuration {
@@ -26,16 +21,12 @@ pub struct Configuration {
 
 #[derive(Clone, Debug, Subcommand)]
 enum Command {
-    /// Create a new baseline configuration.
-    Create(Create),
-
-    /// Validate an existing baseline configuration.
+    /// Validate a baseline configuration.
     Validate(Validate),
 }
 
 pub async fn run_cmd(args: Configuration) -> Result<()> {
     let result: Result<()> = match args.cmd {
-        Command::Create(args) => create(args).await,
         Command::Validate(args) => validate(args).await,
     };
     result
diff --git a/ecosystem/node-checker/src/configuration/types.rs b/ecosystem/node-checker/src/configuration/types.rs
index b395cc2f7df3c..0905d219bd3cd 100644
--- a/ecosystem/node-checker/src/configuration/types.rs
+++ b/ecosystem/node-checker/src/configuration/types.rs
@@ -1,274 +1,96 @@
 // Copyright (c) Aptos
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::evaluator::EvaluationResult;
-use crate::{
-    evaluators::{
-        direct::{
-            get_node_identity, HandshakeEvaluatorArgs, LatencyEvaluatorArgs,
-            NodeIdentityEvaluatorArgs, StateSyncVersionEvaluatorArgs, TpsEvaluatorArgs,
-            TransactionAvailabilityEvaluatorArgs,
-        },
-        metrics::{
-            ConsensusProposalsEvaluatorArgs, ConsensusRoundEvaluatorArgs,
-            ConsensusTimeoutsEvaluatorArgs, NetworkMinimumPeersEvaluatorArgs,
-            NetworkPeersWithinToleranceEvaluatorArgs, StateSyncVersionMetricsEvaluatorArgs,
-        },
-        system_information::{BuildVersionEvaluatorArgs, HardwareEvaluatorArgs},
-    },
-    runner::BlockingRunnerArgs,
-};
-use anyhow::{bail, format_err, Context, Result};
-use aptos_config::config::RoleType;
-use aptos_crypto::{x25519, ValidCryptoMaterialStringExt};
+use crate::checker::{CheckResult, CheckerConfig};
+use crate::runner::SyncRunnerConfig;
+use anyhow::{anyhow, bail, Context, Result};
+use aptos_crypto::x25519;
 use aptos_rest_client::{aptos_api_types::IndexResponse, Client as AptosRestClient};
-use aptos_sdk::types::{chain_id::ChainId, network_address::NetworkAddress};
-use clap::Parser;
-use once_cell::sync::Lazy;
-use poem_openapi::{types::Example, Object as PoemObject};
+use aptos_sdk::types::network_address::NetworkAddress;
 use reqwest::cookie::Jar;
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 use std::time::Duration;
 use url::Url;
 
-pub const DEFAULT_METRICS_PORT: u16 = 9101;
-pub const DEFAULT_API_PORT: u16 = 8080;
-pub const DEFAULT_NOISE_PORT: u16 = 6180;
+/// This defines a single baseline configuration.
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct BaselineConfiguration {
+    /// The address of the baseline node to use for this configuration. This is
+    /// only necessary if this baseline configuration uses a Checker that
+    /// requires information from a baseline node to operate.
+    pub baseline_node_address: Option<NodeAddress>,
 
-pub static DEFAULT_METRICS_PORT_STR: Lazy<String> =
-    Lazy::new(|| format!("{}", DEFAULT_METRICS_PORT));
-pub static DEFAULT_API_PORT_STR: Lazy<String> = Lazy::new(|| format!("{}", DEFAULT_API_PORT));
-pub static DEFAULT_NOISE_PORT_STR: Lazy<String> = Lazy::new(|| format!("{}", DEFAULT_NOISE_PORT));
-
-// To briefly explain why many of these structs derive 3 different classes of traits:
-// - Parser (clap): To allow users to generate configs easily using nhc configuration create
-// - Serialize / Deserialize (serde): So we can read / write configs from / to disk
-// - PoemObject: So we can return the configuration over the API
-
-#[derive(Clone, Debug, Deserialize, Parser, PoemObject, Serialize)]
-#[clap(author, version, about, long_about = None)]
-pub struct NodeConfiguration {
-    #[clap(flatten)]
-    pub node_address: NodeAddress,
-
-    /// This is the name we expect clients to send over the wire to select
+    /// This is the ID we expect clients to send over the wire to select
     /// which configuration they want to use. e.g. devnet_fullnode
-    #[clap(long)]
-    pub configuration_name: String,
+    pub configuration_id: String,
 
     /// This is the name we will show for this configuration to users.
     /// For example, if someone opens the NHC frontend, they will see this name
     /// in a dropdown list of configurations they can test their node against.
-    /// e.g. "Devnet FullNode", "Testnet Validator Node", etc.
-    #[clap(long)]
-    pub configuration_name_pretty: String,
-
-    /// The chain ID we expect to find when we speak to the baseline node
-    /// at `node_address`. Regardless of whether this is set, at startup we
-    /// will contact the node to see what its chain ID is. If `chain_id` is
-    /// set here and doesn't match the chain ID returned by the node, we
-    /// will exit, signalling a configuration error.
-    #[clap(long)]
-    #[oai(skip)]
-    chain_id: Option<ChainId>,
-
-    /// This works the same as `chain_id` above, but for role type. Example
-    /// values: "full_node", "validator", etc.
-    #[clap(long)]
-    #[oai(skip)]
-    role_type: Option<RoleType>,
-
-    /// The evaluators to use, e.g. state_sync_version, consensus_proposals, etc.
-    #[clap(long, required = true, min_values = 1, use_value_delimiter = true)]
-    pub evaluators: Vec<String>,
-
-    #[clap(flatten)]
-    pub evaluator_args: EvaluatorArgs,
-
-    #[clap(flatten)]
-    pub runner_args: RunnerArgs,
-}
-
-// TODO: Having comments like "only call this after X" is obviously a bad sign.
-// It'd be better to have an enum with two variants, e.g. unfetched and fetched.
-impl NodeConfiguration {
-    /// Only call this after fetch_additional_configuration has been called.
-    pub fn get_chain_id(&self) -> ChainId {
-        self.chain_id
-            .expect("get_chain_id called before fetch_additional_configuration")
-    }
-
-    /// Only call this after fetch_additional_configuration has been called.
-    pub fn get_role_type(&self) -> RoleType {
-        self.role_type
-            .expect("get_role_type called before fetch_additional_configuration")
-    }
-
-    /// In this function we fetch the chain ID and role type from the node.
-    /// If chain_id and role_type are already set, we validate that the values
-    /// match up. If they're not set, we set them using the values we find.
-    pub async fn fetch_additional_configuration(&mut self) -> Result<()> {
-        let (reported_chain_id, reported_role_type) =
-            get_node_identity(&self.node_address, Duration::from_secs(5))
-                .await
-                .map_err(|e| {
-                    format_err!(
-                    "Failed to fetch chain ID and role type for baseline node configuration: {}",
-                    e
-                )
-                })?;
-        if let Some(configured_chain_id) = self.chain_id {
-            if configured_chain_id != reported_chain_id {
-                bail!("Chain ID mismatch: The baseline configuration {} says the chain ID is {} but the node reports chain ID {}", self.configuration_name, configured_chain_id, reported_chain_id);
-            }
-        }
-        if let Some(configured_role_type) = self.role_type {
-            if configured_role_type != reported_role_type {
-                bail!("Role type mismatch: The baseline configuration {} says the role type is {} but the node reports role type {}", self.configuration_name, configured_role_type, reported_role_type);
-            }
-        }
-        self.chain_id = Some(reported_chain_id);
-        self.role_type = Some(reported_role_type);
-        Ok(())
-    }
-}
-
-#[derive(Clone, Debug, Deserialize, Parser, PoemObject, Serialize)]
-pub struct EvaluatorArgs {
-    #[clap(flatten)]
-    pub build_version_args: BuildVersionEvaluatorArgs,
-
-    #[clap(flatten)]
-    pub consensus_proposals_args: ConsensusProposalsEvaluatorArgs,
-
-    #[clap(flatten)]
-    pub consensus_round_args: ConsensusRoundEvaluatorArgs,
-
-    #[clap(flatten)]
-    pub consensus_timeouts_args: ConsensusTimeoutsEvaluatorArgs,
-
-    #[clap(flatten)]
-    pub handshake_args: HandshakeEvaluatorArgs,
-
-    #[clap(flatten)]
-    pub hardware_args: HardwareEvaluatorArgs,
-
-    #[clap(flatten)]
-    pub latency_args: LatencyEvaluatorArgs,
-
-    #[clap(flatten)]
-    pub network_minimum_peers_args: NetworkMinimumPeersEvaluatorArgs,
-
-    #[clap(flatten)]
-    pub network_peers_tolerance_args: NetworkPeersWithinToleranceEvaluatorArgs,
-
-    #[clap(flatten)]
-    pub node_identity_args: NodeIdentityEvaluatorArgs,
-
-    #[clap(flatten)]
-    pub state_sync_version_args: StateSyncVersionEvaluatorArgs,
-
-    #[clap(flatten)]
-    pub state_sync_version_metrics_args: StateSyncVersionMetricsEvaluatorArgs,
-
-    #[clap(flatten)]
-    #[oai(skip)]
-    pub tps_args: TpsEvaluatorArgs,
+    /// e.g. "Devnet Fullnode", "Testnet Validator", etc.
+    pub configuration_name: String,
 
-    #[clap(flatten)]
-    pub transaction_availability_args: TransactionAvailabilityEvaluatorArgs,
-}
+    /// Config for the runner.
+    pub runner_config: SyncRunnerConfig,
 
-#[derive(Clone, Debug, Deserialize, Parser, PoemObject, Serialize)]
-pub struct RunnerArgs {
-    #[clap(flatten)]
-    pub blocking_runner_args: BlockingRunnerArgs,
+    /// Configs for the checkers to use.
+    pub checkers: Vec<CheckerConfig>,
 }
 
-#[derive(Clone, Debug, Deserialize, Parser, PoemObject, Serialize)]
-#[oai(example)]
+#[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct NodeAddress {
-    /// Target URL. This should include a scheme (e.g. http://). If there is
-    /// no scheme, we will prepend http://.
-    #[clap(long)]
+    /// Target URL. This should include a scheme (e.g. http://). If there is no
+    /// scheme, we will prepend http://.
     pub url: Url,
 
-    /// Metrics port.
-    #[clap(long, default_value = &DEFAULT_METRICS_PORT_STR)]
-    #[oai(default = "Self::default_metrics_port")]
-    #[serde(default = "NodeAddress::default_metrics_port")]
-    metrics_port: u16,
-
     /// API port.
-    #[clap(long, default_value = &DEFAULT_API_PORT_STR)]
-    #[oai(default = "Self::default_api_port")]
-    #[serde(default = "NodeAddress::default_api_port")]
-    api_port: u16,
+    api_port: Option<u16>,
+
+    /// Metrics port.
+    metrics_port: Option<u16>,
 
     /// Validator communication port.
-    #[clap(long, default_value = &DEFAULT_NOISE_PORT_STR)]
-    #[oai(default = "Self::default_noise_port")]
-    #[serde(default = "NodeAddress::default_noise_port")]
-    noise_port: u16,
+    noise_port: Option<u16>,
 
     /// Public key for the node. This is used for the HandshakeEvaluator.
     /// If that evaluator is not enabled, this is not necessary.
-    #[clap(long, value_parser = x25519::PublicKey::from_encoded_string)]
-    #[oai(skip)]
     public_key: Option<x25519::PublicKey>,
 
-    // Cookie store. We don't include this in anything external (clap, the
-    // OpenAPI spec, serde, etc.), this is just for internal use.
-    #[oai(skip)]
-    #[clap(skip)]
+    // Cookie store.
     #[serde(skip)]
     cookie_store: Arc<Jar>,
 }
 
 impl NodeAddress {
-    pub fn new(url: Url) -> Self {
+    pub fn new(
+        url: Url,
+        api_port: Option<u16>,
+        metrics_port: Option<u16>,
+        noise_port: Option<u16>,
+        public_key: Option<x25519::PublicKey>,
+    ) -> Self {
         Self {
             url,
-            metrics_port: Self::default_metrics_port(),
-            api_port: Self::default_api_port(),
-            noise_port: Self::default_noise_port(),
-            public_key: None,
+            api_port,
+            metrics_port,
+            noise_port,
+            public_key,
             cookie_store: Arc::new(Jar::default()),
         }
     }
 
-    pub fn metrics_port(mut self, port: u16) -> Self {
-        self.metrics_port = port;
-        self
-    }
-
-    pub fn api_port(mut self, port: u16) -> Self {
-        self.api_port = port;
-        self
-    }
-
-    pub fn noise_port(mut self, port: u16) -> Self {
-        self.noise_port = port;
-        self
-    }
-
-    pub fn public_key(mut self, public_key: Option<x25519::PublicKey>) -> Self {
-        self.public_key = public_key;
-        self
-    }
-
     /// Do not use this to build a client, use get_metrics_client.
-    pub fn get_metrics_port(&self) -> u16 {
+    pub fn get_metrics_port(&self) -> Option<u16> {
         self.metrics_port
     }
 
     /// Do not use this to build a client, use get_api_client.
-    pub fn get_api_port(&self) -> u16 {
+    pub fn get_api_port(&self) -> Option<u16> {
         self.api_port
     }
 
-    pub fn get_noise_port(&self) -> u16 {
+    pub fn get_noise_port(&self) -> Option<u16> {
         self.noise_port
     }
 
@@ -276,46 +98,48 @@ impl NodeAddress {
         self.public_key
     }
 
-    pub fn default_metrics_port() -> u16 {
-        DEFAULT_METRICS_PORT
-    }
-
-    pub fn default_api_port() -> u16 {
-        DEFAULT_API_PORT
-    }
-
-    pub fn default_noise_port() -> u16 {
-        DEFAULT_NOISE_PORT
-    }
-
-    pub fn get_api_url(&self) -> Url {
+    pub fn get_api_url(&self) -> Result<Url> {
         let mut url = self.url.clone();
-        url.set_port(Some(self.api_port)).unwrap();
-        url
+        url.set_port(Some(
+            self.api_port
+                .context("Can't build API URL without an API port")?,
+        ))
+        .unwrap();
+        Ok(url)
     }
 
-    pub fn get_metrics_url(&self) -> Url {
+    pub fn get_metrics_url(&self, path: &str) -> Result<Url> {
         let mut url = self.url.clone();
-        url.set_port(Some(self.metrics_port)).unwrap();
-        url
-    }
-
-    pub fn get_metrics_client(&self, timeout: Duration) -> reqwest::Client {
-        reqwest::ClientBuilder::new()
-            .timeout(timeout)
-            .cookie_provider(self.cookie_store.clone())
-            .build()
-            .unwrap()
+        url.set_port(Some(
+            self.api_port
+                .context("Can't build metrics URL without a metrics port")?,
+        ))
+        .unwrap();
+        url.set_path(path);
+        Ok(url)
+    }
+
+    pub fn get_metrics_client(&self, timeout: Duration) -> Result<reqwest::Client> {
+        match self.metrics_port {
+            Some(_) => Ok(reqwest::ClientBuilder::new()
+                .timeout(timeout)
+                .cookie_provider(self.cookie_store.clone())
+                .build()
+                .unwrap()),
+            None => Err(anyhow!(
+                "Cannot build metrics client without a metrics port"
+            )),
+        }
     }
 
-    pub fn get_api_client(&self, timeout: Duration) -> AptosRestClient {
+    pub fn get_api_client(&self, timeout: Duration) -> Result<AptosRestClient> {
         let client = reqwest::ClientBuilder::new()
             .timeout(timeout)
             .cookie_provider(self.cookie_store.clone())
             .build()
             .unwrap();
 
-        AptosRestClient::from((client, self.get_api_url()))
+        Ok(AptosRestClient::from((client, self.get_api_url()?)))
     }
 
     /// Gets the NodeAddress as a NetworkAddress. If the URL is a domain name,
@@ -353,52 +177,41 @@ impl NodeAddress {
 
         // Configure the SocketAddr with the provided noise port.
         let mut socket_addr = socket_addrs[0];
-        socket_addr.set_port(self.noise_port);
+        socket_addr.set_port(
+            self.noise_port
+                .context("Can't build NetworkAddress without a noise port")?,
+        );
 
         // Build a network address, including the public key and protocol.
         Ok(NetworkAddress::from(socket_addr).append_prod_protos(public_key, 0))
     }
 
     pub async fn get_index_response(&self, timeout: Duration) -> Result<IndexResponse> {
-        Ok(self.get_api_client(timeout).get_index().await?.into_inner())
+        Ok(self
+            .get_api_client(timeout)?
+            .get_index()
+            .await?
+            .into_inner())
     }
 
+    /// todo this should probably be a Fetcher instead.
     pub async fn get_index_response_or_evaluation_result(
         &self,
         timeout: Duration,
-    ) -> Result<IndexResponse, EvaluationResult> {
+    ) -> Result<IndexResponse, CheckResult> {
         match self.get_index_response(timeout).await {
             Ok(index_response) => Ok(index_response),
-            Err(error) => Err(EvaluationResult {
-                headline: "Failed to read response from / on API".to_string(),
-                score: 0,
-                explanation: format!(
+            Err(error) => Err(CheckResult::new(
+                "Failed to read response from / on API".to_string(),
+                0,
+                format!(
                     "We received an error response hitting / (the index) of the API of \
                     your node, make sure your API port ({}) is publicly accessible: {}.",
-                    self.api_port, error
+                    self.api_port
+                        .expect("Logic error, API port should be Some at this point"),
+                    error
                 ),
-                category: "api".to_string(),
-                evaluator_name: "index_response".to_string(),
-                links: vec![],
-            }),
-        }
-    }
-}
-
-impl Example for NodeAddress {
-    fn example() -> Self {
-        Self {
-            url: Url::parse("http://mynode.mysite.com").unwrap(),
-            metrics_port: Self::default_metrics_port(),
-            api_port: Self::default_api_port(),
-            noise_port: Self::default_noise_port(),
-            public_key: Some(
-                x25519::PublicKey::from_encoded_string(
-                    "0x44fd1324c66371b4788af0b901c9eb8088781acb29e6b8b9c791d5d9838fbe1f",
-                )
-                .unwrap(),
-            ),
-            cookie_store: Arc::new(Jar::default()),
+            )),
         }
     }
 }
diff --git a/ecosystem/node-checker/src/configuration/validate.rs b/ecosystem/node-checker/src/configuration/validate.rs
index c9a544712fd0a..dab4800cd94b7 100644
--- a/ecosystem/node-checker/src/configuration/validate.rs
+++ b/ecosystem/node-checker/src/configuration/validate.rs
@@ -1,7 +1,9 @@
 // Copyright (c) Aptos
 // SPDX-License-Identifier: Apache-2.0
 
-use super::{common::validate_configuration, read_configuration_from_file};
+use crate::checker::build_checkers;
+
+use super::{read_configuration_from_file, BaselineConfiguration};
 use anyhow::{Context, Result};
 use clap::Parser;
 use log::debug;
@@ -19,3 +21,8 @@ pub async fn validate(args: Validate) -> Result<()> {
     debug!("Validated configuration: {:#?}", configuration);
     Ok(())
 }
+
+pub fn validate_configuration(node_configuration: &BaselineConfiguration) -> Result<()> {
+    build_checkers(&node_configuration.checkers).context("Failed to build Checkers")?;
+    Ok(())
+}
diff --git a/ecosystem/node-checker/src/evaluator/traits.rs b/ecosystem/node-checker/src/evaluator/traits.rs
index 7cfbc3967c943..b8ff77165a70f 100644
--- a/ecosystem/node-checker/src/evaluator/traits.rs
+++ b/ecosystem/node-checker/src/evaluator/traits.rs
@@ -1,24 +1,9 @@
 // Copyright (c) Aptos
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::{
-    configuration::{EvaluatorArgs, NodeAddress},
-    evaluator::EvaluationResult,
-    evaluators::EvaluatorType,
-};
-use log::info;
-use std::{collections::HashSet, error::Error, fmt::Debug};
+use std::{error::Error, fmt::Debug};
 
-// You'll notice that a couple of the methods here require Self: Sized.
-// The intent with the Evaluator trait is that a caller will build a vec
-// of different evaluators with different associated input and error types.
-// To do this, we need to construct trait objects (Box<dyn Evaluator<Input, Error>>),
-// and to do that, the trait needs to be object safe. Object safe traits may
-// not have functions or methods that do not take self as a parameter unless
-// they are marked as `where Self: Sized`. This constraint means these functions
-// can not be called on the trait object itself, just on the struct / Self.
-// For more information on this topic, see:
-// https://doc.rust-lang.org/reference/items/traits.html#object-safety
+use super::EvaluationResult;
 
 /// An Evaluator is a component of NHC that is responsible for evaluating
 /// a particular aspect of the node under investigation, be that metrics,
@@ -37,107 +22,7 @@ pub trait Evaluator: Debug + Sync + Send {
     /// as such.
     async fn evaluate(&self, input: &Self::Input) -> Result<Vec<EvaluationResult>, Self::Error>;
 
-    /// All evaluators must have a category. This is used for building
-    /// EvaluationResults.
-    fn get_category_name() -> String
-    where
-        Self: Sized;
+    fn first_round_fetchers(&self) -> Vec<FetcherId>;
 
-    /// All evaluators must have a name. We use this to select evaluators
-    /// when building them from the initial configuration.
-    fn get_evaluator_name() -> String
-    where
-        Self: Sized;
-
-    /// This is the "fully qualified" identifier used for specifying that you
-    /// want to use this evaluator.
-    fn get_identifier() -> String
-    where
-        Self: Sized,
-    {
-        format!(
-            "{}_{}",
-            Self::get_category_name(),
-            Self::get_evaluator_name()
-        )
-    }
-
-    /// Before the evaluation is run, this function is called for all evaluators
-    /// configured for the given baseline configuration. This gives those evaluators
-    /// an opportunity to error out early if a necessary argument is not provided.
-    fn validate_check_node_call(&self, _target_node_address: &NodeAddress) -> anyhow::Result<()> {
-        Ok(())
-    }
-
-    // It would be better to require From<&EvaluatorArgs> on the trait
-    // itself, but that has a few issues. First, it would introduce a
-    // lifetime parameter on the trait, which makes it harder to use.
-    // Second, I haven't found a way to add that supertrait in only the
-    // case where Self: Sized, whereas I can do that with this function.
-
-    /// We require this function to ensure we can build all evaluators
-    /// from the top level evaluator args.
-    fn from_evaluator_args(evaluator_args: &EvaluatorArgs) -> anyhow::Result<Self>
-    where
-        Self: Sized;
-
-    /// This is useful for build_evaluators.
-    fn evaluator_type_from_evaluator_args(
-        evaluator_args: &EvaluatorArgs,
-    ) -> anyhow::Result<EvaluatorType>
-    where
-        Self: Sized;
-
-    /// This is useful for build_evaluators.
-    /// There should be no reason to override the default impl.
-    fn add_from_evaluator_args(
-        evaluators: &mut Vec<EvaluatorType>,
-        evaluator_identifiers: &mut HashSet<String>,
-        evaluator_args: &EvaluatorArgs,
-    ) -> anyhow::Result<()>
-    where
-        Self: Sized,
-    {
-        let identifier = Self::get_identifier();
-        match evaluator_identifiers.take(&identifier) {
-            Some(_) => evaluators.push(Self::evaluator_type_from_evaluator_args(evaluator_args)?),
-            None => info!("Did not build evaluator {}", identifier),
-        };
-        Ok(())
-    }
-
-    // Helper for building EvaluationResults with the name already filled in.
-    fn build_evaluation_result(
-        &self,
-        headline: String,
-        score: u8,
-        explanation: String,
-    ) -> EvaluationResult
-    where
-        Self: Sized,
-    {
-        self.build_evaluation_result_with_links(headline, score, explanation, vec![])
-    }
-
-    // Helper for building EvaluationResults with the name already filled in
-    // and optionally with links.
-    fn build_evaluation_result_with_links(
-        &self,
-        headline: String,
-        score: u8,
-        explanation: String,
-        links: Vec<String>,
-    ) -> EvaluationResult
-    where
-        Self: Sized,
-    {
-        EvaluationResult {
-            headline,
-            score,
-            explanation,
-            category: Self::get_category_name(),
-            evaluator_name: Self::get_evaluator_name(),
-            links,
-        }
-    }
+    fn second_round_fetchers(&self) -> Vec<FetcherId>;
 }
diff --git a/ecosystem/node-checker/src/evaluator/types.rs b/ecosystem/node-checker/src/evaluator/types.rs
index 918cb5373e5b2..10fcf174348f2 100644
--- a/ecosystem/node-checker/src/evaluator/types.rs
+++ b/ecosystem/node-checker/src/evaluator/types.rs
@@ -4,7 +4,6 @@
 use poem_openapi::Object as PoemObject;
 use serde::{Deserialize, Serialize};
 
-// TODO: Should I find a way to have typed actual + expected fields?
 #[derive(Clone, Debug, Deserialize, PoemObject, Serialize)]
 pub struct EvaluationResult {
     /// Headline of the evaluation, e.g. "Healthy!" or "Metrics missing!".
@@ -16,16 +15,35 @@ pub struct EvaluationResult {
     /// Explanation of the evaluation.
     pub explanation: String,
 
-    /// Name of the evaluator where the evaluation came from, e.g. state_sync_version.
-    pub evaluator_name: String,
-
-    /// Category of the evaluator where the evaluation came from, e.g. state_sync.
-    pub category: String,
+    /// Fetchers that were used to produce this evaluation.
+    pub fetchers_used: Vec<FetcherId>,
 
     /// Links that might help the user fix a potential problem.
     pub links: Vec<String>,
 }
 
+impl EvaluationResult {
+    pub fn new(headline: String, score: u8, explanation: String, evaluator_name: String) -> Self {
+        Self {
+            headline,
+            score,
+            explanation,
+            fetchers_used: Vec::new(),
+            links: Vec::new(),
+        }
+    }
+
+    pub fn fetchers(mut self, fetchers: Vec<FetcherId>) -> Self {
+        self.fetchers_used = fetchers;
+        self
+    }
+
+    pub fn links(mut self, links: Vec<String>) -> Self {
+        self.links = links;
+        self
+    }
+}
+
 #[derive(Clone, Debug, Deserialize, PoemObject, Serialize)]
 pub struct EvaluationSummary {
     /// Results from all the evaluations NHC ran.
@@ -55,7 +73,7 @@ impl From<Vec<EvaluationResult>> for EvaluationSummary {
             summary_score if summary_score > 95 => format!("{}: Awesome!", summary_score),
             summary_score if summary_score > 80 => format!("{}: Good!", summary_score),
             summary_score if summary_score > 50 => format!("{}: Getting there!", summary_score),
-            wildcard => format!("{}: Not good enough :(", wildcard),
+            wildcard => format!("{}: Improvement necessary", wildcard),
         };
         EvaluationSummary {
             evaluation_results,
diff --git a/ecosystem/node-checker/src/fetcher/metrics.rs b/ecosystem/node-checker/src/fetcher/metrics.rs
new file mode 100644
index 0000000000000..b59e7c6058db8
--- /dev/null
+++ b/ecosystem/node-checker/src/fetcher/metrics.rs
@@ -0,0 +1,135 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+use super::traits::{Fetcher, FetcherError};
+use crate::{checker::CheckResult, configuration::NodeAddress};
+use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
+use log::warn;
+use prometheus_parse::{Scrape, Value};
+use std::{fmt::Debug, time::Duration};
+
+#[derive(Debug)]
+pub struct MetricsFetcher {
+    node_address: NodeAddress,
+}
+
+impl MetricsFetcher {
+    pub fn new(node_address: NodeAddress) -> Self {
+        Self { node_address }
+    }
+}
+
+#[async_trait]
+impl Fetcher for MetricsFetcher {
+    type Output = Scrape;
+
+    async fn fetch(&self) -> Result<Self::Output, FetcherError> {
+        let url = self.node_address.get_metrics_url("metrics");
+        let response = self
+            .node_address
+            .get_metrics_client(Duration::from_secs(4))
+            .get(url)
+            .send()
+            .await
+            .with_context(|| format!("Failed to get data from {}", url))
+            .map_err(|e| FetcherError::RetryableEndpointError("/metrics", e))?;
+        let body = response
+            .text()
+            .await
+            .with_context(|| format!("Failed to process response body from {} as text", url))
+            .map_err(|e| FetcherError::ParseError(anyhow!(e)))?;
+        Scrape::parse(body.lines().into_iter().map(|l| Ok(l.to_string())))
+            .with_context(|| {
+                format!(
+                    "Failed to parse response text from {} as a Prometheus scrape",
+                    url
+                )
+            })
+            .map_err(|e| FetcherError::ParseError(anyhow!(e)))
+    }
+}
+
+pub struct Label<'a> {
+    pub key: &'a str,
+    pub value: &'a str,
+}
+
+/// This function searches through the given set of metrics and searches for
+/// a metric with the given metric name. If no label was given, we return that
+/// metric immediately. If a label was given, we search for a metric that
+/// has that label.
+fn get_metric_value(
+    metrics: &Scrape,
+    metric_name: &str,
+    expected_label: Option<&Label>,
+) -> Option<u64> {
+    let mut discovered_sample = None;
+    for sample in &metrics.samples {
+        if sample.metric == metric_name {
+            match &expected_label {
+                Some(expected_label) => {
+                    let label_value = sample.labels.get(expected_label.key);
+                    if let Some(label_value) = label_value {
+                        if label_value == expected_label.value {
+                            discovered_sample = Some(sample);
+                            break;
+                        }
+                    }
+                }
+                None => {
+                    discovered_sample = Some(sample);
+                    break;
+                }
+            }
+        }
+    }
+    match discovered_sample {
+        Some(sample) => match &sample.value {
+            Value::Counter(v) => Some(v.round() as u64),
+            Value::Gauge(v) => Some(v.round() as u64),
+            Value::Untyped(v) => Some(v.round() as u64),
+            wildcard => {
+                warn!("Found unexpected metric type: {:?}", wildcard);
+                None
+            }
+        },
+        None => None,
+    }
+}
+
+/// This is a convenience function that returns the metric value if it was
+/// found, or a CheckResult if not.
+pub fn get_metric<F>(
+    metrics: &Scrape,
+    metric_name: &str,
+    expected_label: Option<&Label>,
+    result_on_missing_fn: F,
+) -> GetMetricResult
+where
+    F: FnOnce() -> CheckResult,
+{
+    let metric_value = get_metric_value(metrics, metric_name, expected_label);
+    match metric_value {
+        Some(v) => GetMetricResult::Present(v),
+        None => GetMetricResult::Missing(result_on_missing_fn()),
+    }
+}
+
+#[derive(Debug)]
+pub enum GetMetricResult {
+    Present(u64),
+    Missing(CheckResult),
+}
+
+impl GetMetricResult {
+    pub fn unwrap(self, check_results: &mut Vec<CheckResult>) -> Option<u64> {
+        match self {
+            GetMetricResult::Present(value) => Some(value),
+            GetMetricResult::Missing(check_result) => {
+                check_results.push(check_result);
+                None
+            }
+        }
+    }
+}
diff --git a/ecosystem/node-checker/src/fetcher/mod.rs b/ecosystem/node-checker/src/fetcher/mod.rs
new file mode 100644
index 0000000000000..6b87c33f9c95a
--- /dev/null
+++ b/ecosystem/node-checker/src/fetcher/mod.rs
@@ -0,0 +1,7 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+pub mod metrics;
+mod traits;
+
+pub use traits::Fetcher;
diff --git a/ecosystem/node-checker/src/fetcher/traits.rs b/ecosystem/node-checker/src/fetcher/traits.rs
new file mode 100644
index 0000000000000..de06735155fe2
--- /dev/null
+++ b/ecosystem/node-checker/src/fetcher/traits.rs
@@ -0,0 +1,47 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+use std::fmt::Debug;
+use thiserror::Error;
+
+/// A Fetcher is a component of NHC that is responsible for fetching some kind
+/// of information, be that from the baseline node, target node, request, or
+/// somewhere else. Evaluations can declare what Fetchers they need called in
+/// order to work.
+#[async_trait::async_trait]
+pub trait Fetcher: Debug + Sync + Send {
+    type Output: Debug;
+
+    async fn fetch(&self) -> Result<Self::Output, FetcherError>;
+}
+
+#[derive(Error, Debug)]
+pub enum FetcherError {
+    #[error("Something went wrong hitting endpoint {0}: {1:#}")]
+    RetryableEndpointError(&'static str, #[source] anyhow::Error),
+
+    #[error("Something went wrong hitting endpoint {0}: {1:#}")]
+    NonRetryableEndpointError(&'static str, #[source] anyhow::Error),
+
+    #[error("Something went wrong parsing the response from the node: {0:#}")]
+    ParseError(#[from] anyhow::Error),
+}
+
+impl FetcherError {
+    pub fn is_retryable(&self) -> bool {
+        match self {
+            FetcherError::RetryableEndpointError(_, __) => true,
+            FetcherError::NonRetryableEndpointError(_, _) => false,
+            FetcherError::ParseError(_) => false,
+        }
+    }
+}
+
+// see the comments in synchronous_handler.rs. it is still okay for this trait
+// to exist i think, but if we keep it, make it clear that it mostly exists for
+// auxilliary purposes like helping with retries.
+
+// perhaps it could also help with memoization if the fetch function was defined
+// once here and did the memoization, and then the user just defines some inner
+// fetch function. the Fetcher would likely need to take in some kind of cache
+// manager then as an associated type.
diff --git a/ecosystem/node-checker/src/handler/mod.rs b/ecosystem/node-checker/src/handler/mod.rs
new file mode 100644
index 0000000000000..fa5c1bfa95285
--- /dev/null
+++ b/ecosystem/node-checker/src/handler/mod.rs
@@ -0,0 +1 @@
+mod synchronous_handler;
diff --git a/ecosystem/node-checker/src/handler/synchronous_handler.rs b/ecosystem/node-checker/src/handler/synchronous_handler.rs
new file mode 100644
index 0000000000000..c298ff2082446
--- /dev/null
+++ b/ecosystem/node-checker/src/handler/synchronous_handler.rs
@@ -0,0 +1,9 @@
+use crate::{configuration::{NodeAddress}, checker::Checker};
+
+#[derive(Debug)]
+pub struct SynchronousHandler {
+    baseline_node_address: Option<NodeAddress>,
+    checkers: Vec<Box<dyn Checker>>,
+}
+
+// todo just have each baseline Option<Fetcher> here? vs a map.
diff --git a/ecosystem/node-checker/src/lib.rs b/ecosystem/node-checker/src/lib.rs
index 12a3c811afa62..a7fb0ef2d4e51 100644
--- a/ecosystem/node-checker/src/lib.rs
+++ b/ecosystem/node-checker/src/lib.rs
@@ -1,12 +1,15 @@
 // Copyright (c) Aptos
 // SPDX-License-Identifier: Apache-2.0
 
-mod common_args;
+mod checker;
+mod common;
 pub mod configuration;
-mod evaluator;
-mod evaluators;
-mod metric_collector;
+//mod evaluator;
+//mod evaluators;
+//mod fetcher;
+mod provider;
+//mod metric_collector;
 mod runner;
 pub mod server;
 
-pub use evaluator::EvaluationSummary;
+pub use checker::CheckSummary;
diff --git a/ecosystem/node-checker/src/metric_collector/reqwest_metric_collector.rs b/ecosystem/node-checker/src/metric_collector/reqwest_metric_collector.rs
index b20f2a0aa577d..54a4d513bab7f 100644
--- a/ecosystem/node-checker/src/metric_collector/reqwest_metric_collector.rs
+++ b/ecosystem/node-checker/src/metric_collector/reqwest_metric_collector.rs
@@ -21,8 +21,7 @@ impl ReqwestMetricCollector {
     }
 
     fn get_url(&self, path: &str) -> Url {
-        let mut url = self.node_address.get_metrics_url();
-        url.set_path(path);
+        let mut url = self.node_address.get_metrics_url(path);
         url
     }
 
diff --git a/ecosystem/node-checker/src/provider/helpers.rs b/ecosystem/node-checker/src/provider/helpers.rs
new file mode 100644
index 0000000000000..e995397520c9f
--- /dev/null
+++ b/ecosystem/node-checker/src/provider/helpers.rs
@@ -0,0 +1,43 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+/// This macro helps you turn an Option<P> where P is a Provider or Arc<Provider>
+/// into a &P, returning a CheckResult if the Option is None and required is true,
+/// or just an empty vec if required is false.
+///
+/// Example invocations:
+///
+/// ```
+/// let target_metrics_provider = get_provider!(
+///     input.target_metrics_provider,
+///     true,
+///     MetricsProvider
+/// );
+/// ```
+///
+/// The first argument is the Option, the second is `required`, and the last is
+/// the type of the Provider, which is used for building error messages.
+#[macro_export]
+macro_rules! get_provider {
+    ($provider_option:expr, $required:expr, $provider_type:ty) => {
+        match $provider_option {
+            Some(ref provider) => provider,
+            None => {
+                if $required {
+                    let type_name = std::any::type_name::<$provider_type>();
+                    return Ok(vec![CheckResult::new(
+                        format!("{}: Incomplete request", type_name),
+                        0,
+                        format!(
+                            "Failed to fetch the data for {} because: {}",
+                            type_name,
+                            <$provider_type as crate::provider::Provider>::explanation()
+                        ),
+                    )]);
+                } else {
+                    return Ok(vec![]);
+                }
+            }
+        }
+    };
+}
diff --git a/ecosystem/node-checker/src/provider/metrics.rs b/ecosystem/node-checker/src/provider/metrics.rs
new file mode 100644
index 0000000000000..b9da5fb968dca
--- /dev/null
+++ b/ecosystem/node-checker/src/provider/metrics.rs
@@ -0,0 +1,150 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+use super::traits::{Provider, ProviderError};
+use crate::{checker::CheckResult, configuration::NodeAddress};
+use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
+use log::warn;
+use prometheus_parse::{Scrape, Value};
+use reqwest::Url;
+
+#[derive(Clone, Debug)]
+pub struct MetricsProvider {
+    client: reqwest::Client,
+    /// This has both the path and the port already rolled into it.
+    metrics_url: Url,
+}
+
+impl MetricsProvider {
+    pub fn new(client: reqwest::Client, url: Url, metrics_port: u16) -> Self {
+        let mut metrics_url = url.clone();
+        metrics_url.set_path("metrics");
+        metrics_url.set_port(Some(metrics_port)).unwrap();
+        Self {
+            client,
+            metrics_url,
+        }
+    }
+}
+
+#[async_trait]
+impl Provider for MetricsProvider {
+    type Output = Scrape;
+
+    async fn provide(&self) -> Result<Self::Output, ProviderError> {
+        let response = self
+            .client
+            .get(self.metrics_url)
+            .send()
+            .await
+            .with_context(|| format!("Failed to get data from {}", self.metrics_url))
+            .map_err(|e| ProviderError::RetryableEndpointError("/metrics", e))?;
+        let body = response
+            .text()
+            .await
+            .with_context(|| {
+                format!(
+                    "Failed to process response body from {} as text",
+                    self.metrics_url
+                )
+            })
+            .map_err(|e| ProviderError::ParseError(anyhow!(e)))?;
+        Scrape::parse(body.lines().into_iter().map(|l| Ok(l.to_string())))
+            .with_context(|| {
+                format!(
+                    "Failed to parse response text from {} as a Prometheus scrape",
+                    self.metrics_url
+                )
+            })
+            .map_err(|e| ProviderError::ParseError(anyhow!(e)))
+    }
+
+    fn explanation() -> &'static str {
+        "The metrics port was not provided."
+    }
+}
+
+pub struct Label<'a> {
+    pub key: &'a str,
+    pub value: &'a str,
+}
+
+/// This function searches through the given set of metrics and searches for
+/// a metric with the given metric name. If no label was given, we return that
+/// metric immediately. If a label was given, we search for a metric that
+/// has that label.
+fn get_metric_value(
+    metrics: &Scrape,
+    metric_name: &str,
+    expected_label: Option<&Label>,
+) -> Option<u64> {
+    let mut discovered_sample = None;
+    for sample in &metrics.samples {
+        if sample.metric == metric_name {
+            match &expected_label {
+                Some(expected_label) => {
+                    let label_value = sample.labels.get(expected_label.key);
+                    if let Some(label_value) = label_value {
+                        if label_value == expected_label.value {
+                            discovered_sample = Some(sample);
+                            break;
+                        }
+                    }
+                }
+                None => {
+                    discovered_sample = Some(sample);
+                    break;
+                }
+            }
+        }
+    }
+    match discovered_sample {
+        Some(sample) => match &sample.value {
+            Value::Counter(v) => Some(v.round() as u64),
+            Value::Gauge(v) => Some(v.round() as u64),
+            Value::Untyped(v) => Some(v.round() as u64),
+            wildcard => {
+                warn!("Found unexpected metric type: {:?}", wildcard);
+                None
+            }
+        },
+        None => None,
+    }
+}
+
+/// This is a convenience function that returns the metric value if it was
+/// found, or a CheckResult if not.
+pub fn get_metric<F>(
+    metrics: &Scrape,
+    metric_name: &str,
+    expected_label: Option<&Label>,
+    result_on_missing_fn: F,
+) -> GetMetricResult
+where
+    F: FnOnce() -> CheckResult,
+{
+    let metric_value = get_metric_value(metrics, metric_name, expected_label);
+    match metric_value {
+        Some(v) => GetMetricResult::Present(v),
+        None => GetMetricResult::Missing(result_on_missing_fn()),
+    }
+}
+
+#[derive(Debug)]
+pub enum GetMetricResult {
+    Present(u64),
+    Missing(CheckResult),
+}
+
+impl GetMetricResult {
+    pub fn unwrap(self, check_results: &mut Vec<CheckResult>) -> Option<u64> {
+        match self {
+            GetMetricResult::Present(value) => Some(value),
+            GetMetricResult::Missing(check_result) => {
+                check_results.push(check_result);
+                None
+            }
+        }
+    }
+}
diff --git a/ecosystem/node-checker/src/provider/mod.rs b/ecosystem/node-checker/src/provider/mod.rs
new file mode 100644
index 0000000000000..c477d42199b90
--- /dev/null
+++ b/ecosystem/node-checker/src/provider/mod.rs
@@ -0,0 +1,13 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+//! todo explain that Providers should take in everything they need to operate
+//! in their constructors.
+
+mod helpers;
+pub mod metrics;
+mod provider_collection;
+mod traits;
+
+pub use provider_collection::ProviderCollection;
+pub use traits::{Provider, ProviderError};
diff --git a/ecosystem/node-checker/src/provider/provider_collection.rs b/ecosystem/node-checker/src/provider/provider_collection.rs
new file mode 100644
index 0000000000000..7d7d7cfa65057
--- /dev/null
+++ b/ecosystem/node-checker/src/provider/provider_collection.rs
@@ -0,0 +1,41 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+use std::sync::Arc;
+
+use prometheus_parse::Scrape;
+
+use super::{metrics::MetricsProvider, traits::Provider};
+
+/// This struct is a container for all the different providers that we have.
+/// It is used to provide a single point of access to all Providers that can
+/// then be passed into Checkers.
+///
+/// When a new Provider is added, it must be added to this struct. The same
+/// Provider can be added multiple times for different sources. It is expected
+/// that at the start of handling a request, this struct will be built with
+/// whatever individual Provider instances can be created based on the request.
+///
+/// Alternative designs were considered, such as having each Checker define in
+/// its type signature what Providers it needs, but that introduces complexities
+/// of its own, namely more duplication in runtime checking. See this post for
+/// more info on the topic: https://stackoverflow.com/questions/74723985.
+///
+/// You'll notice that some of these Providers are wrapped in an Arc. We do this
+/// for Providers that could be used between requests, such Providers created for
+/// querying the baseline node. Providers that are only used for a single request
+/// are not wrapped in an Arc since they're only used once.
+#[derive(Clone, Debug)]
+pub struct ProviderCollection {
+    pub baseline_metrics_provider: Option<Arc<MetricsProvider>>,
+    pub target_metrics_provider: Option<MetricsProvider>,
+}
+
+impl ProviderCollection {
+    pub fn new() -> Self {
+        Self {
+            baseline_metrics_provider: None,
+            target_metrics_provider: None,
+        }
+    }
+}
diff --git a/ecosystem/node-checker/src/provider/traits.rs b/ecosystem/node-checker/src/provider/traits.rs
new file mode 100644
index 0000000000000..a5374ebba0100
--- /dev/null
+++ b/ecosystem/node-checker/src/provider/traits.rs
@@ -0,0 +1,46 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+use std::fmt::Debug;
+use thiserror::Error;
+
+/// This traits defines something that is expected to output some kind of data.
+/// Naturally that is pretty vague, and that's by design, this trait isn't meant
+/// to be too constraining. Having the trait is still helpful though for
+/// generalizing things such as memoization and retries.
+#[async_trait::async_trait]
+pub trait Provider: Debug + Sync + Send {
+    type Output: Debug;
+
+    /// This function is expected to return `Output`.
+    async fn provide(&self) -> Result<Self::Output, ProviderError>;
+
+    // This function is expected to return some explanation for why the Provider
+    // couldn't be built. This is only for use in error messages.
+    fn explanation() -> &'static str;
+}
+
+/// For the sake of simplicity, this error type captures all different types
+/// of errors that a Provider can return, in general terms. Really the only
+/// thing that matters is whether the error is retryable or not.
+#[derive(Error, Debug)]
+pub enum ProviderError {
+    #[error("Something went wrong hitting endpoint {0}: {1:#}")]
+    RetryableEndpointError(&'static str, #[source] anyhow::Error),
+
+    #[error("Something went wrong hitting endpoint {0}: {1:#}")]
+    NonRetryableEndpointError(&'static str, #[source] anyhow::Error),
+
+    #[error("Something went wrong parsing the response from the node: {0:#}")]
+    ParseError(#[from] anyhow::Error),
+}
+
+impl ProviderError {
+    pub fn is_retryable(&self) -> bool {
+        match self {
+            ProviderError::RetryableEndpointError(_, __) => true,
+            ProviderError::NonRetryableEndpointError(_, _) => false,
+            ProviderError::ParseError(_) => false,
+        }
+    }
+}
diff --git a/ecosystem/node-checker/src/runner/blocking_runner.rs b/ecosystem/node-checker/src/runner/blocking_runner.rs
deleted file mode 100644
index 640eaeb13c4f8..0000000000000
--- a/ecosystem/node-checker/src/runner/blocking_runner.rs
+++ /dev/null
@@ -1,274 +0,0 @@
-// Copyright (c) Aptos
-// SPDX-License-Identifier: Apache-2.0
-
-use super::{Runner, RunnerError};
-use crate::{
-    configuration::NodeAddress,
-    evaluator::{EvaluationResult, EvaluationSummary, Evaluator},
-    evaluators::{
-        direct::{DirectEvaluatorInput, NodeIdentityEvaluator},
-        metrics::{parse_metrics, MetricsEvaluatorInput},
-        system_information::SystemInformationEvaluatorInput,
-        EvaluatorSet, EvaluatorType,
-    },
-    metric_collector::{MetricCollector, SystemInformation},
-    server::NodeInformation,
-};
-use anyhow::{Context, Result};
-use async_trait::async_trait;
-use clap::Parser;
-use futures::future::{try_join_all, BoxFuture, TryFutureExt};
-use log::{debug, info};
-use poem_openapi::Object as PoemObject;
-use prometheus_parse::Scrape as PrometheusScrape;
-use serde::{Deserialize, Serialize};
-use tokio::{time::Duration, try_join};
-
-#[derive(Clone, Debug, Deserialize, Parser, PoemObject, Serialize)]
-pub struct BlockingRunnerArgs {
-    #[clap(long, default_value = "5")]
-    pub metrics_fetch_delay_secs: u64,
-
-    #[clap(long, default_value_t = 4)]
-    pub api_client_timeout_secs: u64,
-}
-
-#[derive(Debug)]
-pub struct BlockingRunner<M: MetricCollector> {
-    args: BlockingRunnerArgs,
-    baseline_node_information: NodeInformation,
-    baseline_metric_collector: M,
-    node_identity_evaluator: NodeIdentityEvaluator,
-    evaluator_set: EvaluatorSet,
-}
-
-impl<M: MetricCollector> BlockingRunner<M> {
-    pub fn new(
-        args: BlockingRunnerArgs,
-        baseline_node_information: NodeInformation,
-        baseline_metric_collector: M,
-        node_identity_evaluator: NodeIdentityEvaluator,
-        evaluator_set: EvaluatorSet,
-    ) -> Self {
-        Self {
-            args,
-            baseline_node_information,
-            baseline_metric_collector,
-            node_identity_evaluator,
-            evaluator_set,
-        }
-    }
-
-    fn collect_metrics_failed(address: &NodeAddress, error: RunnerError) -> EvaluationResult {
-        EvaluationResult {
-            headline: "Failed to collect metrics from target node".to_string(),
-            score: 0,
-            explanation: format!("Failed to collect metrics from your node, make sure your metrics port ({}) is publicly accessible: {}", address.get_metrics_port(), error),
-            category: "metrics".to_string(),
-            evaluator_name: "metrics_port".to_string(),
-            links: vec![],
-        }
-    }
-
-    async fn collect_metrics<MC: MetricCollector>(
-        metric_collector: &MC,
-    ) -> Result<PrometheusScrape, RunnerError> {
-        let lines = metric_collector.collect_metrics().await?;
-        parse_metrics(lines)
-            .context("Failed to parse metrics response")
-            .map_err(RunnerError::ParseMetricsError)
-    }
-
-    async fn collect_system_information<MC: MetricCollector>(
-        metric_collector: &MC,
-    ) -> Result<SystemInformation, RunnerError> {
-        Ok(metric_collector.collect_system_information().await?)
-    }
-
-    async fn run_metrics_evaluators<T: MetricCollector>(
-        &self,
-        target_metric_collector: &T,
-        target_node_address: &NodeAddress,
-    ) -> Result<Vec<EvaluationResult>, RunnerError> {
-        let evaluators = self.evaluator_set.get_metrics_evaluators();
-
-        if evaluators.is_empty() {
-            return Ok(vec![]);
-        }
-
-        let first_target_metrics = match Self::collect_metrics(target_metric_collector).await {
-            Ok(scrape) => scrape,
-            Err(e) => return Ok(vec![Self::collect_metrics_failed(target_node_address, e)]),
-        };
-        let first_baseline_metrics = Self::collect_metrics(&self.baseline_metric_collector).await?;
-
-        tokio::time::sleep(Duration::from_secs(self.args.metrics_fetch_delay_secs)).await;
-
-        let second_target_metrics = match Self::collect_metrics(target_metric_collector).await {
-            Ok(scrape) => scrape,
-            Err(e) => return Ok(vec![Self::collect_metrics_failed(target_node_address, e)]),
-        };
-        let second_baseline_metrics =
-            Self::collect_metrics(&self.baseline_metric_collector).await?;
-
-        let input = MetricsEvaluatorInput {
-            previous_baseline_metrics: first_baseline_metrics,
-            previous_target_metrics: first_target_metrics,
-            latest_baseline_metrics: second_baseline_metrics,
-            latest_target_metrics: second_target_metrics,
-        };
-
-        let futures: Vec<BoxFuture<_>> = evaluators
-            .iter()
-            .map(|evaluator| evaluator.evaluate(&input))
-            .collect();
-
-        Ok(try_join_all(futures).await?.into_iter().flatten().collect())
-    }
-
-    async fn run_system_information_evaluators<T: MetricCollector>(
-        &self,
-        target_metric_collector: &T,
-        target_node_address: &NodeAddress,
-    ) -> Result<Vec<EvaluationResult>, RunnerError> {
-        let evaluators = self.evaluator_set.get_system_information_evaluators();
-
-        if evaluators.is_empty() {
-            return Ok(vec![]);
-        }
-
-        let target_system_information =
-            match Self::collect_system_information(target_metric_collector).await {
-                Ok(info) => info,
-                Err(e) => return Ok(vec![Self::collect_metrics_failed(target_node_address, e)]),
-            };
-        let baseline_system_information =
-            Self::collect_system_information(&self.baseline_metric_collector).await?;
-
-        let input = SystemInformationEvaluatorInput {
-            baseline_system_information,
-            target_system_information,
-        };
-
-        let futures: Vec<BoxFuture<_>> = evaluators
-            .iter()
-            .map(|evaluator| evaluator.evaluate(&input))
-            .collect();
-
-        Ok(try_join_all(futures).await?.into_iter().flatten().collect())
-    }
-
-    async fn run_direct_evaluators(
-        &self,
-        direct_evaluator_input: &DirectEvaluatorInput,
-    ) -> Result<Vec<EvaluationResult>, RunnerError> {
-        let evaluators = self.evaluator_set.get_direct_evaluators();
-
-        let mut futures: Vec<BoxFuture<_>> = vec![];
-        for evaluator in &evaluators {
-            futures.push(match evaluator {
-                EvaluatorType::Tps(evaluator) => Box::pin(
-                    evaluator
-                        .evaluate(direct_evaluator_input)
-                        .err_into::<RunnerError>(),
-                ),
-                EvaluatorType::Api(evaluator) => Box::pin(
-                    evaluator
-                        .evaluate(direct_evaluator_input)
-                        .err_into::<RunnerError>(),
-                ),
-                EvaluatorType::Noise(evaluator) => Box::pin(
-                    evaluator
-                        .evaluate(direct_evaluator_input)
-                        .err_into::<RunnerError>(),
-                ),
-                _ => continue,
-            });
-        }
-
-        Ok(try_join_all(futures).await?.into_iter().flatten().collect())
-    }
-}
-
-/// This runner doesn't block in the multithreading sense, but from the user
-/// perspective. To run the health check, we pull metrics once, wait, and then
-/// pull the metrics again. It does not support continually running beyond this
-/// point. You can imagine smarter versions of this where you store the last seen
-/// set of metrics, then compare against that, or perhaps even multiple previously
-/// seen sets of metrics and do more complex analysis. Additionally we could leverage
-/// things like long polling +/ sticky routing to make it that the client request
-/// doesn't just hang waiting for the run to complete.
-#[async_trait]
-impl<M: MetricCollector> Runner for BlockingRunner<M> {
-    async fn run<T: MetricCollector>(
-        &self,
-        target_node_address: &NodeAddress,
-        target_metric_collector: &T,
-    ) -> Result<EvaluationSummary, RunnerError> {
-        info!("Running evaluation for {}", target_node_address.url);
-
-        let api_client_timeout = Duration::from_secs(self.args.api_client_timeout_secs);
-
-        let baseline_index_response = self
-            .baseline_node_information
-            .node_address
-            .get_index_response(api_client_timeout)
-            .await
-            .context(format!(
-            "Failed to read index response from baseline node. Make sure its API is open (port {})",
-            self.baseline_node_information.node_address.get_api_port()
-        ))
-            .map_err(RunnerError::BaselineMissingDataError)?;
-
-        let target_index_response = match target_node_address
-            .get_index_response_or_evaluation_result(api_client_timeout)
-            .await
-        {
-            Ok(response) => response,
-            Err(evaluation_result) => return Ok(EvaluationSummary::from(vec![evaluation_result])),
-        };
-
-        let direct_evaluator_input = DirectEvaluatorInput {
-            baseline_node_information: self.baseline_node_information.clone(),
-            target_node_address: target_node_address.clone(),
-            baseline_index_response,
-            target_index_response,
-        };
-
-        debug!("Confirming node identity matches");
-        let node_identity_evaluations = self
-            .node_identity_evaluator
-            .evaluate(&direct_evaluator_input)
-            .await
-            .map_err(RunnerError::NodeIdentityEvaluatorError)?;
-
-        // Exit early if a node identity evaluation returned a non-passing result.
-        for evaluation in &node_identity_evaluations {
-            if evaluation.score != 100 {
-                return Ok(EvaluationSummary::from(node_identity_evaluations));
-            }
-        }
-        let mut evaluation_results = node_identity_evaluations;
-
-        // Run these different classes of evaluator wrappers simultaneously.
-        // By evaluator wrapper I mean, these are functions that collect all
-        // the information necessary, e.g. fetching metrics, and then run all
-        // the evaluators that depend on that information.
-        let (mut metrics_results, mut system_information_results, mut direct_results) = try_join!(
-            self.run_metrics_evaluators(target_metric_collector, target_node_address),
-            self.run_system_information_evaluators(target_metric_collector, target_node_address),
-            self.run_direct_evaluators(&direct_evaluator_input)
-        )?;
-        evaluation_results.append(&mut metrics_results);
-        evaluation_results.append(&mut system_information_results);
-        evaluation_results.append(&mut direct_results);
-
-        let complete_evaluation = EvaluationSummary::from(evaluation_results);
-
-        Ok(complete_evaluation)
-    }
-
-    fn get_evaluator_set(&self) -> &EvaluatorSet {
-        &self.evaluator_set
-    }
-}
diff --git a/ecosystem/node-checker/src/runner/mod.rs b/ecosystem/node-checker/src/runner/mod.rs
index ff37594a25119..05e9c400383d2 100644
--- a/ecosystem/node-checker/src/runner/mod.rs
+++ b/ecosystem/node-checker/src/runner/mod.rs
@@ -1,8 +1,8 @@
 // Copyright (c) Aptos
 // SPDX-License-Identifier: Apache-2.0
 
-mod blocking_runner;
+mod sync_runner;
 mod traits;
 
-pub use blocking_runner::{BlockingRunner, BlockingRunnerArgs};
+pub use sync_runner::{SyncRunner, SyncRunnerConfig};
 pub use traits::{Runner, RunnerError};
diff --git a/ecosystem/node-checker/src/runner/sync_runner.rs b/ecosystem/node-checker/src/runner/sync_runner.rs
new file mode 100644
index 0000000000000..0e2b2345466ba
--- /dev/null
+++ b/ecosystem/node-checker/src/runner/sync_runner.rs
@@ -0,0 +1,122 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+use super::Runner;
+use crate::{
+    checker::{CheckResult, Checker, CheckerError},
+    configuration::NodeAddress,
+    provider::ProviderCollection,
+    CheckSummary,
+};
+use anyhow::Result;
+use async_trait::async_trait;
+use futures::future::try_join_all;
+use log::info;
+use serde::{Deserialize, Serialize};
+use std::time::Duration;
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct SyncRunnerConfig {
+    /// If > 0, Checkers that failed with a retryable error will be retried
+    /// this many times with the configured delay.
+    #[serde(default)]
+    pub num_retries: u8,
+
+    /// If num_retries > 0, this is the delay in seconds between retries.
+    #[serde(default = "SyncRunnerConfig::default_retry_delay_secs")]
+    pub retry_delay_secs: u16,
+}
+
+impl SyncRunnerConfig {
+    fn default_retry_delay_secs() -> u16 {
+        2
+    }
+}
+
+#[derive(Debug)]
+pub struct SyncRunner {
+    /// Any arguments for the runner itself can be configured here.
+    config: SyncRunnerConfig,
+    /// At startup we may have been able to build some Providers already based
+    /// on the information in the BaselineConfiguration. If so, those Providers
+    /// will be in this ProviderCollection.
+    provider_collection: ProviderCollection,
+    /// All the Checkers we built based on the BaselineConfiguration.
+    checkers: Vec<Box<dyn Checker>>,
+}
+
+impl SyncRunner {
+    pub fn new(
+        config: SyncRunnerConfig,
+        provider_collection: ProviderCollection,
+        checkers: Vec<Box<dyn Checker>>,
+    ) -> Self {
+        Self {
+            config,
+            provider_collection,
+            checkers,
+        }
+    }
+}
+
+/// SyncRunner doesn't imply synchronous execution, but rather that it
+/// is synchronous from the user's perspective, vs some kind of continuous
+/// streaming back of (partial) results.
+#[async_trait]
+impl Runner for SyncRunner {
+    async fn run(&self, target_node_address: &NodeAddress) -> Result<CheckSummary> {
+        info!("Running evaluation for {}", target_node_address.url);
+
+        // Here we build a ProviderCollection and try to build every Provider
+        // we can based on the request. We clone the ProviderCollection from
+        // the runner itself to start with, since it might already have some
+        // prebuilt Providers in it (for the baseline node). Cloning this
+        // ProviderCollection the nice property that the Providers within are
+        // wrapped in Arcs, so we're still using the same Provider instances
+        // between requests, allowing us to do some smart memoization.
+        let provider_collection = self.provider_collection.clone();
+
+        // Call each of the Checkers without awaiting them yet.
+        let futures = Vec::new();
+        for checker in &self.checkers {
+            futures.push(self.call_check(&checker, &provider_collection));
+        }
+
+        // Run all the Checkers concurrently and collect their results.
+        let check_results: Vec<CheckResult> =
+            try_join_all(futures).await?.into_iter().flatten().collect();
+
+        let check_summary = CheckSummary::from(check_results);
+
+        Ok(check_summary)
+    }
+}
+
+impl SyncRunner {
+    /// This function handles calling a checker multiple times if it failed with
+    /// a retryable policy and the Runner is configured with a retry policy.
+    async fn call_check(
+        &self,
+        checker: &Box<dyn Checker>,
+        provider_collection: &ProviderCollection,
+    ) -> Result<Vec<CheckResult>, CheckerError> {
+        let mut num_attempts = 0;
+        let check_result = loop {
+            match checker.check(provider_collection).await {
+                Ok(check_result) => break check_result,
+                Err(e) => {
+                    if num_attempts < self.config.num_retries {
+                        num_attempts += 1;
+                        tokio::time::sleep(Duration::from_secs(
+                            self.config.retry_delay_secs.into(),
+                        ))
+                        .await;
+                    } else {
+                        return Err(e);
+                    }
+                }
+            }
+        };
+        Ok(check_result)
+    }
+}
diff --git a/ecosystem/node-checker/src/runner/traits.rs b/ecosystem/node-checker/src/runner/traits.rs
index 6cbd26690db83..113ebdc14a1c4 100644
--- a/ecosystem/node-checker/src/runner/traits.rs
+++ b/ecosystem/node-checker/src/runner/traits.rs
@@ -5,32 +5,17 @@ use anyhow::{Error, Result};
 use async_trait::async_trait;
 use thiserror::Error as ThisError;
 
-use crate::{
-    configuration::NodeAddress,
-    evaluator::EvaluationSummary,
-    evaluators::{
-        direct::{
-            ApiEvaluatorError, NodeIdentityEvaluatorError, NoiseEvaluatorError, TpsEvaluatorError,
-        },
-        metrics::MetricsEvaluatorError,
-        system_information::SystemInformationEvaluatorError,
-        EvaluatorSet,
-    },
-    metric_collector::{MetricCollector, MetricCollectorError},
-};
+use crate::{configuration::NodeAddress, CheckSummary};
 
 #[derive(Debug, ThisError)]
 pub enum RunnerError {
+    /*
     #[error("Failed to evaluate API: {0}")]
     ApiEvaluatorError(#[from] ApiEvaluatorError),
 
     #[error("Data was missing from the baseline node: {0}")]
     BaselineMissingDataError(Error),
 
-    /// We failed to collect metrics for some reason.
-    #[error("Failed to collect metrics: {0}")]
-    MetricCollectorError(#[from] MetricCollectorError),
-
     /// One of the metrics evaluators failed. This is not the same as a poor score from
     /// an evaluator, this is an actual failure in the evaluation process.
     #[error("Failed to evaluate metrics: {0}")]
@@ -57,13 +42,13 @@ pub enum RunnerError {
     /// evaluator, this is an actual failure in the evaluation process.
     #[error("Failed to evaluate TPS: {0}")]
     TpsEvaluatorError(#[from] TpsEvaluatorError),
+    */
 }
 
-/// This trait describes a Runner, something that can take in instances of other
-/// necessary traits, such as a metric collector for the baseline node, and then,
-/// upon a `run` call, drive a node evaluation end to end. This is the top level
-/// entrypoint to the core functionality of NHC, it should be hooked up fairly
-/// directly to the API endpoint handlers.
+/// This trait describes a Runner. The Runner is the top level coordinator
+/// reponsible for driving a check. It is expected to take in the information
+/// of a target node, build whatever Providers it can based on the request,
+/// run the configured Checkers, and finally return a CheckSummary.
 ///
 /// Note on trait bounds:
 ///  - Sync + Send is required because this will be a member of the Api which
@@ -72,11 +57,5 @@ pub enum RunnerError {
 ///    which needs to be 'static.
 #[async_trait]
 pub trait Runner: Sync + Send + 'static {
-    async fn run<M: MetricCollector>(
-        &self,
-        target_node_address: &NodeAddress,
-        target_metric_collector: &M,
-    ) -> Result<EvaluationSummary, RunnerError>;
-
-    fn get_evaluator_set(&self) -> &EvaluatorSet;
+    async fn run(&self, target_node_address: &NodeAddress) -> Result<CheckSummary>;
 }
diff --git a/ecosystem/node-checker/src/server/api.rs b/ecosystem/node-checker/src/server/api.rs
index 6dab0d609e91a..b437c40deb2f9 100644
--- a/ecosystem/node-checker/src/server/api.rs
+++ b/ecosystem/node-checker/src/server/api.rs
@@ -3,67 +3,17 @@
 
 use std::convert::TryInto;
 
-use super::{
-    common::ServerArgs,
-    configurations_manager::{ConfigurationsManager, NodeConfigurationWrapper},
-};
-use crate::{
-    configuration::{NodeAddress, NodeConfiguration},
-    evaluator::EvaluationSummary,
-    metric_collector::{MetricCollector, ReqwestMetricCollector},
-    runner::Runner,
-};
-use anyhow::anyhow;
+use super::{build::BaselineConfigurationRunners, common::ServerArgs};
+use crate::{configuration::NodeAddress, runner::Runner, CheckSummary};
+use anyhow::{anyhow, Context};
 use aptos_crypto::x25519;
 use aptos_crypto::ValidCryptoMaterialStringExt;
-use poem::{http::StatusCode, Error as PoemError, Result as PoemResult};
-use poem_openapi::{param::Query, payload::Json, Object as PoemObject, OpenApi, OpenApiService};
+use poem::http::StatusCode;
+use poem_openapi::{param::Query, payload::Json, Object, OpenApi, OpenApiService};
 use url::Url;
 
-pub struct PreconfiguredNode<M: MetricCollector> {
-    pub node_address: NodeAddress,
-    pub metric_collector: M,
-}
-
-pub struct Api<M: MetricCollector, R: Runner> {
-    pub configurations_manager: ConfigurationsManager<R>,
-    pub preconfigured_test_node: Option<PreconfiguredNode<M>>,
-    pub allow_preconfigured_test_node_only: bool,
-}
-
-impl<M: MetricCollector, R: Runner> Api<M, R> {
-    fn get_baseline_node_configuration(
-        &self,
-        baseline_configuration_name: &Option<String>,
-    ) -> PoemResult<&NodeConfigurationWrapper<R>> {
-        let baseline_configuration_name = match baseline_configuration_name {
-            Some(name) => name,
-            // TODO: Auto detect this based on the target node.
-            None => {
-                return Err(PoemError::from((
-                    StatusCode::BAD_REQUEST,
-                    anyhow!("You must provide a baseline configuration name for now"),
-                )))
-            }
-        };
-        let node_configuration = match self
-            .configurations_manager
-            .configurations
-            .get(baseline_configuration_name)
-        {
-            Some(runner) => runner,
-            None => {
-                return Err(PoemError::from((
-                    StatusCode::BAD_REQUEST,
-                    anyhow!(
-                        "No baseline configuration found with name {}",
-                        baseline_configuration_name
-                    ),
-                )))
-            }
-        };
-        Ok(node_configuration)
-    }
+pub struct Api<R: Runner> {
+    pub baseline_configurations: BaselineConfigurationRunners<R>,
 }
 
 // I choose to keep both methods rather than making these two separate APIs because it'll
@@ -72,31 +22,34 @@ impl<M: MetricCollector, R: Runner> Api<M, R> {
 // doesn't support "flattening" a struct into separate query parameters, so I do that
 // myself. See https://github.com/poem-web/poem/issues/241.
 #[OpenApi]
-impl<M: MetricCollector, R: Runner> Api<M, R> {
+impl<R: Runner> Api<R> {
     /// Check the health of a given target node. You may specify a baseline
     /// node configuration to use for the evaluation. If you don't specify
     /// a baseline node configuration, we will attempt to determine the
     /// appropriate baseline based on your target node.
-    #[oai(path = "/check_node", method = "get")]
-    async fn check_node(
+    #[oai(path = "/check", method = "get")]
+    async fn check(
         &self,
+        /// The name of the baseline node configuration to use for the evaluation, e.g. devnet_fullnode
+        baseline_configuration_name: Query<String>,
         /// The URL of the node to check. e.g. http://44.238.19.217 or http://fullnode.mysite.com
         node_url: Query<Url>,
-        /// The name of the baseline node configuration to use for the evaluation, e.g. devnet_fullnode
-        baseline_configuration_name: Query<Option<String>>,
-        #[oai(default = "NodeAddress::default_metrics_port")] metrics_port: Query<u16>,
-        #[oai(default = "NodeAddress::default_api_port")] api_port: Query<u16>,
-        #[oai(default = "NodeAddress::default_noise_port")] noise_port: Query<u16>,
+        /// If given, we will assume the metrics service is available at the given port.
+        metrics_port: Query<Option<u16>>,
+        /// If given, we will assume the API is available at the given port.
+        api_port: Query<Option<u16>>,
+        /// If given, we will assume that clients can communicate with your node via noise at the given port.
+        noise_port: Query<Option<u16>>,
         /// A public key for the node, e.g. 0x44fd1324c66371b4788af0b901c9eb8088781acb29e6b8b9c791d5d9838fbe1f.
         /// This is only necessary for certain evaluators, e.g. HandshakeEvaluator.
         public_key: Query<Option<String>>,
-    ) -> PoemResult<Json<EvaluationSummary>> {
+    ) -> poem::Result<Json<CheckSummary>> {
         // Ensure the public key, if given, is in a valid format.
         let public_key = match public_key.0 {
             Some(public_key) => match x25519::PublicKey::from_encoded_string(&public_key) {
                 Ok(public_key) => Some(public_key),
                 Err(e) => {
-                    return Err(PoemError::from((
+                    return Err(poem::Error::from((
                         StatusCode::BAD_REQUEST,
                         anyhow!("Invalid public key \"{}\": {:#}", public_key, e),
                     )))
@@ -105,6 +58,16 @@ impl<M: MetricCollector, R: Runner> Api<M, R> {
             None => None,
         };
 
+        let baseline_configuration = self
+            .baseline_configurations
+            .0
+            .get(&baseline_configuration_name.0)
+            .context(format!(
+                "Baseline configuration {} does not exist",
+                baseline_configuration_name.0
+            ))
+            .map_err(|e| poem::Error::from((StatusCode::BAD_REQUEST, e)))?;
+
         // Within a single NHC run we want to use the same client so that cookies
         // can be collected and used. This is important because the nodes we're
         // talking to might be a behind a LB that does cookie based sticky routing.
@@ -112,142 +75,60 @@ impl<M: MetricCollector, R: Runner> Api<M, R> {
         // that the node has transaction version X, but then we fail to retrieve the
         // transaction at the version because the LB routes us to a different node.
         // In this function, which comprises a single NHC run, we build a NodeAddress
-        // for the baseline and target and use that throughout the request. Further
-        // functions deeper down might clone these structs, but that is fine, because
-        // the important part, the CookieStore (Jar) is in an Arc, so each time we
-        // clone the struct we're just cloning the reference to the same jar.
-
-        let target_node_address = NodeAddress::new(node_url.0)
-            .metrics_port(metrics_port.0)
-            .api_port(api_port.0)
-            .noise_port(noise_port.0)
-            .public_key(public_key);
-
-        let baseline_node_configuration =
-            self.get_baseline_node_configuration(&baseline_configuration_name.0)?;
-
-        // Ensure the given arguments are valid for the configured evaluators.
-        for evaluator in &baseline_node_configuration
-            .runner
-            .get_evaluator_set()
-            .evaluators
-        {
-            if let Err(e) = evaluator.validate_check_node_call(&target_node_address) {
-                return Err(PoemError::from((
-                    StatusCode::BAD_REQUEST,
-                    anyhow!("Invalid request: {}", e),
-                )));
-            }
-        }
-
-        if self.allow_preconfigured_test_node_only {
-            return Err(PoemError::from((
-                StatusCode::METHOD_NOT_ALLOWED,
-                anyhow!(
-                "This node health checker is configured to only check its preconfigured test node"),
-            )));
-        }
-
-        let target_metric_collector = ReqwestMetricCollector::new(target_node_address.clone());
-
-        let complete_evaluation_result = baseline_node_configuration
+        // for the target and use that throughout the request. Further functions
+        // deeper down might clone these structs, but that is fine, because the
+        // important part, the CookieStore (Jar) is in an Arc, so each time we clone
+        // the struct we're just cloning the reference to the same jar.
+        let target_node_address = NodeAddress::new(
+            node_url.0,
+            api_port.0,
+            metrics_port.0,
+            noise_port.0,
+            public_key,
+        );
+
+        let complete_evaluation_result = baseline_configuration
             .runner
-            .run(&target_node_address, &target_metric_collector)
+            .run(&target_node_address)
             .await;
 
         match complete_evaluation_result {
             Ok(complete_evaluation) => Ok(Json(complete_evaluation)),
-            Err(e) => Err(PoemError::from((
+            Err(e) => Err(poem::Error::from((
                 StatusCode::INTERNAL_SERVER_ERROR,
                 anyhow!(e),
             ))),
         }
     }
 
-    /// Check the health of the preconfigured node. If none was specified when
-    /// this instance of the node checker was started, this will return an error.
-    /// You may specify a baseline node configuration to use for the evaluation.
-    /// If you don't specify a baseline node configuration, we will attempt to
-    /// determine the appropriate baseline based on your target node.
-    #[oai(path = "/check_preconfigured_node", method = "get")]
-    async fn check_preconfigured_node(
-        &self,
-        baseline_configuration_name: Query<Option<String>>,
-    ) -> PoemResult<Json<EvaluationSummary>> {
-        if self.preconfigured_test_node.is_none() {
-            return Err(PoemError::from((
-                StatusCode::METHOD_NOT_ALLOWED,
-                anyhow!(
-                    "This node health checker has not been set up with a preconfigured test node"
-                ),
-            )));
-        }
-        let preconfigured_test_node = self.preconfigured_test_node.as_ref().unwrap();
-
-        let baseline_node_configuration =
-            self.get_baseline_node_configuration(&baseline_configuration_name)?;
-
-        let complete_evaluation_result = baseline_node_configuration
-            .runner
-            .run(
-                &preconfigured_test_node.node_address,
-                &preconfigured_test_node.metric_collector,
-            )
-            .await;
-
-        match complete_evaluation_result {
-            Ok(complete_evaluation) => Ok(Json(complete_evaluation)),
-            // Consider returning error codes within the response.
-            Err(e) => Err(PoemError::from((
-                StatusCode::INTERNAL_SERVER_ERROR,
-                anyhow!(e),
-            ))),
-        }
-    }
-
-    /// Get the different baseline configurations the instance of NHC is
-    /// configured with. This method is best effort, it is infeasible to
-    /// derive (or even represent) some fields of the spec via OpenAPI,
-    /// so note that some fields will be missing from the response.
-    #[oai(path = "/get_configurations", method = "get")]
-    async fn get_configurations(&self) -> Json<Vec<NodeConfiguration>> {
-        Json(
-            self.configurations_manager
-                .configurations
-                .values()
-                .map(|n| n.node_configuration.clone())
-                .collect(),
-        )
-    }
-
     /// Get just the keys and pretty names for the configurations, meaning
     /// the configuration_name and configuration_name_pretty fields.
     #[oai(path = "/get_configuration_keys", method = "get")]
-    async fn get_configuration_keys(&self) -> Json<Vec<ConfigurationKey>> {
+    async fn get_configuration_keys(&self) -> Json<Vec<ConfigurationDescriptor>> {
         Json(
-            self.configurations_manager
-                .configurations
-                .values()
-                .map(|n| ConfigurationKey {
-                    key: n.node_configuration.configuration_name.clone(),
-                    pretty_name: n.node_configuration.configuration_name_pretty.clone(),
+            self.baseline_configurations
+                .0
+                .iter()
+                .map(|(k, v)| ConfigurationDescriptor {
+                    key: k.clone(),
+                    name: v.configuration.configuration_name.clone(),
                 })
                 .collect(),
         )
     }
 }
 
-#[derive(Clone, Debug, PoemObject)]
-struct ConfigurationKey {
+#[derive(Clone, Debug, Object)]
+struct ConfigurationDescriptor {
     pub key: String,
-    pub pretty_name: String,
+    pub name: String,
 }
 
-pub fn build_openapi_service<M: MetricCollector, R: Runner>(
-    api: Api<M, R>,
+pub fn build_openapi_service<R: Runner>(
+    api: Api<R>,
     server_args: ServerArgs,
-) -> OpenApiService<Api<M, R>, ()> {
-    let version = std::env::var("CARGO_PKG_VERSION").unwrap_or_else(|_| "0.1.0".to_string());
+) -> OpenApiService<Api<R>, ()> {
+    let version = std::env::var("CARGO_PKG_VERSION").unwrap_or_else(|_| "0.2.0".to_string());
     // These should have already been validated at this point, so we panic.
     let url: Url = server_args
         .try_into()
diff --git a/ecosystem/node-checker/src/server/build.rs b/ecosystem/node-checker/src/server/build.rs
new file mode 100644
index 0000000000000..c2d431cec3201
--- /dev/null
+++ b/ecosystem/node-checker/src/server/build.rs
@@ -0,0 +1,96 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+use std::{collections::HashMap, path::PathBuf, sync::Arc};
+
+use crate::{
+    checker::build_checkers,
+    configuration::{read_configuration_from_file, BaselineConfiguration},
+    provider::{metrics::MetricsProvider, Provider, ProviderCollection},
+    runner::{Runner, SyncRunner},
+};
+use anyhow::{Context, Result};
+use log::info;
+
+/// This struct is a wrapper to help with all the different baseline
+/// node configurations. The key here is the configuration_id.
+#[derive(Debug)]
+pub struct BaselineConfigurationRunners<R: Runner>(
+    pub HashMap<String, BaselineConfigurationRunner<R>>,
+);
+
+/// Just a baseline configuration plus the relevant Handler.
+#[derive(Debug)]
+pub struct BaselineConfigurationRunner<R: Runner> {
+    pub configuration: BaselineConfiguration,
+    pub runner: R,
+}
+
+/// Build BaselineConfigurationRunners from the baseline configs.
+pub async fn build_baseline_configuration_runners(
+    baseline_node_config_paths: &[PathBuf],
+) -> Result<BaselineConfigurationRunners<SyncRunner>> {
+    let mut baseline_configuration_runners = HashMap::new();
+    for path in baseline_node_config_paths.iter() {
+        info!("Building baseline configuration from {}", path.display());
+        let cfg = read_configuration_from_file(path.to_path_buf())
+            .with_context(|| format!("Failed to read configuration from {}", path.display()))?;
+        let name = cfg.configuration_name.clone();
+        let bcr = build_baseline_configuration_runner(cfg).await?;
+        info!(
+            "Successfully built baseline configuration from {}: {}",
+            path.display(),
+            name
+        );
+        baseline_configuration_runners.insert(name, bcr);
+    }
+    Ok(BaselineConfigurationRunners(baseline_configuration_runners))
+}
+
+/// Given a baseline configuration, return a BaselineConfigurationRunner, which
+/// is a wrapper around the baseline configuration and its Runner, which contains
+/// all the Checkers, Providers, etc. we built based on the configuration.
+async fn build_baseline_configuration_runner(
+    configuration: BaselineConfiguration,
+) -> Result<BaselineConfigurationRunner<SyncRunner>> {
+    // Build Checkers based on the baseline configuration.
+    let checkers = build_checkers(&configuration.checkers).context("Failed to build Checkers")?;
+
+    // Build Providers that support being built based on the baseline configuration.
+    let mut provider_collection = ProviderCollection::new();
+
+    if let Some(node_address) = &configuration.baseline_node_address {
+        let api_client = node_address
+            .get_api_client(std::time::Duration::from_secs(4))
+            .ok();
+        let metrics_client = node_address
+            .get_metrics_client(std::time::Duration::from_secs(4))
+            .ok();
+
+        // Build a MetricsProvider for the baseline. Confirm it works against
+        // that baseline node (i.e. make sure the metrics endpoint is accessible).
+        if let Some(metrics_client) = metrics_client {
+            let metrics_provider = MetricsProvider::new(
+                metrics_client,
+                node_address.url.clone(),
+                node_address.get_metrics_port().unwrap(),
+            );
+            metrics_provider.provide().await.context(format!(
+                "Failed to build MetricsProvider for baseline configuration {}, ensure the metrics endpoint is accessible",
+                configuration.configuration_id
+            ))?;
+            provider_collection.baseline_metrics_provider = Some(Arc::new(metrics_provider));
+        }
+    }
+
+    let runner = SyncRunner::new(
+        configuration.runner_config.clone(),
+        provider_collection,
+        checkers,
+    );
+
+    Ok(BaselineConfigurationRunner {
+        configuration,
+        runner,
+    })
+}
diff --git a/ecosystem/node-checker/src/server/common.rs b/ecosystem/node-checker/src/server/common.rs
index 24518d7332907..8143e2cfcb5e9 100644
--- a/ecosystem/node-checker/src/server/common.rs
+++ b/ecosystem/node-checker/src/server/common.rs
@@ -18,8 +18,8 @@ pub struct ServerArgs {
     #[clap(long, default_value_t = DEFAULT_LISTEN_PORT)]
     pub listen_port: u16,
 
-    /// What endpoint to run the API on. e.g. setting this to "api" will result
-    /// in you calling an endpoint like http://nhc.mysite.com:20121/api/check_node
+    /// What endpoint to run the API on. e.g. setting this to "api" for example will
+    /// result in you calling an endpoint like http://nhc.mysite.com:20121/api/check_node
     #[clap(long, default_value = "", parse(from_str = parse_api_path))]
     pub api_path: String,
 }
diff --git a/ecosystem/node-checker/src/server/configurations_manager.rs b/ecosystem/node-checker/src/server/configurations_manager.rs
deleted file mode 100644
index 5dd810465be37..0000000000000
--- a/ecosystem/node-checker/src/server/configurations_manager.rs
+++ /dev/null
@@ -1,88 +0,0 @@
-// Copyright (c) Aptos
-// SPDX-License-Identifier: Apache-2.0
-
-use std::{collections::HashMap, path::PathBuf};
-
-use crate::{
-    configuration::{read_configuration_from_file, NodeConfiguration},
-    evaluator::Evaluator,
-    evaluators::{build_evaluators, direct::NodeIdentityEvaluator},
-    metric_collector::ReqwestMetricCollector,
-    runner::{BlockingRunner, Runner},
-};
-use anyhow::{Context, Result};
-
-use super::NodeInformation;
-
-/// This struct is a wrapper to help with all the different baseline
-/// node configurations.
-#[derive(Debug)]
-pub struct ConfigurationsManager<R: Runner> {
-    /// The key here is the configuration_name.
-    pub configurations: HashMap<String, NodeConfigurationWrapper<R>>,
-}
-
-#[derive(Debug)]
-pub struct NodeConfigurationWrapper<R: Runner> {
-    pub node_configuration: NodeConfiguration,
-    pub runner: R,
-}
-
-// In this function we finally build our trait objects with concrete implementations.
-// We've piped trait bounds throughout our code but here we're finally facing the
-// music and actually choosing some concrete types.
-fn build_node_configuration_wrapper_with_blocking_runner(
-    node_configuration: NodeConfiguration,
-) -> Result<NodeConfigurationWrapper<BlockingRunner<ReqwestMetricCollector>>> {
-    let baseline_node_information = NodeInformation {
-        node_address: node_configuration.node_address.clone(),
-        chain_id: node_configuration.get_chain_id(),
-        role_type: node_configuration.get_role_type(),
-    };
-
-    let baseline_metric_collector =
-        ReqwestMetricCollector::new(node_configuration.node_address.clone());
-
-    let evaluator_set = build_evaluators(
-        &node_configuration.evaluators,
-        &node_configuration.evaluator_args,
-    )
-    .context("Failed to build evaluators")?;
-
-    let node_identity_evaluator =
-        NodeIdentityEvaluator::from_evaluator_args(&node_configuration.evaluator_args)?;
-
-    let runner = BlockingRunner::new(
-        node_configuration.runner_args.blocking_runner_args.clone(),
-        baseline_node_information,
-        baseline_metric_collector,
-        node_identity_evaluator,
-        evaluator_set,
-    );
-
-    let wrapper = NodeConfigurationWrapper {
-        node_configuration,
-        runner,
-    };
-
-    Ok(wrapper)
-}
-
-pub async fn build_server_with_blocking_runner(
-    baseline_node_config_paths: &[PathBuf],
-) -> Result<ConfigurationsManager<BlockingRunner<ReqwestMetricCollector>>> {
-    let mut configurations = HashMap::new();
-    for path in baseline_node_config_paths.iter() {
-        let mut cfg = read_configuration_from_file(path.to_path_buf())
-            .with_context(|| format!("Failed to read configuration from {}", path.display()))?;
-        let name = cfg.configuration_name.clone();
-
-        cfg.fetch_additional_configuration()
-            .await
-            .with_context(|| format!("Failed to fetch chain ID and role type for {}", name))?;
-
-        let configuration_wrapper = build_node_configuration_wrapper_with_blocking_runner(cfg)?;
-        configurations.insert(name, configuration_wrapper);
-    }
-    Ok(ConfigurationsManager { configurations })
-}
diff --git a/ecosystem/node-checker/src/server/errors.rs b/ecosystem/node-checker/src/server/errors.rs
new file mode 100644
index 0000000000000..8b137891791fe
--- /dev/null
+++ b/ecosystem/node-checker/src/server/errors.rs
@@ -0,0 +1 @@
+
diff --git a/ecosystem/node-checker/src/server/mod.rs b/ecosystem/node-checker/src/server/mod.rs
index d6514d58eaeb3..fb81e0d03aa39 100644
--- a/ecosystem/node-checker/src/server/mod.rs
+++ b/ecosystem/node-checker/src/server/mod.rs
@@ -2,8 +2,9 @@
 // SPDX-License-Identifier: Apache-2.0
 
 mod api;
+mod build;
 mod common;
-mod configurations_manager;
+mod errors;
 mod generate_openapi;
 mod node_information;
 mod run;
diff --git a/ecosystem/node-checker/src/server/run.rs b/ecosystem/node-checker/src/server/run.rs
index 5d1e6016790a3..5f508d96c0ad9 100644
--- a/ecosystem/node-checker/src/server/run.rs
+++ b/ecosystem/node-checker/src/server/run.rs
@@ -28,7 +28,7 @@ pub struct Run {
     #[clap(flatten)]
     server_args: ServerArgs,
 
-    /// File paths leading to baseline node configurations.
+    /// File paths leading to baseline configurations.
     #[clap(
         long,
         required = true,
@@ -36,36 +36,7 @@ pub struct Run {
         use_value_delimiter = true,
         parse(from_os_str)
     )]
-    pub baseline_node_config_paths: Vec<PathBuf>,
-
-    /// If this is given, the user will be able to call the check_preconfigured_node
-    /// endpoint, which takes no target, instead using this as the target. If
-    /// allow_preconfigured_test_node_only is set, only the check_preconfigured_node
-    /// endpoint will work, the node will not respond to requests for other nodes.
-    #[clap(long)]
-    pub target_node_url: Option<Url>,
-
-    /// The metrics port for the target node.
-    #[clap(long, requires = "target-node-url", default_value = &DEFAULT_METRICS_PORT_STR)]
-    pub target_metrics_port: u16,
-
-    /// The API port for the target node.
-    #[clap(long, requires = "target-node-url", default_value = &DEFAULT_API_PORT_STR)]
-    pub target_api_port: u16,
-
-    /// The port over which validator nodes can talk to the target node.
-    #[clap(long, requires = "target-node-url", default_value = &DEFAULT_NOISE_PORT_STR)]
-    pub target_noise_port: u16,
-
-    /// Public key for the node. This is used for the HandshakeEvaluator.
-    /// If that evaluator is not enabled, this is not necessary.
-    #[clap(long, requires = "target-node-url", value_parser = x25519::PublicKey::from_encoded_string)]
-    pub target_public_key: Option<x25519::PublicKey>,
-
-    /// If a test node is preconfigured, you can set this to prevent the server
-    /// from responding to requests for any node but that one.
-    #[clap(long)]
-    pub allow_preconfigured_test_node_only: bool,
+    pub baseline_config_paths: Vec<PathBuf>,
 }
 
 pub async fn run(args: Run) -> Result<()> {
@@ -79,22 +50,6 @@ pub async fn run(args: Run) -> Result<()> {
         configurations_manager.configurations
     );
 
-    let preconfigured_test_node = match args.target_node_url {
-        Some(ref url) => {
-            let node_address = NodeAddress::new(url.clone())
-                .api_port(args.target_api_port)
-                .metrics_port(args.target_metrics_port)
-                .noise_port(args.target_noise_port)
-                .public_key(args.target_public_key);
-            let metric_collector = ReqwestMetricCollector::new(node_address.clone());
-            Some(PreconfiguredNode {
-                node_address,
-                metric_collector,
-            })
-        }
-        None => None,
-    };
-
     let api = Api {
         configurations_manager,
         preconfigured_test_node,