Merge pull request #1652 from eqlabs/chris/api-key

feat(gateway-client): add api key support

CHANGELOG.md

@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- `gateway-api-key API_KEY` configuration option. If enabled, each time a request is sent to the Starknet gateway or the feeder gateway a `X-Throttling-Bypass: API_KEY` header will be set.
+
 ## [0.10.3-rc0] - 2023-12-14
 
 ### Changed

Cargo.toml

@@ -56,6 +56,7 @@ flate2 = "1.0.27"
 futures = { version = "0.3", default-features = false, features = ["std"] }
 hex = "0.4.3"
 http = "0.2.9"
+httpmock = "0.7.0-rc.1"
 lazy_static = "1.4.0"
 metrics = "0.20.1"
 num-bigint = { version = "0.4.4", features = ["serde"] }

crates/ethereum/Cargo.toml

@@ -23,5 +23,5 @@ tokio = { workspace = true }
 tracing = { workspace = true }
 
 [dev-dependencies]
-httpmock = "0.7.0-rc.1"
+httpmock = { workspace = true }
 tokio = { workspace = true, features = ["macros"] }

crates/gateway-client/Cargo.toml

@@ -32,7 +32,9 @@ warp = { version = "0.3.5" }
 [dev-dependencies]
 assert_matches = { workspace = true }
 base64 = { workspace = true }
+fake = { workspace = true }
 flate2 = { workspace = true }
+httpmock = { workspace = true }
 lazy_static = { workspace = true }
 pathfinder-crypto = { path = "../crypto" }
 pretty_assertions_sorted = { workspace = true }

crates/gateway-client/src/builder.rs

@@ -13,10 +13,13 @@ use crate::metrics::{with_metrics, BlockTag, RequestMetadata};
 use pathfinder_common::{BlockId, ClassHash, TransactionHash};
 use starknet_gateway_types::error::SequencerError;
 
+const X_THROTTLING_BYPASS: &str = "X-Throttling-Bypass";
+
 /// A Sequencer Request builder.
 pub struct Request<'a, S: RequestState> {
     state: S,
     url: reqwest::Url,
+    api_key: Option<String>,
     client: &'a reqwest::Client,
 }
 
@@ -64,10 +67,15 @@ pub mod stage {
 
 impl<'a> Request<'a, stage::Init> {
     /// Initialize a [Request] builder.
-    pub fn builder(client: &'a reqwest::Client, url: reqwest::Url) -> Request<'a, stage::Method> {
+    pub fn builder(
+        client: &'a reqwest::Client,
+        url: reqwest::Url,
+        api_key: Option<String>,
+    ) -> Request<'a, stage::Method> {
         Request {
             url,
             client,
+            api_key,
             state: stage::Method,
         }
     }
@@ -144,6 +152,7 @@ impl<'a> Request<'a, stage::Method> {
         Request {
             url: self.url,
             client: self.client,
+            api_key: self.api_key,
             state: stage::Params {
                 meta: RequestMetadata::new(method),
             },
@@ -201,6 +210,7 @@ impl<'a> Request<'a, stage::Params> {
         Request {
             url: self.url,
             client: self.client,
+            api_key: self.api_key,
             state: stage::Final {
                 meta: self.state.meta,
                 retry,
@@ -217,24 +227,31 @@ impl<'a> Request<'a, stage::Final> {
     {
         async fn send_request<T: serde::de::DeserializeOwned>(
             url: reqwest::Url,
+            api_key: Option<String>,
             client: &reqwest::Client,
             meta: RequestMetadata,
         ) -> Result<T, SequencerError> {
             with_metrics(meta, async move {
                 tracing::trace!(%url, "Fetching data from feeder gateway");
-                let response = client.get(url).send().await?;
+                let request = client.get(url);
+                let request = match api_key {
+                    Some(api_key) => request.header(X_THROTTLING_BYPASS, api_key),
+                    None => request,
+                };
+                let response = request.send().await?;
                 parse::<T>(response).await
             })
             .await
         }
 
         match self.state.retry {
-            false => send_request(self.url, self.client, self.state.meta).await,
+            false => send_request(self.url, self.api_key, self.client, self.state.meta).await,
             true => {
                 retry0(
                     || async {
-                        let clone_url = self.url.clone();
-                        send_request(clone_url, self.client, self.state.meta).await
+                        let url = self.url.clone();
+                        let api_key = self.api_key.clone();
+                        send_request(url, api_key, self.client, self.state.meta).await
                     },
                     retry_condition,
                 )
@@ -247,12 +264,18 @@ impl<'a> Request<'a, stage::Final> {
     pub async fn get_as_bytes(self) -> Result<bytes::Bytes, SequencerError> {
         async fn get_as_bytes_inner(
             url: reqwest::Url,
+            api_key: Option<String>,
             client: &reqwest::Client,
             meta: RequestMetadata,
         ) -> Result<bytes::Bytes, SequencerError> {
             with_metrics(meta, async {
                 tracing::trace!(%url, "Fetching binary data from feeder gateway");
-                let response = client.get(url).send().await?;
+                let request = client.get(url);
+                let request = match api_key {
+                    Some(api_key) => request.header(X_THROTTLING_BYPASS, api_key),
+                    None => request,
+                };
+                let response = request.send().await?;
                 let response = parse_raw(response).await?;
                 let bytes = response.bytes().await?;
                 Ok(bytes)
@@ -261,12 +284,13 @@ impl<'a> Request<'a, stage::Final> {
         }
 
         match self.state.retry {
-            false => get_as_bytes_inner(self.url, self.client, self.state.meta).await,
+            false => get_as_bytes_inner(self.url, self.api_key, self.client, self.state.meta).await,
             true => {
                 retry0(
                     || async {
-                        let clone_url = self.url.clone();
-                        get_as_bytes_inner(clone_url, self.client, self.state.meta).await
+                        let url = self.url.clone();
+                        let api_key = self.api_key.clone();
+                        get_as_bytes_inner(url, api_key, self.client, self.state.meta).await
                     },
                     retry_condition,
                 )
@@ -284,6 +308,7 @@ impl<'a> Request<'a, stage::Final> {
     {
         async fn post_with_json_inner<T, J>(
             url: reqwest::Url,
+            api_key: Option<String>,
             client: &reqwest::Client,
             meta: RequestMetadata,
             json: &J,
@@ -293,20 +318,29 @@ impl<'a> Request<'a, stage::Final> {
             J: serde::Serialize + ?Sized,
         {
             with_metrics(meta, async {
-                let response = client.post(url).json(json).send().await?;
+                let request = client.post(url);
+                let request = match api_key {
+                    Some(api_key) => request.header(X_THROTTLING_BYPASS, api_key),
+                    None => request,
+                };
+                let response = request.json(json).send().await?;
                 parse::<T>(response).await
             })
             .await
         }
 
         match self.state.retry {
-            false => post_with_json_inner(self.url, self.client, self.state.meta, json).await,
+            false => {
+                post_with_json_inner(self.url, self.api_key, self.client, self.state.meta, json)
+                    .await
+            }
             true => {
                 retry0(
                     || async {
                         tracing::trace!(url=%self.url, "Posting data to gateway");
-                        let clone_url = self.url.clone();
-                        post_with_json_inner(clone_url, self.client, self.state.meta, json).await
+                        let url = self.url.clone();
+                        let api_key = self.api_key.clone();
+                        post_with_json_inner(url, api_key, self.client, self.state.meta, json).await
                     },
                     retry_condition,
                 )
@@ -596,4 +630,104 @@ mod tests {
             );
         }
     }
+
+    mod api_key_is_set_when_configured {
+        use crate::Client;
+        use fake::{Fake, Faker};
+        use httpmock::{prelude::*, Mock};
+        use serde_json::json;
+
+        async fn setup_with_fake_api_key(server: &MockServer) -> (Mock<'_>, Client) {
+            let api_key = Faker.fake::<String>();
+
+            let mock = server.mock(|when, then| {
+                when.any_request().header("X-Throttling-Bypass", &api_key);
+                then.status(200).json_body(json!({}));
+            });
+
+            let client = Client::with_base_url(server.base_url().parse().unwrap())
+                .unwrap()
+                .with_api_key(Some(api_key.clone()));
+
+            (mock, client)
+        }
+
+        #[tokio::test]
+        async fn get() -> anyhow::Result<()> {
+            let server = MockServer::start_async().await;
+            let (mock, client) = setup_with_fake_api_key(&server).await;
+
+            let _: serde_json::Value = client
+                .clone()
+                .gateway_request()
+                .with_method("")
+                .with_retry(false)
+                .get()
+                .await?;
+
+            let _: serde_json::Value = client
+                .clone()
+                .feeder_gateway_request()
+                .with_method("")
+                .with_retry(false)
+                .get()
+                .await?;
+
+            mock.assert_hits(2);
+
+            Ok(())
+        }
+
+        #[tokio::test]
+        async fn get_as_bytes() -> anyhow::Result<()> {
+            let server = MockServer::start_async().await;
+            let (mock, client) = setup_with_fake_api_key(&server).await;
+
+            let _: bytes::Bytes = client
+                .clone()
+                .gateway_request()
+                .with_method("")
+                .with_retry(false)
+                .get_as_bytes()
+                .await?;
+
+            let _: bytes::Bytes = client
+                .clone()
+                .feeder_gateway_request()
+                .with_method("")
+                .with_retry(false)
+                .get_as_bytes()
+                .await?;
+
+            mock.assert_hits(2);
+
+            Ok(())
+        }
+
+        #[tokio::test]
+        async fn post_with_json() -> anyhow::Result<()> {
+            let server = MockServer::start_async().await;
+            let (mock, client) = setup_with_fake_api_key(&server).await;
+
+            let _: serde_json::Value = client
+                .clone()
+                .gateway_request()
+                .with_method("")
+                .with_retry(false)
+                .post_with_json(&json!({}))
+                .await?;
+
+            let _: serde_json::Value = client
+                .clone()
+                .feeder_gateway_request()
+                .with_method("")
+                .with_retry(false)
+                .post_with_json(&json!({}))
+                .await?;
+
+            mock.assert_hits(2);
+
+            Ok(())
+        }
+    }
 }

crates/gateway-client/src/lib.rs

@@ -240,6 +240,8 @@ pub struct Client {
     /// Whether __read only__ requests should be retried, defaults to __true__ for production.
     /// Use [disable_retry_for_tests](Client::disable_retry_for_tests) to disable retry logic for all __read only__ requests when testing.
     retry: bool,
+    /// Api key added to each request as a value for 'X-Throttling-Bypass' header.
+    api_key: Option<String>,
 }
 
 impl Client {
@@ -289,9 +291,16 @@ impl Client {
             gateway,
             feeder_gateway,
             retry: true,
+            api_key: None,
         })
     }
 
+    /// Sets the api key to be used for each request as a value for 'X-Throttling-Bypass' header.
+    pub fn with_api_key(mut self, api_key: Option<String>) -> Self {
+        self.api_key = api_key;
+        self
+    }
+
     /// Use this method to disable retry logic for all __non write__ requests when testing.
     pub fn disable_retry_for_tests(self) -> Self {
         Self {
@@ -301,11 +310,15 @@ impl Client {
     }
 
     fn gateway_request(&self) -> builder::Request<'_, builder::stage::Method> {
-        builder::Request::builder(&self.inner, self.gateway.clone())
+        builder::Request::builder(&self.inner, self.gateway.clone(), self.api_key.clone())
     }
 
     fn feeder_gateway_request(&self) -> builder::Request<'_, builder::stage::Method> {
-        builder::Request::builder(&self.inner, self.feeder_gateway.clone())
+        builder::Request::builder(
+            &self.inner,
+            self.feeder_gateway.clone(),
+            self.api_key.clone(),
+        )
     }
 
     async fn block_with_retry_behaviour(

crates/gateway-types/Cargo.toml

@@ -9,7 +9,7 @@ rust-version = { workspace = true }
 
 [dependencies]
 anyhow = { workspace = true }
-fake = { workspace = true }
+fake = { workspace = true, features = ["serde_json"] }
 lazy_static = { workspace = true }
 pathfinder-common = { path = "../common" }
 pathfinder-crypto = { path = "../crypto" }

crates/pathfinder/src/bin/pathfinder/config.rs

@@ -197,6 +197,14 @@ This should only be enabled for debugging purposes as it adds substantial proces
         action=ArgAction::Set
     )]
     is_rpc_enabled: bool,
+
+    #[arg(
+        long = "gateway-api-key",
+        value_name = "API_KEY",
+        long_help = "Specify an API key for both the Starknet feeder gateway and gateway.",
+        env = "PATHFINDER_GATEWAY_API_KEY"
+    )]
+    gateway_api_key: Option<String>,
 }
 
 #[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq)]
@@ -438,6 +446,7 @@ pub struct Config {
     pub rpc_batch_concurrency_limit: NonZeroUsize,
     pub is_sync_enabled: bool,
     pub is_rpc_enabled: bool,
+    pub gateway_api_key: Option<String>,
 }
 
 pub struct Ethereum {
@@ -613,6 +622,7 @@ impl Config {
             rpc_batch_concurrency_limit: cli.rpc_batch_concurrency_limit,
             is_sync_enabled: cli.is_sync_enabled,
             is_rpc_enabled: cli.is_rpc_enabled,
+            gateway_api_key: cli.gateway_api_key,
         }
     }
 }