Add content proxy (ordinals#3216)

docs/src/guides/testing.md

@@ -48,68 +48,96 @@ Regtest doesn't require downloading the blockchain or indexing ord.
 Example
 -------
 
-Run bitcoind in regtest with:
+Run `bitcoind` in regtest with:
+
 ```
 bitcoind -regtest -txindex
 ```
 
+Run `ord server` in regtest with:
+
+```
+ord --regtest server
+```
+
 Create a wallet in regtest with:
+
 ```
-ord -r wallet create
+ord --regtest wallet create
 ```
 
 Get a regtest receive address with:
+
 ```
-ord -r wallet receive
+ord --regtest wallet receive
 ```
 
 Mine 101 blocks (to unlock the coinbase) with:
+
 ```
 bitcoin-cli -regtest generatetoaddress 101 <receive address>
 ```
 
 Inscribe in regtest with:
+
 ```
-ord -r wallet inscribe --fee-rate 1 --file <file>
+ord --regtest wallet inscribe --fee-rate 1 --file <file>
 ```
 
 Mine the inscription with:
-```
-bitcoin-cli -regtest generatetoaddress 1 <receive address>
-```
 
-View the inscription in the regtest explorer:
 ```
-ord -r server
+bitcoin-cli -regtest generatetoaddress 1 <receive address>
 ```
 
 By default, browsers don't support compression over HTTP. To test compressed
 content over HTTP, use the `--decompress` flag:
+
 ```
-ord -r server --decompress
+ord --regtest server --decompress
 ```
 
 Testing Recursion
 -----------------
 
 When testing out [recursion](../inscriptions/recursion.md), inscribe the
 dependencies first (example with [p5.js](https://p5js.org)):
+
 ```
-ord -r wallet inscribe --fee-rate 1 --file p5.js
+ord --regtest wallet inscribe --fee-rate 1 --file p5.js
 ```
-This should return a `inscription_id` which you can then reference in your
-recursive inscription.
 
-ATTENTION: These ids will be different when inscribing on
-mainnet or signet, so be sure to change those in your recursive inscription for
-each chain.
+This will return the inscription ID of the dependency which you can then
+reference in your inscription.
+
+However, inscription IDs differ between mainnet and test chains, so you must
+change the inscription IDs in your inscription to the mainnet inscription IDs of
+your dependencies before making the final inscription on mainnet.
 
 Then you can inscribe your recursive inscription with:
+
 ```
-ord -r wallet inscribe --fee-rate 1 --file recursive-inscription.html
+ord --regtest wallet inscribe --fee-rate 1 --file recursive-inscription.html
 ```
+
 Finally you will have to mine some blocks and start the server:
+
 ```
 bitcoin-cli generatetoaddress 6 <receive address>
-ord -r server
+```
+
+### Mainnet Dependencies
+
+To avoid having to change dependency inscription IDs to mainnet inscription IDs,
+you may utilize a content proxy when testing. `ord server` accepts a
+`--content-proxy` option, which takes the URL of a another `ord server`
+instance. When making a request to `/content/<INSCRIPTION_ID>` when a content
+proxy is set and the inscription is not found, `ord server` will forward the
+request to the content proxy. This allows you to run a test `ord server`
+instance with a mainnet content proxy. You can then use mainnet inscription IDs
+in your test inscription, which will then return the content of the mainnet
+inscriptions.
+
+```
+ord --regtest server --content-proxy https://ordinals.com
 ```

src/lib.rs

@@ -51,6 +51,7 @@ use {
   lazy_static::lazy_static,
   ordinals::{DeserializeFromStr, Epoch, Height, Rarity, Sat, SatPoint},
   regex::Regex,
+  reqwest::Url,
   serde::{Deserialize, Deserializer, Serialize, Serializer},
   std::{
     cmp::{self, Reverse},

src/server_config.rs

@@ -3,6 +3,7 @@ use super::*;
 #[derive(Default)]
 pub(crate) struct ServerConfig {
   pub(crate) chain: Chain,
+  pub(crate) content_proxy: Option<Url>,
   pub(crate) csp_origin: Option<String>,
   pub(crate) decompress: bool,
   pub(crate) domain: Option<String>,

src/subcommand/server.rs

@@ -135,6 +135,11 @@ pub struct Server {
   pub(crate) redirect_http_to_https: bool,
   #[arg(long, alias = "nosync", help = "Do not update the index.")]
   pub(crate) no_sync: bool,
+  #[arg(
+    long,
+    help = "Proxy `/content/INSCRIPTION_ID` requests to `<CONTENT_PROXY>/content/INSCRIPTION_ID` if the inscription is not present on current chain."
+  )]
+  pub(crate) content_proxy: Option<Url>,
   #[arg(
     long,
     default_value = "5s",
@@ -173,11 +178,12 @@ impl Server {
 
       let server_config = Arc::new(ServerConfig {
         chain: settings.chain(),
+        content_proxy: self.content_proxy.clone(),
         csp_origin: self.csp_origin.clone(),
+        decompress: self.decompress,
         domain: acme_domains.first().cloned(),
         index_sats: index.has_sat_index(),
         json_api_enabled: !self.disable_json_api,
-        decompress: self.decompress,
       });
 
       let router = Router::new()
@@ -1200,6 +1206,32 @@ impl Server {
     Redirect::to("https://docs.ordinals.com/bounty/")
   }
 
+  fn proxy_content(proxy: &Url, inscription_id: InscriptionId) -> ServerResult<Response> {
+    let response = reqwest::blocking::Client::new()
+      .get(format!("{}content/{}", proxy, inscription_id))
+      .send()
+      .map_err(|err| anyhow!(err))?;
+
+    let mut headers = response.headers().clone();
+
+    headers.insert(
+      header::CONTENT_SECURITY_POLICY,
+      HeaderValue::from_str(&format!(
+        "default-src 'self' {proxy} 'unsafe-eval' 'unsafe-inline' data: blob:"
+      ))
+      .map_err(|err| ServerError::Internal(Error::from(err)))?,
+    );
+
+    Ok(
+      (
+        response.status(),
+        headers,
+        response.bytes().map_err(|err| anyhow!(err))?,
+      )
+        .into_response(),
+    )
+  }
+
   async fn content(
     Extension(index): Extension<Arc<Index>>,
     Extension(settings): Extension<Arc<Settings>>,
@@ -1212,9 +1244,16 @@ impl Server {
         return Ok(PreviewUnknownHtml.into_response());
       }
 
-      let mut inscription = index
-        .get_inscription_by_id(inscription_id)?
-        .ok_or_not_found(|| format!("inscription {inscription_id}"))?;
+      let Some(mut inscription) = index.get_inscription_by_id(inscription_id)? else {
+        return if let Some(proxy) = server_config.content_proxy.as_ref() {
+          Self::proxy_content(proxy, inscription_id)
+        } else {
+          Err(ServerError::NotFound(format!(
+            "{} not found",
+            inscription_id
+          )))
+        };
+      };
 
       if let Some(delegate) = inscription.delegate() {
         inscription = index
@@ -1771,6 +1810,11 @@ mod tests {
       self
     }
 
+    fn server_option(mut self, option: &str, value: &str) -> Self {
+      self.server_args.insert(option.into(), Some(value.into()));
+      self
+    }
+
     fn server_flag(mut self, flag: &str) -> Self {
       self.server_args.insert(flag.into(), None);
       self
@@ -5459,6 +5503,40 @@ next
     server.assert_response(format!("/preview/{id}"), StatusCode::OK, "foo");
   }
 
+  #[test]
+  fn proxy() {
+    let server = TestServer::builder().chain(Chain::Regtest).build();
+
+    server.mine_blocks(1);
+
+    let inscription = Inscription {
+      content_type: Some("text/html".into()),
+      body: Some("foo".into()),
+      ..Default::default()
+    };
+
+    let txid = server.bitcoin_rpc_server.broadcast_tx(TransactionTemplate {
+      inputs: &[(1, 0, 0, inscription.to_witness())],
+      ..Default::default()
+    });
+
+    server.mine_blocks(1);
+
+    let id = InscriptionId { txid, index: 0 };
+
+    server.assert_response(format!("/content/{id}"), StatusCode::OK, "foo");
+
+    let server_with_proxy = TestServer::builder()
+      .chain(Chain::Regtest)
+      .server_option("--content-proxy", server.url.as_ref())
+      .build();
+
+    server_with_proxy.mine_blocks(1);
+
+    server.assert_response(format!("/content/{id}"), StatusCode::OK, "foo");
+    server_with_proxy.assert_response(format!("/content/{id}"), StatusCode::OK, "foo");
+  }
+
   #[test]
   fn chainwork_conversion_to_integer() {
     assert_eq!(chainwork(&[]), 0);