Support reading query param and header values from a file

README.md

@@ -121,7 +121,8 @@ Run `xh help` for more detailed information.
 - `@` for including files in multipart requests e.g `picture@hello.jpg` or `picture@hello.jpg;type=image/jpeg;filename=goodbye.jpg`.
 - `:` for adding or removing headers e.g `connection:keep-alive` or `connection:`.
 - `;` for including headers with empty values e.g `header-without-value;`.
-- `=@`/`:=@` for setting the request body's JSON or form fields from a file (`=@` for strings and `:=@` for other JSON types).
+
+An `@` prefix can be used to read a value from a file. For example: `x-api-key:@api-key.txt`.
 
 The request body can also be read from standard input, or from a file using `@filename`.
 

doc/xh.1

@@ -1,4 +1,4 @@
-.TH XH 1 2022-12-13 0.17.0 "User Commands"
+.TH XH 1 2022-12-30 0.17.0 "User Commands"
 
 .SH NAME
 xh \- Friendly and fast tool for sending HTTP requests
@@ -50,19 +50,11 @@ key=value
 Add a JSON property (\-\-json) or form field (\-\-form) to
 the request body.
 .TP 4
-key=@filename
-Add a JSON property (\-\-json) or form field (\-\-form) from a
-file to the request body.
-.TP 4
 key:=value
 Add a field with a literal JSON value to the request body.
 
 Example: "numbers:=[1,2,3] enabled:=true"
 .TP 4
-key:=@filename
-Add a field with a literal JSON value from a file to the
-request body.
-.TP 4
 key@filename
 Upload a file (requires \-\-form or \-\-multipart).
 
@@ -85,6 +77,8 @@ Add a header with an empty value.
 .RE
 
 .RS
+An `@` prefix can be used to read a value from a file. For example: `x\-api\-key:@api\-key.txt`.
+
 A backslash can be used to escape special characters, e.g. "weird\\:key=value".
 
 To construct a complex JSON object, the REQUEST_ITEM's key can be set to a JSON path instead of a field name.

src/cli.rs

@@ -347,19 +347,11 @@ Defaults to \"format\" if the NO_COLOR env is set and to \"none\" if stdout is n
     ///         Add a JSON property (--json) or form field (--form) to
     ///         the request body.
     ///
-    ///     key=@filename
-    ///         Add a JSON property (--json) or form field (--form) from a
-    ///         file to the request body.
-    ///
     ///     key:=value
     ///         Add a field with a literal JSON value to the request body.
     ///
     ///         Example: "numbers:=[1,2,3] enabled:=true"
     ///
-    ///     key:=@filename
-    ///         Add a field with a literal JSON value from a file to the
-    ///         request body.
-    ///
     ///     key@filename
     ///         Upload a file (requires --form or --multipart).
     ///
@@ -380,6 +372,8 @@ Defaults to \"format\" if the NO_COLOR env is set and to \"none\" if stdout is n
     ///     header;
     ///         Add a header with an empty value.
     ///
+    /// An `@` prefix can be used to read a value from a file. For example: `x-api-key:@api-key.txt`.
+    ///
     /// A backslash can be used to escape special characters, e.g. "weird\:key=value".
     ///
     /// To construct a complex JSON object, the REQUEST_ITEM's key can be set to a JSON path instead of a field name.
@@ -510,12 +504,7 @@ impl Cli {
 
         cli.process_relations(&matches)?;
 
-        cli.url = construct_url(
-            &raw_url,
-            cli.default_scheme.as_deref(),
-            cli.request_items.query(),
-        )
-        .map_err(|err| {
+        cli.url = construct_url(&raw_url, cli.default_scheme.as_deref()).map_err(|err| {
             app.error(
                 ErrorKind::ValueValidation,
                 format!("Invalid <URL>: {}", err),
@@ -662,13 +651,12 @@ fn parse_method(method: &str) -> Option<Method> {
 fn construct_url(
     url: &str,
     default_scheme: Option<&str>,
-    query: Vec<(&str, &str)>,
 ) -> std::result::Result<Url, url::ParseError> {
     let mut default_scheme = default_scheme.unwrap_or("http://").to_string();
     if !default_scheme.ends_with("://") {
         default_scheme.push_str("://");
     }
-    let mut url: Url = if let Some(url) = url.strip_prefix("://") {
+    let url: Url = if let Some(url) = url.strip_prefix("://") {
         // Allow users to quickly convert a URL copied from a clipboard to xh/HTTPie command
         // by simply adding a space before `://`.
         // Example: https://example.org -> https ://example.org
@@ -680,14 +668,6 @@ fn construct_url(
     } else {
         url.parse()?
     };
-    if !query.is_empty() {
-        // If we run this even without adding pairs it adds a `?`, hence
-        // the .is_empty() check
-        let mut pairs = url.query_pairs_mut();
-        for (name, value) in query {
-            pairs.append_pair(name, value);
-        }
-    }
     Ok(url)
 }
 

src/main.rs

@@ -42,7 +42,7 @@ use crate::middleware::ClientWithMiddleware;
 use crate::printer::Printer;
 use crate::request_items::{Body, FORM_CONTENT_TYPE, JSON_ACCEPT, JSON_CONTENT_TYPE};
 use crate::session::Session;
-use crate::utils::{test_mode, test_pretend_term};
+use crate::utils::{test_mode, test_pretend_term, url_with_query};
 use crate::vendored::reqwest_cookie_store;
 
 #[cfg(not(any(feature = "native-tls", feature = "rustls")))]
@@ -115,6 +115,7 @@ fn run(args: Cli) -> Result<i32> {
     };
 
     let (mut headers, headers_to_unset) = args.request_items.headers()?;
+    let url = url_with_query(args.url, &args.request_items.query()?);
 
     let use_stdin = !(args.ignore_stdin || atty::is(Stream::Stdin) || test_pretend_term());
     let body_type = args.request_items.body_type;
@@ -196,7 +197,7 @@ fn run(args: Cli) -> Result<i32> {
     #[cfg(feature = "native-tls")]
     if args.native_tls {
         client = client.use_native_tls();
-    } else if utils::url_requires_native_tls(&args.url) {
+    } else if utils::url_requires_native_tls(&url) {
         // We should be loud about this to prevent confusion
         warn("rustls does not support HTTPS for IP addresses. native-tls will be enabled. Use --native-tls to silence this warning.");
         client = client.use_native_tls();
@@ -212,7 +213,7 @@ fn run(args: Cli) -> Result<i32> {
     let mut auth = None;
     let mut save_auth_in_session = true;
 
-    if args.url.scheme() == "https" {
+    if url.scheme() == "https" {
         let verify = args.verify.unwrap_or_else(|| {
             // requests library which is used by HTTPie checks for both
             // REQUESTS_CA_BUNDLE and CURL_CA_BUNDLE environment variables.
@@ -321,7 +322,7 @@ fn run(args: Cli) -> Result<i32> {
 
     let mut session = match &args.session {
         Some(name_or_path) => Some(
-            Session::load_session(&args.url, name_or_path.clone(), args.is_session_read_only)
+            Session::load_session(&url, name_or_path.clone(), args.is_session_read_only)
                 .with_context(|| {
                     format!("couldn't load session {:?}", name_or_path.to_string_lossy())
                 })?,
@@ -338,21 +339,21 @@ fn run(args: Cli) -> Result<i32> {
 
         let mut cookie_jar = cookie_jar.lock().unwrap();
         for cookie in s.cookies() {
-            match cookie_jar.insert_raw(&cookie, &args.url) {
+            match cookie_jar.insert_raw(&cookie, &url) {
                 Ok(..) | Err(cookie_store::CookieError::Expired) => {}
                 Err(err) => return Err(err.into()),
             }
         }
         if let Some(cookie) = headers.remove(COOKIE) {
             for cookie in cookie.to_str()?.split(';') {
-                cookie_jar.insert_raw(&cookie.parse()?, &args.url)?;
+                cookie_jar.insert_raw(&cookie.parse()?, &url)?;
             }
         }
     }
 
     let mut request = {
         let mut request_builder = client
-            .request(method, args.url.clone())
+            .request(method, url.clone())
             .header(
                 ACCEPT_ENCODING,
                 HeaderValue::from_static("gzip, deflate, br"),
@@ -428,12 +429,12 @@ fn run(args: Cli) -> Result<i32> {
             auth = Some(Auth::from_str(
                 &auth_from_arg,
                 auth_type,
-                args.url.host_str().unwrap_or("<host>"),
+                url.host_str().unwrap_or("<host>"),
             )?);
         } else if !args.ignore_netrc {
             // I don't know if it's possible for host() to return None
             // But if it does we still want to use the default entry, if there is one
-            let host = args.url.host().unwrap_or(url::Host::Domain(""));
+            let host = url.host().unwrap_or(url::Host::Domain(""));
             if let Some(entry) = netrc::find_entry(host) {
                 auth = Auth::from_netrc(auth_type, entry);
                 save_auth_in_session = false;
@@ -556,7 +557,7 @@ fn run(args: Cli) -> Result<i32> {
                 download_file(
                     response,
                     args.output,
-                    &args.url,
+                    &url,
                     resume,
                     pretty.color(),
                     args.quiet,
@@ -571,7 +572,7 @@ fn run(args: Cli) -> Result<i32> {
         let cookie_jar = cookie_jar.lock().unwrap();
         s.save_cookies(
             cookie_jar
-                .matches(&args.url)
+                .matches(&url)
                 .into_iter()
                 .map(|c| cookie_crate::Cookie::from(c.clone()))
                 .collect(),

src/request_items.rs

@@ -1,4 +1,5 @@
 use std::{
+    borrow::Cow,
     collections::HashSet,
     fs::{self, File},
     io,
@@ -21,8 +22,10 @@ pub const JSON_ACCEPT: &str = "application/json, */*;q=0.5";
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum RequestItem {
     HttpHeader(String, String),
+    HttpHeaderFromFile(String, String),
     HttpHeaderToUnset(String),
     UrlParam(String, String),
+    UrlParamFromFile(String, String),
     DataField {
         key: String,
         raw_key: String,
@@ -47,7 +50,7 @@ impl FromStr for RequestItem {
     type Err = clap::Error;
     fn from_str(request_item: &str) -> clap::Result<RequestItem> {
         const SPECIAL_CHARS: &str = "=@:;\\";
-        const SEPS: &[&str] = &["=@", ":=@", "==", ":=", "=", "@", ":"];
+        const SEPS: &[&str] = &["==@", "=@", ":=@", ":@", "==", ":=", "=", "@", ":"];
 
         fn split(request_item: &str) -> Option<(&str, &'static str, &str)> {
             let mut char_inds = request_item.char_indices();
@@ -108,12 +111,14 @@ impl FromStr for RequestItem {
                 }
                 ":" if value.is_empty() => Ok(RequestItem::HttpHeaderToUnset(key)),
                 ":" => Ok(RequestItem::HttpHeader(key, value)),
+                "==@" => Ok(RequestItem::UrlParamFromFile(key, value)),
                 "=@" => Ok(RequestItem::DataFieldFromFile {
                     key,
                     raw_key,
                     value,
                 }),
                 ":=@" => Ok(RequestItem::JsonFieldFromFile(raw_key, value)),
+                ":@" => Ok(RequestItem::HttpHeaderFromFile(key, value)),
                 _ => unreachable!(),
             }
         } else if let Some(header) = request_item.strip_suffix(';') {
@@ -264,12 +269,20 @@ impl RequestItems {
                     headers_to_unset.remove(&key);
                     headers.append(key, value);
                 }
+                RequestItem::HttpHeaderFromFile(key, value) => {
+                    let key = HeaderName::from_bytes(key.as_bytes())?;
+                    let value = fs::read_to_string(expand_tilde(value))?;
+                    let value = HeaderValue::from_str(value.trim())?;
+                    headers_to_unset.remove(&key);
+                    headers.append(key, value);
+                }
                 RequestItem::HttpHeaderToUnset(key) => {
                     let key = HeaderName::from_bytes(key.as_bytes())?;
                     headers.remove(&key);
                     headers_to_unset.insert(key);
                 }
                 RequestItem::UrlParam(..) => {}
+                RequestItem::UrlParamFromFile(..) => {}
                 RequestItem::DataField { .. } => {}
                 RequestItem::DataFieldFromFile { .. } => {}
                 RequestItem::JsonField(..) => {}
@@ -280,14 +293,17 @@ impl RequestItems {
         Ok((headers, headers_to_unset))
     }
 
-    pub fn query(&self) -> Vec<(&str, &str)> {
-        let mut query = vec![];
+    pub fn query(&self) -> Result<Vec<(&str, Cow<str>)>> {
+        let mut query: Vec<(&str, Cow<str>)> = vec![];
         for item in &self.items {
             if let RequestItem::UrlParam(key, value) = item {
-                query.push((key.as_str(), value.as_str()));
+                query.push((key, Cow::Borrowed(value)));
+            } else if let RequestItem::UrlParamFromFile(key, value) = item {
+                let value = fs::read_to_string(expand_tilde(value))?;
+                query.push((key, Cow::Owned(value)));
             }
         }
-        query
+        Ok(query)
     }
 
     fn body_as_json(self) -> Result<Body> {
@@ -307,8 +323,10 @@ impl RequestItems {
                 }
                 RequestItem::FormFile { .. } => unreachable!(),
                 RequestItem::HttpHeader(..)
+                | RequestItem::HttpHeaderFromFile(..)
                 | RequestItem::HttpHeaderToUnset(..)
-                | RequestItem::UrlParam(..) => continue,
+                | RequestItem::UrlParam(..)
+                | RequestItem::UrlParamFromFile(..) => continue,
             };
             let json_path = nested_json::parse_path(&raw_key)?;
             body = nested_json::insert(body, &json_path, value)
@@ -332,8 +350,10 @@ impl RequestItems {
                 }
                 RequestItem::FormFile { .. } => unreachable!(),
                 RequestItem::HttpHeader(..) => {}
+                RequestItem::HttpHeaderFromFile(..) => {}
                 RequestItem::HttpHeaderToUnset(..) => {}
                 RequestItem::UrlParam(..) => {}
+                RequestItem::UrlParamFromFile(..) => {}
             }
         }
         Ok(Body::Form(text_fields))
@@ -369,8 +389,10 @@ impl RequestItems {
                     form = form.part(key, part);
                 }
                 RequestItem::HttpHeader(..) => {}
+                RequestItem::HttpHeaderFromFile(..) => {}
                 RequestItem::HttpHeaderToUnset(..) => {}
                 RequestItem::UrlParam(..) => {}
+                RequestItem::UrlParamFromFile(..) => {}
             }
         }
         Ok(Body::Multipart(form))
@@ -418,8 +440,10 @@ impl RequestItems {
                     });
                 }
                 RequestItem::HttpHeader(..)
+                | RequestItem::HttpHeaderFromFile(..)
                 | RequestItem::HttpHeaderToUnset(..)
-                | RequestItem::UrlParam(..) => {}
+                | RequestItem::UrlParam(..)
+                | RequestItem::UrlParamFromFile(..) => {}
             }
         }
         let body = body.expect("Should have had at least one file field");
@@ -459,8 +483,10 @@ impl RequestItems {
         for item in &self.items {
             match item {
                 RequestItem::HttpHeader(..)
+                | RequestItem::HttpHeaderFromFile(..)
                 | RequestItem::HttpHeaderToUnset(..)
-                | RequestItem::UrlParam(..) => continue,
+                | RequestItem::UrlParam(..)
+                | RequestItem::UrlParamFromFile(..) => continue,
                 RequestItem::DataField { .. }
                 | RequestItem::DataFieldFromFile { .. }
                 | RequestItem::JsonField(..)
@@ -522,6 +548,11 @@ mod tests {
         );
         // URL param
         assert_eq!(parse("foo==bar"), UrlParam("foo".into(), "bar".into()));
+        // URL param from file
+        assert_eq!(
+            parse("foo==@data.txt"),
+            UrlParamFromFile("foo".into(), "data.txt".into())
+        );
         // Escaped right before separator
         assert_eq!(
             parse(r"foo\==bar"),
@@ -533,6 +564,11 @@ mod tests {
         );
         // Header
         assert_eq!(parse("foo:bar"), HttpHeader("foo".into(), "bar".into()));
+        // Header from file
+        assert_eq!(
+            parse("foo:@data.txt"),
+            HttpHeaderFromFile("foo".into(), "data.txt".into())
+        );
         // JSON field
         assert_eq!(parse("foo:=[1,2]"), JsonField("foo".into(), json!([1, 2])));
         // JSON field from file