From 3b0aaae9165344451f0b79af699934047b060554 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Mon, 29 May 2023 15:24:27 +0100 Subject: [PATCH 01/28] support persisting cookies from multiple domains --- src/session.rs | 109 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 80 insertions(+), 29 deletions(-) diff --git a/src/session.rs b/src/session.rs index 72ec6b18..fe3f4660 100644 --- a/src/session.rs +++ b/src/session.rs @@ -45,8 +45,20 @@ struct Auth { } // Unlike xh, HTTPie serializes path, secure and expires with defaults of "/", false, and null respectively. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +struct LegacyCookie { + value: String, + #[serde(skip_serializing_if = "Option::is_none")] + expires: Option, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + secure: Option, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] struct Cookie { + name: String, value: String, #[serde(skip_serializing_if = "Option::is_none")] expires: Option, @@ -54,6 +66,22 @@ struct Cookie { path: Option, #[serde(skip_serializing_if = "Option::is_none")] secure: Option, + // TODO: store cookie domain as well +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(untagged)] +enum Cookies { + // old cookie format kept for backward compatibility + Map(HashMap), + // new cookie format that closely resembles a cookie jar + List(Vec), +} + +impl Default for Cookies { + fn default() -> Self { + Cookies::List(Vec::new()) + } } #[derive(Debug, Serialize, Deserialize)] @@ -82,7 +110,7 @@ struct Content { #[serde(rename = "__meta__")] meta: Meta, auth: Auth, - cookies: HashMap, + cookies: Cookies, headers: Headers, } @@ -97,6 +125,20 @@ impl Content { .collect(), ); } + if let Cookies::Map(cookies) = self.cookies { + self.cookies = Cookies::List( + cookies + .into_iter() + .map(|(name, legacy_cookie)| Cookie { + name, + value: legacy_cookie.value, + expires: legacy_cookie.expires, + path: legacy_cookie.path, + secure: legacy_cookie.secure, + }) + .collect(), + ) + } self } @@ -218,39 +260,48 @@ impl Session { } pub fn cookies(&self) -> Vec { - let mut cookies = vec![]; - for (name, c) in &self.content.cookies { - let mut cookie_builder = cookie_crate::Cookie::build(name, &c.value); - if let Some(expires) = c.expires { - cookie_builder = - cookie_builder.expires(time::OffsetDateTime::from_unix_timestamp(expires)); - } - if let Some(ref path) = c.path { - cookie_builder = cookie_builder.path(path); - } - if let Some(secure) = c.secure { - cookie_builder = cookie_builder.secure(secure); - } - cookies.push(cookie_builder.finish()); + match &self.content.cookies { + Cookies::Map(_) => unreachable!(), + Cookies::List(cookies) => cookies + .iter() + .map(|cookie| { + let mut cookie_builder = + cookie_crate::Cookie::build(cookie.name.clone(), cookie.value.clone()); + if let Some(expires) = cookie.expires { + cookie_builder = cookie_builder + .expires(time::OffsetDateTime::from_unix_timestamp(expires)); + } + if let Some(path) = &cookie.path { + cookie_builder = cookie_builder.path(path.clone()); + } + if let Some(secure) = cookie.secure { + cookie_builder = cookie_builder.secure(secure); + } + cookie_builder.finish() + }) + .collect(), } - cookies } pub fn save_cookies(&mut self, cookies: Vec) { - self.content.cookies.clear(); + let session_cookies = match self.content.cookies { + Cookies::Map(_) => unreachable!(), + Cookies::List(ref mut cookies) => cookies, + }; + + session_cookies.clear(); + for cookie in cookies { - self.content.cookies.insert( - cookie.name().into(), - Cookie { - value: cookie.value().into(), - expires: cookie - .expires() - .and_then(|v| v.datetime()) - .map(|v| v.unix_timestamp()), - path: cookie.path().map(Into::into), - secure: cookie.secure(), - }, - ); + session_cookies.push(Cookie { + name: cookie.name().into(), + value: cookie.value().into(), + expires: cookie + .expires() + .and_then(|v| v.datetime()) + .map(|v| v.unix_timestamp()), + path: cookie.path().map(Into::into), + secure: cookie.secure(), + }); } } From 903c9d6425ead07027bb40b6303518313cbcd793 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sat, 10 Jun 2023 13:59:41 +0100 Subject: [PATCH 02/28] store cookie domain in session --- src/main.rs | 4 +++- src/session.rs | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 110bf4e8..3a31a602 100644 --- a/src/main.rs +++ b/src/main.rs @@ -333,7 +333,9 @@ fn run(args: Cli) -> Result { let mut cookie_jar = cookie_jar.lock().unwrap(); for cookie in s.cookies() { match cookie_jar.insert_raw(&cookie, &url) { - Ok(..) | Err(cookie_store::CookieError::Expired) => {} + Ok(..) + | Err(cookie_store::CookieError::Expired) + | Err(cookie_store::CookieError::DomainMismatch) => {} Err(err) => return Err(err.into()), } } diff --git a/src/session.rs b/src/session.rs index fe3f4660..8b9477c1 100644 --- a/src/session.rs +++ b/src/session.rs @@ -66,7 +66,8 @@ struct Cookie { path: Option, #[serde(skip_serializing_if = "Option::is_none")] secure: Option, - // TODO: store cookie domain as well + #[serde(skip_serializing_if = "Option::is_none")] + domain: Option, } #[derive(Debug, Serialize, Deserialize)] @@ -135,6 +136,7 @@ impl Content { expires: legacy_cookie.expires, path: legacy_cookie.path, secure: legacy_cookie.secure, + domain: None, }) .collect(), ) @@ -277,6 +279,9 @@ impl Session { if let Some(secure) = cookie.secure { cookie_builder = cookie_builder.secure(secure); } + if let Some(domain) = &cookie.domain { + cookie_builder = cookie_builder.domain(domain.clone()) + } cookie_builder.finish() }) .collect(), @@ -301,6 +306,7 @@ impl Session { .map(|v| v.unix_timestamp()), path: cookie.path().map(Into::into), secure: cookie.secure(), + domain: cookie.domain().map(Into::into), }); } } From 7ad0b54d6c2ecd729d337dc206895d882707597f Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sat, 10 Jun 2023 14:00:04 +0100 Subject: [PATCH 03/28] test new style cookie parsing --- src/session.rs | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/session.rs b/src/session.rs index 8b9477c1..38297aaa 100644 --- a/src/session.rs +++ b/src/session.rs @@ -472,4 +472,51 @@ mod tests { Ok(()) } + + #[test] + fn can_parse_session_with_new_style_cookies() -> Result<()> { + let session = load_session_from_str(indoc::indoc! {r#" + { + "__meta__": { + "about": "HTTPie session file", + "help": "https://httpie.io/docs#sessions", + "httpie": "3.0.2" + }, + "auth": {}, + "cookies": [ + { + "name": "baz", + "value": "quux", + "expires": null, + "path": "/", + "secure": false, + "domain": "example.com" + }, + { + "name": "foo", + "value": "bar", + "expires": null, + "path": "/", + "secure": false, + "domain": null + } + ], + "headers": [] + } + "#})?; + + let cookies = session.cookies(); + + assert_eq!(cookies[0].name_value(), ("baz", "quux")); + assert_eq!(cookies[0].path(), Some("/")); + assert_eq!(cookies[0].secure(), Some(false)); + assert_eq!(cookies[0].domain(), Some("example.com")); + + assert_eq!(cookies[1].name_value(), ("foo", "bar")); + assert_eq!(cookies[1].path(), Some("/")); + assert_eq!(cookies[1].secure(), Some(false)); + assert_eq!(cookies[1].domain(), None); + + Ok(()) + } } From 156cf0365227952ef49c0f8fdf2a4d29e958a0bd Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sat, 10 Jun 2023 14:31:37 +0100 Subject: [PATCH 04/28] fix existing tests --- src/session.rs | 4 ++- tests/cli.rs | 84 ++++++++++++++++++++++++++++---------------------- 2 files changed, 50 insertions(+), 38 deletions(-) diff --git a/src/session.rs b/src/session.rs index 38297aaa..54f9309a 100644 --- a/src/session.rs +++ b/src/session.rs @@ -288,7 +288,7 @@ impl Session { } } - pub fn save_cookies(&mut self, cookies: Vec) { + pub fn save_cookies(&mut self, mut cookies: Vec) { let session_cookies = match self.content.cookies { Cookies::Map(_) => unreachable!(), Cookies::List(ref mut cookies) => cookies, @@ -296,6 +296,8 @@ impl Session { session_cookies.clear(); + cookies.sort_by(|a, b| a.name().cmp(b.name())); + for cookie in cookies { session_cookies.push(Cookie { name: cookie.name().into(), diff --git a/tests/cli.rs b/tests/cli.rs index f459a209..c20709a7 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -2018,10 +2018,10 @@ fn named_sessions() { "xh": "0.0.0" }, "auth": { "type": "bearer", "raw_auth": "hello" }, - "cookies": { - "cook1": { "value": "one", "path": "/" }, - "lang": { "value": "en" } - }, + "cookies": [ + { "name": "cook1", "value": "one", "path": "/" }, + { "name": "lang", "value": "en" } + ], "headers": [] }) ); @@ -2060,7 +2060,9 @@ fn anonymous_sessions() { "xh": "0.0.0" }, "auth": { "type": "basic", "raw_auth": "me:pass" }, - "cookies": { "cook1": { "value": "one" } }, + "cookies": [ + { "name": "cook1", "value": "one" } + ], "headers": [ { "name": "hello", "value": "world" } ] @@ -2081,7 +2083,9 @@ fn anonymous_read_only_session() { let old_session_content = serde_json::json!({ "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, - "cookies": { "cookie1": { "value": "one" } }, + "cookies": [ + { "name": "cookie1", "value": "one" } + ], "headers": [ { "name": "hello", "value": "world" } ] @@ -2141,9 +2145,9 @@ fn session_files_are_created_in_read_only_mode() { "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, - "cookies": { - "lang": { "value": "ar" } - }, + "cookies": [ + { "name": "lang", "value": "ar" } + ], "headers": [ { "name": "hello", "value": "world" } ] @@ -2175,9 +2179,9 @@ fn named_read_only_session() { let old_session_content = serde_json::json!({ "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, - "cookies": { - "cookie1": { "value": "one" } - }, + "cookies": [ + { "name": "cookie1", "value": "one" } + ], "headers": [ { "name": "hello", "value": "world" } ] @@ -2218,19 +2222,22 @@ fn expired_cookies_are_removed_from_session() { serde_json::json!({ "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, - "cookies": { - "expired_cookie": { + "cookies": [ + { + "name": "expired_cookie", "value": "random_string", "expires": past_timestamp }, - "unexpired_cookie": { + { + "name": "unexpired_cookie", "value": "random_string", "expires": future_timestamp }, - "with_out_expiry": { + { + "name": "with_out_expiry", "value": "random_string", } - }, + ], "headers": [] }) .to_string(), @@ -2253,15 +2260,17 @@ fn expired_cookies_are_removed_from_session() { serde_json::json!({ "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, - "cookies": { - "unexpired_cookie": { + "cookies": [ + { + "name": "unexpired_cookie", "value": "random_string", "expires": future_timestamp }, - "with_out_expiry": { - "value": "random_string", + { + "name": "with_out_expiry", + "value": "random_string" } - }, + ], "headers": [] }) ); @@ -2295,10 +2304,10 @@ fn cookies_override_each_other_in_the_correct_order() { serde_json::json!({ "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, - "cookies": { - "lang": { "value": "fr" }, - "cook2": { "value": "three" } - }, + "cookies": [ + { "name": "lang", "value": "fr" }, + { "name": "cook2", "value": "three" } + ], "headers": [] }) .to_string(), @@ -2324,11 +2333,11 @@ fn cookies_override_each_other_in_the_correct_order() { serde_json::json!({ "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, - "cookies": { - "lang": { "value": "en" }, - "cook1": { "value": "one" }, - "cook2": { "value": "two" } - }, + "cookies": [ + { "name": "cook1", "value": "one" }, + { "name": "cook2", "value": "two" }, + { "name": "lang", "value": "en" } + ], "headers": [] }) ); @@ -2348,7 +2357,7 @@ fn basic_auth_from_session_is_used() { serde_json::json!({ "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": "basic", "raw_auth": "user:pass" }, - "cookies": {}, + "cookies": [], "headers": [] }) .to_string(), @@ -2380,7 +2389,7 @@ fn bearer_auth_from_session_is_used() { serde_json::json!({ "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": "bearer", "raw_auth": "secret-token" }, - "cookies": {}, + "cookies": [], "headers": [] }) .to_string(), @@ -2437,7 +2446,7 @@ fn auth_netrc_is_not_persisted_in_session() { "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, - "cookies": {}, + "cookies": [], "headers": [ { "name": "hello", "value": "world" } ] @@ -2469,7 +2478,7 @@ fn multiple_headers_with_same_key_in_session() { serde_json::json!({ "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": {}, - "cookies": {}, + "cookies": [], "headers": [ { "name": "hello", "value": "world" }, { "name": "hello", "value": "people" }, @@ -2511,7 +2520,7 @@ fn headers_from_session_are_overwritten() { serde_json::json!({ "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": {}, - "cookies": {}, + "cookies": [], "headers": [ { "name": "hello", "value": "world" }, ] @@ -2541,6 +2550,7 @@ fn old_session_format_is_automatically_migrated() { let session_file = NamedTempFile::new().unwrap(); + // TODO std::fs::write( &session_file, serde_json::json!({ @@ -2568,7 +2578,7 @@ fn old_session_format_is_automatically_migrated() { serde_json::json!({ "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, - "cookies": {}, + "cookies": [], "headers": [ { "name": "hello", "value": "world" } ] From eed5d43630f6fcee5e0b270c8bd39c570fe48f96 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sun, 11 Jun 2023 15:04:58 +0100 Subject: [PATCH 05/28] default to request host if cookie domain missing --- src/session.rs | 12 +++++++++--- tests/cli.rs | 39 ++++++++++++++++++++++----------------- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/session.rs b/src/session.rs index 54f9309a..6a97af7f 100644 --- a/src/session.rs +++ b/src/session.rs @@ -147,8 +147,9 @@ impl Content { } pub struct Session { + host: Option, pub path: PathBuf, - pub read_only: bool, + read_only: bool, content: Content, } @@ -172,6 +173,7 @@ impl Session { }; Ok(Session { + host: url.host_str().map(Into::into), path, read_only, content, @@ -280,7 +282,7 @@ impl Session { cookie_builder = cookie_builder.secure(secure); } if let Some(domain) = &cookie.domain { - cookie_builder = cookie_builder.domain(domain.clone()) + cookie_builder = cookie_builder.domain(domain.clone()); } cookie_builder.finish() }) @@ -308,7 +310,10 @@ impl Session { .map(|v| v.unix_timestamp()), path: cookie.path().map(Into::into), secure: cookie.secure(), - domain: cookie.domain().map(Into::into), + domain: cookie + .domain() + .map(Into::into) + .or_else(|| self.host.clone()), }); } } @@ -360,6 +365,7 @@ mod tests { fn load_session_from_str(s: &str) -> Result { Ok(Session { content: serde_json::from_str::(s)?.migrate(), + host: Some("example.com".into()), path: PathBuf::new(), read_only: false, }) diff --git a/tests/cli.rs b/tests/cli.rs index c20709a7..9696c575 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -2019,8 +2019,8 @@ fn named_sessions() { }, "auth": { "type": "bearer", "raw_auth": "hello" }, "cookies": [ - { "name": "cook1", "value": "one", "path": "/" }, - { "name": "lang", "value": "en" } + { "name": "cook1", "value": "one", "path": "/", "domain": "127.0.0.1" }, + { "name": "lang", "value": "en", "domain": "127.0.0.1" } ], "headers": [] }) @@ -2061,7 +2061,7 @@ fn anonymous_sessions() { }, "auth": { "type": "basic", "raw_auth": "me:pass" }, "cookies": [ - { "name": "cook1", "value": "one" } + { "name": "cook1", "value": "one", "domain": "127.0.0.1" } ], "headers": [ { "name": "hello", "value": "world" } @@ -2084,7 +2084,7 @@ fn anonymous_read_only_session() { "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, "cookies": [ - { "name": "cookie1", "value": "one" } + { "name": "cookie1", "value": "one", "domain": "127.0.0.1" } ], "headers": [ { "name": "hello", "value": "world" } @@ -2146,7 +2146,7 @@ fn session_files_are_created_in_read_only_mode() { }, "auth": { "type": null, "raw_auth": null }, "cookies": [ - { "name": "lang", "value": "ar" } + { "name": "lang", "value": "ar", "domain": "127.0.0.1" } ], "headers": [ { "name": "hello", "value": "world" } @@ -2180,7 +2180,7 @@ fn named_read_only_session() { "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, "cookies": [ - { "name": "cookie1", "value": "one" } + { "name": "cookie1", "value": "one", "domain": "127.0.0.1" } ], "headers": [ { "name": "hello", "value": "world" } @@ -2226,16 +2226,19 @@ fn expired_cookies_are_removed_from_session() { { "name": "expired_cookie", "value": "random_string", - "expires": past_timestamp + "expires": past_timestamp, + "domain": "127.0.0.1" }, { "name": "unexpired_cookie", "value": "random_string", - "expires": future_timestamp + "expires": future_timestamp, + "domain": "127.0.0.1" }, { "name": "with_out_expiry", "value": "random_string", + "domain": "127.0.0.1" } ], "headers": [] @@ -2245,7 +2248,7 @@ fn expired_cookies_are_removed_from_session() { .unwrap(); get_command() - .arg(":") + .arg("127.0.0.1") .arg(format!( "--session={}", session_file.path().to_string_lossy() @@ -2264,11 +2267,13 @@ fn expired_cookies_are_removed_from_session() { { "name": "unexpired_cookie", "value": "random_string", - "expires": future_timestamp + "expires": future_timestamp, + "domain": "127.0.0.1" }, { "name": "with_out_expiry", - "value": "random_string" + "value": "random_string", + "domain": "127.0.0.1" } ], "headers": [] @@ -2305,8 +2310,8 @@ fn cookies_override_each_other_in_the_correct_order() { "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, "cookies": [ - { "name": "lang", "value": "fr" }, - { "name": "cook2", "value": "three" } + { "name": "lang", "value": "fr", "domain": "127.0.0.1" }, + { "name": "cook2", "value": "three", "domain": "127.0.0.1" } ], "headers": [] }) @@ -2334,9 +2339,9 @@ fn cookies_override_each_other_in_the_correct_order() { "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, "cookies": [ - { "name": "cook1", "value": "one" }, - { "name": "cook2", "value": "two" }, - { "name": "lang", "value": "en" } + { "name": "cook1", "value": "one", "domain": "127.0.0.1" }, + { "name": "cook2", "value": "two", "domain": "127.0.0.1" }, + { "name": "lang", "value": "en", "domain": "127.0.0.1" } ], "headers": [] }) @@ -2550,7 +2555,7 @@ fn old_session_format_is_automatically_migrated() { let session_file = NamedTempFile::new().unwrap(); - // TODO + // TODO: test migrating old format cookies std::fs::write( &session_file, serde_json::json!({ From 5d82351352e9314761bd27d2534c3e9647bfa770 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sun, 11 Jun 2023 15:15:11 +0100 Subject: [PATCH 06/28] add todos --- src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main.rs b/src/main.rs index 3a31a602..0c4410ba 100644 --- a/src/main.rs +++ b/src/main.rs @@ -332,6 +332,7 @@ fn run(args: Cli) -> Result { let mut cookie_jar = cookie_jar.lock().unwrap(); for cookie in s.cookies() { + // TODO: load cookies from other domains? match cookie_jar.insert_raw(&cookie, &url) { Ok(..) | Err(cookie_store::CookieError::Expired) @@ -586,6 +587,7 @@ fn run(args: Cli) -> Result { if let Some(ref mut s) = session { let cookie_jar = cookie_jar.lock().unwrap(); s.save_cookies( + // TODO: save everything in the cookie jar by replacing matches() with iter_unexpired() cookie_jar .matches(&url) .into_iter() From 368bcf92eddcfd8dea5bd8e3aeb343d387e4bda1 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Tue, 13 Jun 2023 22:08:40 +0100 Subject: [PATCH 07/28] enable cookie_store's preserve_order feature --- Cargo.lock | 1 + Cargo.toml | 2 +- src/session.rs | 4 +--- tests/cli.rs | 8 ++++---- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 905fa2d1..acd2acfc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -283,6 +283,7 @@ checksum = "b3f7034c0932dc36f5bd8ec37368d971346809435824f277cb3b8299fc56167c" dependencies = [ "cookie 0.15.2", "idna 0.2.3", + "indexmap", "log", "publicsuffix", "serde", diff --git a/Cargo.toml b/Cargo.toml index b629d9a9..63b321c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ chardetng = "0.1.15" clap = { version = "3.1.0", features = ["derive", "wrap_help"] } clap_complete = { version = "3.1.0", optional = true } cookie_crate = { version = "0.15", package = "cookie" } -cookie_store = { version = "0.15.0" } +cookie_store = { version = "0.15.0", features = ["preserve_order"] } digest_auth = "0.3.0" dirs = "3.0.1" encoding_rs = "0.8.28" diff --git a/src/session.rs b/src/session.rs index 6a97af7f..a42b3043 100644 --- a/src/session.rs +++ b/src/session.rs @@ -290,7 +290,7 @@ impl Session { } } - pub fn save_cookies(&mut self, mut cookies: Vec) { + pub fn save_cookies(&mut self, cookies: Vec) { let session_cookies = match self.content.cookies { Cookies::Map(_) => unreachable!(), Cookies::List(ref mut cookies) => cookies, @@ -298,8 +298,6 @@ impl Session { session_cookies.clear(); - cookies.sort_by(|a, b| a.name().cmp(b.name())); - for cookie in cookies { session_cookies.push(Cookie { name: cookie.name().into(), diff --git a/tests/cli.rs b/tests/cli.rs index 9696c575..8b80e0b6 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -2019,8 +2019,8 @@ fn named_sessions() { }, "auth": { "type": "bearer", "raw_auth": "hello" }, "cookies": [ - { "name": "cook1", "value": "one", "path": "/", "domain": "127.0.0.1" }, - { "name": "lang", "value": "en", "domain": "127.0.0.1" } + { "name": "lang", "value": "en", "domain": "127.0.0.1" }, + { "name": "cook1", "value": "one", "path": "/", "domain": "127.0.0.1" } ], "headers": [] }) @@ -2339,9 +2339,9 @@ fn cookies_override_each_other_in_the_correct_order() { "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, "cookies": [ - { "name": "cook1", "value": "one", "domain": "127.0.0.1" }, + { "name": "lang", "value": "en", "domain": "127.0.0.1" }, { "name": "cook2", "value": "two", "domain": "127.0.0.1" }, - { "name": "lang", "value": "en", "domain": "127.0.0.1" } + { "name": "cook1", "value": "one", "domain": "127.0.0.1" }, ], "headers": [] }) From f439fda3a487518c02113c8152a3c7ce21da44f3 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Tue, 13 Jun 2023 22:25:11 +0100 Subject: [PATCH 08/28] add todos --- src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 0c4410ba..4996557e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -331,8 +331,8 @@ fn run(args: Cli) -> Result { s.save_headers(&headers)?; let mut cookie_jar = cookie_jar.lock().unwrap(); + // TODO: load all cookies including those from other domains for cookie in s.cookies() { - // TODO: load cookies from other domains? match cookie_jar.insert_raw(&cookie, &url) { Ok(..) | Err(cookie_store::CookieError::Expired) @@ -341,6 +341,7 @@ fn run(args: Cli) -> Result { } } if let Some(cookie) = headers.remove(COOKIE) { + // TODO: use Cookie::split_parse() for cookie in cookie.to_str()?.split(';') { cookie_jar.insert_raw(&cookie.parse()?, &url)?; } @@ -587,9 +588,8 @@ fn run(args: Cli) -> Result { if let Some(ref mut s) = session { let cookie_jar = cookie_jar.lock().unwrap(); s.save_cookies( - // TODO: save everything in the cookie jar by replacing matches() with iter_unexpired() cookie_jar - .matches(&url) + .iter_unexpired() .into_iter() .map(|c| cookie_crate::Cookie::from(c.clone())) .collect(), From 3ac20ea3105bb54c44948cd6b83b34682481f582 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Tue, 13 Jun 2023 22:47:49 +0100 Subject: [PATCH 09/28] fix useless_conversion clippy warning --- src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 4996557e..3ec81a24 100644 --- a/src/main.rs +++ b/src/main.rs @@ -590,7 +590,6 @@ fn run(args: Cli) -> Result { s.save_cookies( cookie_jar .iter_unexpired() - .into_iter() .map(|c| cookie_crate::Cookie::from(c.clone())) .collect(), ); From 00730816f5a10327219ef948bfa3da765afe83ae Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sat, 17 Jun 2023 15:32:10 +0100 Subject: [PATCH 10/28] revert defaulting to request host for domain --- src/session.rs | 8 +------- tests/cli.rs | 27 +++++++++++---------------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/src/session.rs b/src/session.rs index a42b3043..cc598857 100644 --- a/src/session.rs +++ b/src/session.rs @@ -147,7 +147,6 @@ impl Content { } pub struct Session { - host: Option, pub path: PathBuf, read_only: bool, content: Content, @@ -173,7 +172,6 @@ impl Session { }; Ok(Session { - host: url.host_str().map(Into::into), path, read_only, content, @@ -308,10 +306,7 @@ impl Session { .map(|v| v.unix_timestamp()), path: cookie.path().map(Into::into), secure: cookie.secure(), - domain: cookie - .domain() - .map(Into::into) - .or_else(|| self.host.clone()), + domain: cookie.domain().map(Into::into), }); } } @@ -363,7 +358,6 @@ mod tests { fn load_session_from_str(s: &str) -> Result { Ok(Session { content: serde_json::from_str::(s)?.migrate(), - host: Some("example.com".into()), path: PathBuf::new(), read_only: false, }) diff --git a/tests/cli.rs b/tests/cli.rs index 8b80e0b6..d961167d 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -2019,8 +2019,8 @@ fn named_sessions() { }, "auth": { "type": "bearer", "raw_auth": "hello" }, "cookies": [ - { "name": "lang", "value": "en", "domain": "127.0.0.1" }, - { "name": "cook1", "value": "one", "path": "/", "domain": "127.0.0.1" } + { "name": "lang", "value": "en" }, + { "name": "cook1", "value": "one", "path": "/" } ], "headers": [] }) @@ -2061,7 +2061,7 @@ fn anonymous_sessions() { }, "auth": { "type": "basic", "raw_auth": "me:pass" }, "cookies": [ - { "name": "cook1", "value": "one", "domain": "127.0.0.1" } + { "name": "cook1", "value": "one" } ], "headers": [ { "name": "hello", "value": "world" } @@ -2084,7 +2084,7 @@ fn anonymous_read_only_session() { "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, "cookies": [ - { "name": "cookie1", "value": "one", "domain": "127.0.0.1" } + { "name": "cookie1", "value": "one" } ], "headers": [ { "name": "hello", "value": "world" } @@ -2146,7 +2146,7 @@ fn session_files_are_created_in_read_only_mode() { }, "auth": { "type": null, "raw_auth": null }, "cookies": [ - { "name": "lang", "value": "ar", "domain": "127.0.0.1" } + { "name": "lang", "value": "ar" } ], "headers": [ { "name": "hello", "value": "world" } @@ -2180,7 +2180,7 @@ fn named_read_only_session() { "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, "cookies": [ - { "name": "cookie1", "value": "one", "domain": "127.0.0.1" } + { "name": "cookie1", "value": "one" } ], "headers": [ { "name": "hello", "value": "world" } @@ -2227,18 +2227,15 @@ fn expired_cookies_are_removed_from_session() { "name": "expired_cookie", "value": "random_string", "expires": past_timestamp, - "domain": "127.0.0.1" }, { "name": "unexpired_cookie", "value": "random_string", "expires": future_timestamp, - "domain": "127.0.0.1" }, { "name": "with_out_expiry", "value": "random_string", - "domain": "127.0.0.1" } ], "headers": [] @@ -2268,12 +2265,10 @@ fn expired_cookies_are_removed_from_session() { "name": "unexpired_cookie", "value": "random_string", "expires": future_timestamp, - "domain": "127.0.0.1" }, { "name": "with_out_expiry", "value": "random_string", - "domain": "127.0.0.1" } ], "headers": [] @@ -2310,8 +2305,8 @@ fn cookies_override_each_other_in_the_correct_order() { "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, "cookies": [ - { "name": "lang", "value": "fr", "domain": "127.0.0.1" }, - { "name": "cook2", "value": "three", "domain": "127.0.0.1" } + { "name": "lang", "value": "fr" }, + { "name": "cook2", "value": "three" } ], "headers": [] }) @@ -2339,9 +2334,9 @@ fn cookies_override_each_other_in_the_correct_order() { "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, "cookies": [ - { "name": "lang", "value": "en", "domain": "127.0.0.1" }, - { "name": "cook2", "value": "two", "domain": "127.0.0.1" }, - { "name": "cook1", "value": "one", "domain": "127.0.0.1" }, + { "name": "lang", "value": "en" }, + { "name": "cook2", "value": "two" }, + { "name": "cook1", "value": "one" }, ], "headers": [] }) From 3cc1fff0e0db9dc81d57edc7ba45f19524f1ba26 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sat, 17 Jun 2023 16:53:12 +0100 Subject: [PATCH 11/28] update cookie_store and its peer dependencies --- Cargo.lock | 205 +++++-------------------------------------------- Cargo.toml | 6 +- src/main.rs | 2 +- src/session.rs | 20 ++--- 4 files changed, 32 insertions(+), 201 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index acd2acfc..4b99d190 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,12 +76,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "base-x" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" - [[package]] name = "base64" version = "0.13.1" @@ -247,23 +241,6 @@ dependencies = [ "windows-sys 0.42.0", ] -[[package]] -name = "const_fn" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" - -[[package]] -name = "cookie" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc6e25dfc584d06a3dbf775d207ff00d7de98d824c952dd2233dfbb261889a42" -dependencies = [ - "percent-encoding", - "time 0.2.27", - "version_check", -] - [[package]] name = "cookie" version = "0.16.2" @@ -271,40 +248,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding", - "time 0.3.20", + "time", "version_check", ] [[package]] name = "cookie_store" -version = "0.15.1" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3f7034c0932dc36f5bd8ec37368d971346809435824f277cb3b8299fc56167c" +checksum = "2e4b6aa369f41f5faa04bb80c9b1f4216ea81646ed6124d76ba5c49a7aafd9cd" dependencies = [ - "cookie 0.15.2", + "cookie", "idna 0.2.3", - "indexmap", "log", "publicsuffix", "serde", "serde_json", - "time 0.2.27", + "time", "url", ] [[package]] name = "cookie_store" -version = "0.16.1" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e4b6aa369f41f5faa04bb80c9b1f4216ea81646ed6124d76ba5c49a7aafd9cd" +checksum = "d5a18f35792056f8c7c2de9c002e7e4fe44c7b5f66e7d99f46468dbb730a7ea7" dependencies = [ - "cookie 0.16.2", - "idna 0.2.3", + "cookie", + "idna 0.3.0", + "indexmap", "log", "publicsuffix", "serde", + "serde_derive", "serde_json", - "time 0.3.20", + "time", "url", ] @@ -390,12 +368,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "discard" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" - [[package]] name = "doc-comment" version = "0.3.3" @@ -1157,7 +1129,7 @@ dependencies = [ "line-wrap", "quick-xml", "serde", - "time 0.3.20", + "time", ] [[package]] @@ -1232,12 +1204,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" version = "1.0.56" @@ -1371,7 +1337,7 @@ checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ "base64 0.21.0", "bytes", - "cookie 0.16.2", + "cookie", "cookie_store 0.16.1", "encoding_rs", "futures-core", @@ -1441,15 +1407,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "0.37.11" @@ -1570,21 +1527,6 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "serde" version = "1.0.159" @@ -1629,21 +1571,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha1" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" -dependencies = [ - "sha1_smol", -] - -[[package]] -name = "sha1_smol" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" - [[package]] name = "sha2" version = "0.9.9" @@ -1682,64 +1609,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -[[package]] -name = "standback" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" -dependencies = [ - "version_check", -] - -[[package]] -name = "stdweb" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" -dependencies = [ - "discard", - "rustc_version", - "stdweb-derive", - "stdweb-internal-macros", - "stdweb-internal-runtime", - "wasm-bindgen", -] - -[[package]] -name = "stdweb-derive" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "serde_derive", - "syn 1.0.109", -] - -[[package]] -name = "stdweb-internal-macros" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" -dependencies = [ - "base-x", - "proc-macro2", - "quote", - "serde", - "serde_derive", - "serde_json", - "sha1", - "syn 1.0.109", -] - -[[package]] -name = "stdweb-internal-runtime" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" - [[package]] name = "strsim" version = "0.10.0" @@ -1857,21 +1726,6 @@ dependencies = [ "syn 2.0.13", ] -[[package]] -name = "time" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" -dependencies = [ - "const_fn", - "libc", - "standback", - "stdweb", - "time-macros 0.1.1", - "version_check", - "winapi", -] - [[package]] name = "time" version = "0.3.20" @@ -1881,7 +1735,7 @@ dependencies = [ "itoa", "serde", "time-core", - "time-macros 0.2.8", + "time-macros", ] [[package]] @@ -1890,16 +1744,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" -[[package]] -name = "time-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" -dependencies = [ - "proc-macro-hack", - "time-macros-impl", -] - [[package]] name = "time-macros" version = "0.2.8" @@ -1909,19 +1753,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "time-macros-impl" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" -dependencies = [ - "proc-macro-hack", - "proc-macro2", - "quote", - "standback", - "syn 1.0.109", -] - [[package]] name = "tinyvec" version = "1.6.0" @@ -2430,8 +2261,8 @@ dependencies = [ "chardetng", "clap", "clap_complete", - "cookie 0.15.2", - "cookie_store 0.15.1", + "cookie", + "cookie_store 0.19.1", "digest_auth", "dirs", "encoding_rs", @@ -2462,7 +2293,7 @@ dependencies = [ "syntect", "tempfile", "termcolor", - "time 0.2.27", + "time", "tokio", "unicode-width", "url", diff --git a/Cargo.toml b/Cargo.toml index 63b321c6..8f456a1d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,8 +21,8 @@ brotli = { version = "3.3.0", default-features = false, features = ["std"] } chardetng = "0.1.15" clap = { version = "3.1.0", features = ["derive", "wrap_help"] } clap_complete = { version = "3.1.0", optional = true } -cookie_crate = { version = "0.15", package = "cookie" } -cookie_store = { version = "0.15.0", features = ["preserve_order"] } +cookie_crate = { version = "0.16.1", package = "cookie" } +cookie_store = { version = "0.19.1", features = ["preserve_order"] } digest_auth = "0.3.0" dirs = "3.0.1" encoding_rs = "0.8.28" @@ -46,7 +46,7 @@ serde = "1.0" serde_json = { version = "1.0", features = ["preserve_order"] } serde_urlencoded = "0.7.0" termcolor = "1.1.2" -time = "0.2.26" +time = "0.3.16" unicode-width = "0.1.9" url = "2.2.2" diff --git a/src/main.rs b/src/main.rs index 3ec81a24..2591652a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -332,7 +332,7 @@ fn run(args: Cli) -> Result { let mut cookie_jar = cookie_jar.lock().unwrap(); // TODO: load all cookies including those from other domains - for cookie in s.cookies() { + for cookie in s.cookies()? { match cookie_jar.insert_raw(&cookie, &url) { Ok(..) | Err(cookie_store::CookieError::Expired) diff --git a/src/session.rs b/src/session.rs index cc598857..de3052ba 100644 --- a/src/session.rs +++ b/src/session.rs @@ -261,7 +261,7 @@ impl Session { } } - pub fn cookies(&self) -> Vec { + pub fn cookies(&self) -> Result> { match &self.content.cookies { Cookies::Map(_) => unreachable!(), Cookies::List(cookies) => cookies @@ -271,7 +271,7 @@ impl Session { cookie_crate::Cookie::build(cookie.name.clone(), cookie.value.clone()); if let Some(expires) = cookie.expires { cookie_builder = cookie_builder - .expires(time::OffsetDateTime::from_unix_timestamp(expires)); + .expires(time::OffsetDateTime::from_unix_timestamp(expires)?); } if let Some(path) = &cookie.path { cookie_builder = cookie_builder.path(path.clone()); @@ -282,7 +282,7 @@ impl Session { if let Some(domain) = &cookie.domain { cookie_builder = cookie_builder.domain(domain.clone()); } - cookie_builder.finish() + Ok(cookie_builder.finish()) }) .collect(), } @@ -384,9 +384,9 @@ mod tests { session.headers()?.get("hello"), Some(&HeaderValue::from_static("world")), ); - assert_eq!(session.cookies()[0].name_value(), ("baz", "quux")); - assert_eq!(session.cookies()[0].path(), Some("/")); - assert_eq!(session.cookies()[0].secure(), Some(false)); + assert_eq!(session.cookies()?[0].name_value(), ("baz", "quux")); + assert_eq!(session.cookies()?[0].path(), Some("/")); + assert_eq!(session.cookies()?[0].secure(), Some(false)); assert_eq!(session.content.auth, Auth::default()); Ok(()) @@ -412,9 +412,9 @@ mod tests { session.headers()?.get("hello"), Some(&HeaderValue::from_static("world")), ); - assert_eq!(session.cookies()[0].name_value(), ("baz", "quux")); - assert_eq!(session.cookies()[0].path(), Some("/")); - assert_eq!(session.cookies()[0].secure(), Some(false)); + assert_eq!(session.cookies()?[0].name_value(), ("baz", "quux")); + assert_eq!(session.cookies()?[0].path(), Some("/")); + assert_eq!(session.cookies()?[0].secure(), Some(false)); assert_eq!( session.content.auth, Auth { @@ -505,7 +505,7 @@ mod tests { } "#})?; - let cookies = session.cookies(); + let cookies = session.cookies()?; assert_eq!(cookies[0].name_value(), ("baz", "quux")); assert_eq!(cookies[0].path(), Some("/")); From f30f73f396548daf3744ee131c896ccdcc8b3d9e Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sat, 17 Jun 2023 18:07:29 +0100 Subject: [PATCH 12/28] ensure stored cookies always have a domain --- src/main.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 2591652a..bc668b00 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,6 +28,7 @@ use std::sync::Arc; use anyhow::{anyhow, Context, Result}; use atty::Stream; use cli::FormatOptions; +use cookie_store::CookieDomain; use network_interface::{NetworkInterface, NetworkInterfaceConfig}; use redirect::RedirectFollower; use reqwest::blocking::Client; @@ -590,7 +591,13 @@ fn run(args: Cli) -> Result { s.save_cookies( cookie_jar .iter_unexpired() - .map(|c| cookie_crate::Cookie::from(c.clone())) + .map(|c| { + let mut cookie = cookie_crate::Cookie::from(c.clone()); + if let CookieDomain::HostOnly(s) = &c.domain { + cookie.set_domain(s.clone()) + } + cookie + }) .collect(), ); s.persist() From 31aaa903af7f2f4c1449f292e6e59da83b8339d3 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sat, 17 Jun 2023 18:09:52 +0100 Subject: [PATCH 13/28] load all cookies regardless of their domain --- src/main.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index bc668b00..e0dfc17e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -332,12 +332,12 @@ fn run(args: Cli) -> Result { s.save_headers(&headers)?; let mut cookie_jar = cookie_jar.lock().unwrap(); - // TODO: load all cookies including those from other domains for cookie in s.cookies()? { - match cookie_jar.insert_raw(&cookie, &url) { - Ok(..) - | Err(cookie_store::CookieError::Expired) - | Err(cookie_store::CookieError::DomainMismatch) => {} + let cookie_url = &cookie + .domain() + .and_then(|d| format!("http://{d}").parse().ok()); + match cookie_jar.insert_raw(&cookie, cookie_url.as_ref().unwrap_or(&url)) { + Ok(..) | Err(CookieError::Expired) | Err(CookieError::DomainMismatch) => {} Err(err) => return Err(err.into()), } } From ef841b034bc7ccd594d6729ada82e87a730fe550 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sat, 17 Jun 2023 22:08:24 +0100 Subject: [PATCH 14/28] ensure cookie's domain attribute is not IP --- src/main.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index e0dfc17e..5968f456 100644 --- a/src/main.rs +++ b/src/main.rs @@ -36,6 +36,7 @@ use reqwest::header::{ HeaderValue, ACCEPT, ACCEPT_ENCODING, CONNECTION, CONTENT_TYPE, COOKIE, RANGE, USER_AGENT, }; use reqwest::tls; +use url::Host; use crate::auth::{Auth, DigestAuthMiddleware}; use crate::buffer::Buffer; @@ -332,10 +333,19 @@ fn run(args: Cli) -> Result { s.save_headers(&headers)?; let mut cookie_jar = cookie_jar.lock().unwrap(); - for cookie in s.cookies()? { - let cookie_url = &cookie + for mut cookie in s.cookies()? { + let cookie_url: &Option = &cookie .domain() .and_then(|d| format!("http://{d}").parse().ok()); + + // The cookie's domain attribute cannot be an IP address. + // See https://stackoverflow.com/a/30676300/5915221 + if let Some(url) = &cookie_url { + if let Some(Host::Ipv4(_) | Host::Ipv6(_)) = url.host() { + cookie.unset_domain() + } + } + match cookie_jar.insert_raw(&cookie, cookie_url.as_ref().unwrap_or(&url)) { Ok(..) | Err(CookieError::Expired) | Err(CookieError::DomainMismatch) => {} Err(err) => return Err(err.into()), @@ -434,7 +444,7 @@ fn run(args: Cli) -> Result { } else if !args.ignore_netrc { // I don't know if it's possible for host() to return None // But if it does we still want to use the default entry, if there is one - let host = url.host().unwrap_or(url::Host::Domain("")); + let host = url.host().unwrap_or(Host::Domain("")); if let Some(entry) = netrc::find_entry(host) { auth = Auth::from_netrc(auth_type, entry); save_auth_in_session = false; From 89a4659e3713945f5ba057538c7ed304ba06ed2d Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sun, 18 Jun 2023 13:11:28 +0100 Subject: [PATCH 15/28] fix failing tests --- tests/cli.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/cli.rs b/tests/cli.rs index d961167d..17453e47 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -2019,8 +2019,8 @@ fn named_sessions() { }, "auth": { "type": "bearer", "raw_auth": "hello" }, "cookies": [ - { "name": "lang", "value": "en" }, - { "name": "cook1", "value": "one", "path": "/" } + { "name": "lang", "value": "en", "domain": "127.0.0.1" }, + { "name": "cook1", "value": "one", "path": "/", "domain": "127.0.0.1" } ], "headers": [] }) @@ -2061,7 +2061,7 @@ fn anonymous_sessions() { }, "auth": { "type": "basic", "raw_auth": "me:pass" }, "cookies": [ - { "name": "cook1", "value": "one" } + { "name": "cook1", "value": "one", "domain": "127.0.0.1" } ], "headers": [ { "name": "hello", "value": "world" } @@ -2146,7 +2146,7 @@ fn session_files_are_created_in_read_only_mode() { }, "auth": { "type": null, "raw_auth": null }, "cookies": [ - { "name": "lang", "value": "ar" } + { "name": "lang", "value": "ar", "domain": "127.0.0.1" } ], "headers": [ { "name": "hello", "value": "world" } @@ -2227,15 +2227,18 @@ fn expired_cookies_are_removed_from_session() { "name": "expired_cookie", "value": "random_string", "expires": past_timestamp, + "domain": "127.0.0.1" }, { "name": "unexpired_cookie", "value": "random_string", "expires": future_timestamp, + "domain": "127.0.0.1" }, { "name": "with_out_expiry", "value": "random_string", + "domain": "127.0.0.1" } ], "headers": [] @@ -2265,10 +2268,12 @@ fn expired_cookies_are_removed_from_session() { "name": "unexpired_cookie", "value": "random_string", "expires": future_timestamp, + "domain": "127.0.0.1" }, { "name": "with_out_expiry", "value": "random_string", + "domain": "127.0.0.1" } ], "headers": [] @@ -2305,8 +2310,8 @@ fn cookies_override_each_other_in_the_correct_order() { "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, "cookies": [ - { "name": "lang", "value": "fr" }, - { "name": "cook2", "value": "three" } + { "name": "lang", "value": "fr", "domain": "127.0.0.1" }, + { "name": "cook2", "value": "three", "domain": "127.0.0.1" } ], "headers": [] }) @@ -2334,9 +2339,9 @@ fn cookies_override_each_other_in_the_correct_order() { "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, "cookies": [ - { "name": "lang", "value": "en" }, - { "name": "cook2", "value": "two" }, - { "name": "cook1", "value": "one" }, + { "name": "lang", "value": "en", "domain": "127.0.0.1" }, + { "name": "cook2", "value": "two", "domain": "127.0.0.1" }, + { "name": "cook1", "value": "one", "domain": "127.0.0.1" }, ], "headers": [] }) From f6c6267f23725f7ec60b959489318373685d5983 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sun, 18 Jun 2023 13:30:07 +0100 Subject: [PATCH 16/28] testing migration of old cookie format --- src/main.rs | 2 ++ tests/cli.rs | 25 ++++++++++++++++++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index 5968f456..d6e53642 100644 --- a/src/main.rs +++ b/src/main.rs @@ -606,6 +606,8 @@ fn run(args: Cli) -> Result { if let CookieDomain::HostOnly(s) = &c.domain { cookie.set_domain(s.clone()) } + // TODO: fix this in cookie_store crate + cookie.set_secure(c.secure()); cookie }) .collect(), diff --git a/tests/cli.rs b/tests/cli.rs index 17453e47..118bf4de 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -10,7 +10,7 @@ use std::iter::FromIterator; use std::net::IpAddr; use std::pin::Pin; use std::str::FromStr; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use assert_cmd::cmd::Command; use indoc::indoc; @@ -2207,7 +2207,6 @@ fn named_read_only_session() { #[test] fn expired_cookies_are_removed_from_session() { - use std::time::{SystemTime, UNIX_EPOCH}; let future_timestamp = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() @@ -2555,13 +2554,20 @@ fn old_session_format_is_automatically_migrated() { let session_file = NamedTempFile::new().unwrap(); - // TODO: test migrating old format cookies + let future_timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() + + 1000; + std::fs::write( &session_file, serde_json::json!({ "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": {}, - "cookies": {}, + "cookies": { + "lang": { "value": "en", "expires": future_timestamp, "path": "/", "secure": false }, + }, "headers": { "hello": "world" } }) .to_string(), @@ -2583,7 +2589,16 @@ fn old_session_format_is_automatically_migrated() { serde_json::json!({ "__meta__": { "about": "xh session file", "xh": "0.0.0" }, "auth": { "type": null, "raw_auth": null }, - "cookies": [], + "cookies": [ + { + "name": "lang", + "value": "en", + "expires": future_timestamp, + "path": "/", + "secure": false, + "domain": "127.0.0.1" + }, + ], "headers": [ { "name": "hello", "value": "world" } ] From 81b7b65f61887f710870f92e32cce7f028158dc9 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sun, 18 Jun 2023 13:40:06 +0100 Subject: [PATCH 17/28] =?UTF-8?q?`Date::as=5Fymd`=20=E2=86=92=20`Date::to?= =?UTF-8?q?=5Fcalendar=5Fdate`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was a breaking change introduced in time v0.3.0 --- src/cli.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli.rs b/src/cli.rs index 1e0acacd..bba7b438 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -886,7 +886,7 @@ fn generate_manpages(mut app: clap::Command, rest_args: Vec) -> Error { let mut manpage = fs::read_to_string(format!("{}/man-template.roff", rest_args[0])).unwrap(); let current_date = { - let (year, month, day) = DateTime::now_utc().date().as_ymd(); + let (year, month, day) = DateTime::now_utc().date().to_calendar_date(); format!("{}-{:02}-{:02}", year, month, day) }; From 24969862ea0ef3d8f2f1c25a64c1d729900da627 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sun, 18 Jun 2023 22:51:25 +0100 Subject: [PATCH 18/28] test session cookies from multiple domain --- tests/cli.rs | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tests/cli.rs b/tests/cli.rs index 118bf4de..b0855e09 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -2347,6 +2347,75 @@ fn cookies_override_each_other_in_the_correct_order() { ); } +#[test] +fn cookies_are_segmented_by_domain() { + let session_file = NamedTempFile::new().unwrap(); + + std::fs::write( + &session_file, + serde_json::json!({ + "__meta__": { "about": "xh session file", "xh": "0.0.0" }, + "auth": { "type": null, "raw_auth": null }, + "cookies": [ + // will be overwritten by set-cookie header from example.com + { "name": "lang", "value": "fi", "domain": "example.com" }, + // will not be overwritten + { "name": "lang", "value": "fr", "domain": "example.org" }, + ], + "headers": [] + }) + .to_string(), + ) + .unwrap(); + + let server = server::http(|req| async move { + match req.uri().host() { + Some("example.com") => { + assert_eq!(req.headers()["cookie"].to_str().unwrap(), "lang=fi"); + hyper::Response::builder() + .header("set-cookie", "lang=en") + .body("".into()) + .unwrap() + } + Some("example.net") => { + assert!(req.headers().get("cookie").is_none()); + hyper::Response::builder() + .header("set-cookie", "lang=ar") + .body("".into()) + .unwrap() + } + _ => panic!("unknown path"), + } + }); + + for url in ["http://example.com", "http://example.net"] { + get_command() + .arg(url) + .arg(format!("--proxy=all:{}", server.base_url())) + .arg(format!( + "--session={}", + session_file.path().to_string_lossy() + )) + .assert() + .success(); + } + + let session_content = fs::read_to_string(session_file.path()).unwrap(); + assert_eq!( + serde_json::from_str::(&session_content).unwrap(), + serde_json::json!({ + "__meta__": { "about": "xh session file", "xh": "0.0.0" }, + "auth": { "type": null, "raw_auth": null }, + "cookies": [ + { "name": "lang", "value": "en", "domain": "example.com" }, + { "name": "lang", "value": "fr", "domain": "example.org" }, + { "name": "lang", "value": "ar", "domain": "example.net" } + ], + "headers": [] + }) + ); +} + #[test] fn basic_auth_from_session_is_used() { let server = server::http(|req| async move { From d50f94b6af39bdbae6ad9df2bf0892d227273367 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sun, 18 Jun 2023 23:21:51 +0100 Subject: [PATCH 19/28] update cookie_store and use re-exported RawCookie --- Cargo.lock | 24 +++++++++++++++++------- Cargo.toml | 3 +-- src/main.rs | 2 +- src/session.rs | 6 +++--- src/vendored/reqwest_cookie_store.rs | 3 +-- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b99d190..e0a1678d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -252,13 +252,24 @@ dependencies = [ "version_check", ] +[[package]] +name = "cookie" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7efb37c3e1ccb1ff97164ad95ac1606e8ccd35b3fa0a7d99a304c7f4a428cc24" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + [[package]] name = "cookie_store" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e4b6aa369f41f5faa04bb80c9b1f4216ea81646ed6124d76ba5c49a7aafd9cd" dependencies = [ - "cookie", + "cookie 0.16.2", "idna 0.2.3", "log", "publicsuffix", @@ -270,11 +281,11 @@ dependencies = [ [[package]] name = "cookie_store" -version = "0.19.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5a18f35792056f8c7c2de9c002e7e4fe44c7b5f66e7d99f46468dbb730a7ea7" +checksum = "387461abbc748185c3a6e1673d826918b450b87ff22639429c694619a83b6cf6" dependencies = [ - "cookie", + "cookie 0.17.0", "idna 0.3.0", "indexmap", "log", @@ -1337,7 +1348,7 @@ checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ "base64 0.21.0", "bytes", - "cookie", + "cookie 0.16.2", "cookie_store 0.16.1", "encoding_rs", "futures-core", @@ -2261,8 +2272,7 @@ dependencies = [ "chardetng", "clap", "clap_complete", - "cookie", - "cookie_store 0.19.1", + "cookie_store 0.20.0", "digest_auth", "dirs", "encoding_rs", diff --git a/Cargo.toml b/Cargo.toml index 8f456a1d..1eca3d13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,8 +21,7 @@ brotli = { version = "3.3.0", default-features = false, features = ["std"] } chardetng = "0.1.15" clap = { version = "3.1.0", features = ["derive", "wrap_help"] } clap_complete = { version = "3.1.0", optional = true } -cookie_crate = { version = "0.16.1", package = "cookie" } -cookie_store = { version = "0.19.1", features = ["preserve_order"] } +cookie_store = { version = "0.20.0", features = ["preserve_order"] } digest_auth = "0.3.0" dirs = "3.0.1" encoding_rs = "0.8.28" diff --git a/src/main.rs b/src/main.rs index d6e53642..69384b8e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -602,7 +602,7 @@ fn run(args: Cli) -> Result { cookie_jar .iter_unexpired() .map(|c| { - let mut cookie = cookie_crate::Cookie::from(c.clone()); + let mut cookie = cookie_store::RawCookie::from(c.clone()); if let CookieDomain::HostOnly(s) = &c.domain { cookie.set_domain(s.clone()) } diff --git a/src/session.rs b/src/session.rs index de3052ba..d2258540 100644 --- a/src/session.rs +++ b/src/session.rs @@ -261,14 +261,14 @@ impl Session { } } - pub fn cookies(&self) -> Result> { + pub fn cookies(&self) -> Result> { match &self.content.cookies { Cookies::Map(_) => unreachable!(), Cookies::List(cookies) => cookies .iter() .map(|cookie| { let mut cookie_builder = - cookie_crate::Cookie::build(cookie.name.clone(), cookie.value.clone()); + cookie_store::RawCookie::build(cookie.name.clone(), cookie.value.clone()); if let Some(expires) = cookie.expires { cookie_builder = cookie_builder .expires(time::OffsetDateTime::from_unix_timestamp(expires)?); @@ -288,7 +288,7 @@ impl Session { } } - pub fn save_cookies(&mut self, cookies: Vec) { + pub fn save_cookies(&mut self, cookies: Vec) { let session_cookies = match self.content.cookies { Cookies::Map(_) => unreachable!(), Cookies::List(ref mut cookies) => cookies, diff --git a/src/vendored/reqwest_cookie_store.rs b/src/vendored/reqwest_cookie_store.rs index a042b672..b203502d 100644 --- a/src/vendored/reqwest_cookie_store.rs +++ b/src/vendored/reqwest_cookie_store.rs @@ -7,8 +7,7 @@ use std::sync::{Mutex, MutexGuard, PoisonError, RwLock}; -use cookie_crate::{Cookie as RawCookie, ParseError as RawCookieParseError}; -use cookie_store::CookieStore; +use cookie_store::{CookieStore, RawCookie, RawCookieParseError}; use reqwest::header::HeaderValue; use url; From ac02b56e4c24e8a96b691398d47119316e0b7ea2 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sun, 18 Jun 2023 23:23:06 +0100 Subject: [PATCH 20/28] replace custom cookie splitting with split_parse --- src/main.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 69384b8e..47ad22f8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -352,9 +352,8 @@ fn run(args: Cli) -> Result { } } if let Some(cookie) = headers.remove(COOKIE) { - // TODO: use Cookie::split_parse() - for cookie in cookie.to_str()?.split(';') { - cookie_jar.insert_raw(&cookie.parse()?, &url)?; + for cookie in cookie_store::RawCookie::split_parse(cookie.to_str()?) { + cookie_jar.insert_raw(&cookie?, &url)?; } } } From 66320a0822b755345682ceb615dc6c3a64781df4 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Mon, 19 Jun 2023 23:10:03 +0100 Subject: [PATCH 21/28] take and return Cookie instead of RawCookie --- src/main.rs | 37 ++++--------------------- src/session.rs | 74 ++++++++++++++++++++++++++++++++++---------------- 2 files changed, 57 insertions(+), 54 deletions(-) diff --git a/src/main.rs b/src/main.rs index 47ad22f8..29adab3e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,7 +28,7 @@ use std::sync::Arc; use anyhow::{anyhow, Context, Result}; use atty::Stream; use cli::FormatOptions; -use cookie_store::CookieDomain; +use cookie_store::RawCookie; use network_interface::{NetworkInterface, NetworkInterfaceConfig}; use redirect::RedirectFollower; use reqwest::blocking::Client; @@ -314,7 +314,7 @@ fn run(args: Cli) -> Result { let mut session = match &args.session { Some(name_or_path) => Some( - Session::load_session(&url, name_or_path.clone(), args.is_session_read_only) + Session::load_session(url.clone(), name_or_path.clone(), args.is_session_read_only) .with_context(|| { format!("couldn't load session {:?}", name_or_path.to_string_lossy()) })?, @@ -333,26 +333,14 @@ fn run(args: Cli) -> Result { s.save_headers(&headers)?; let mut cookie_jar = cookie_jar.lock().unwrap(); - for mut cookie in s.cookies()? { - let cookie_url: &Option = &cookie - .domain() - .and_then(|d| format!("http://{d}").parse().ok()); - - // The cookie's domain attribute cannot be an IP address. - // See https://stackoverflow.com/a/30676300/5915221 - if let Some(url) = &cookie_url { - if let Some(Host::Ipv4(_) | Host::Ipv6(_)) = url.host() { - cookie.unset_domain() - } - } - - match cookie_jar.insert_raw(&cookie, cookie_url.as_ref().unwrap_or(&url)) { + for (cookie, cookie_url) in s.cookies()? { + match cookie_jar.insert_raw(&cookie, &cookie_url) { Ok(..) | Err(CookieError::Expired) | Err(CookieError::DomainMismatch) => {} Err(err) => return Err(err.into()), } } if let Some(cookie) = headers.remove(COOKIE) { - for cookie in cookie_store::RawCookie::split_parse(cookie.to_str()?) { + for cookie in RawCookie::split_parse(cookie.to_str()?) { cookie_jar.insert_raw(&cookie?, &url)?; } } @@ -597,20 +585,7 @@ fn run(args: Cli) -> Result { if let Some(ref mut s) = session { let cookie_jar = cookie_jar.lock().unwrap(); - s.save_cookies( - cookie_jar - .iter_unexpired() - .map(|c| { - let mut cookie = cookie_store::RawCookie::from(c.clone()); - if let CookieDomain::HostOnly(s) = &c.domain { - cookie.set_domain(s.clone()) - } - // TODO: fix this in cookie_store crate - cookie.set_secure(c.secure()); - cookie - }) - .collect(), - ); + s.save_cookies(cookie_jar.iter_unexpired()); s.persist() .with_context(|| format!("couldn't persist session {}", s.path.display()))?; } diff --git a/src/session.rs b/src/session.rs index d2258540..1bed1dff 100644 --- a/src/session.rs +++ b/src/session.rs @@ -7,8 +7,8 @@ use std::path::PathBuf; use anyhow::{anyhow, Context, Result}; use reqwest::header::HeaderMap; -use reqwest::Url; use serde::{Deserialize, Serialize}; +use url::Url; use crate::auth; use crate::utils::{config_dir, test_mode}; @@ -147,19 +147,20 @@ impl Content { } pub struct Session { + url: Url, pub path: PathBuf, read_only: bool, content: Content, } impl Session { - pub fn load_session(url: &Url, mut name_or_path: OsString, read_only: bool) -> Result { + pub fn load_session(url: Url, mut name_or_path: OsString, read_only: bool) -> Result { let path = if is_path(&name_or_path) { PathBuf::from(name_or_path) } else { let mut path = config_dir() .context("couldn't get config directory")? - .join::(["sessions", &path_from_url(url)?].iter().collect()); + .join::(["sessions", &path_from_url(&url)?].iter().collect()); name_or_path.push(".json"); path.push(name_or_path); path @@ -172,6 +173,7 @@ impl Session { }; Ok(Session { + url, path, read_only, content, @@ -261,7 +263,7 @@ impl Session { } } - pub fn cookies(&self) -> Result> { + pub fn cookies(&self) -> Result> { match &self.content.cookies { Cookies::Map(_) => unreachable!(), Cookies::List(cookies) => cookies @@ -269,26 +271,44 @@ impl Session { .map(|cookie| { let mut cookie_builder = cookie_store::RawCookie::build(cookie.name.clone(), cookie.value.clone()); + if let Some(expires) = cookie.expires { cookie_builder = cookie_builder .expires(time::OffsetDateTime::from_unix_timestamp(expires)?); } if let Some(path) = &cookie.path { - cookie_builder = cookie_builder.path(path.clone()); + cookie_builder = cookie_builder.path(path); } if let Some(secure) = cookie.secure { cookie_builder = cookie_builder.secure(secure); } + + let mut cookie_url = self.url.clone(); if let Some(domain) = &cookie.domain { - cookie_builder = cookie_builder.domain(domain.clone()); + cookie_url = format!("http://{domain}").parse()?; + // The cookie's domain attribute cannot be an IP address. + // See https://stackoverflow.com/a/30676300/5915221 + if let Some(url::Host::Domain(_)) = cookie_url.host() { + cookie_builder = cookie_builder.domain(domain.clone()); + } } - Ok(cookie_builder.finish()) + + Ok(( + cookie_store::Cookie::try_from_raw_cookie( + &cookie_builder.finish(), + &cookie_url, + )?, + cookie_url, + )) }) .collect(), } } - pub fn save_cookies(&mut self, cookies: Vec) { + pub fn save_cookies<'a, I>(&mut self, cookies: I) + where + I: Iterator>, + { let session_cookies = match self.content.cookies { Cookies::Map(_) => unreachable!(), Cookies::List(ref mut cookies) => cookies, @@ -297,6 +317,11 @@ impl Session { session_cookies.clear(); for cookie in cookies { + let mut domain = cookie.domain(); + if let cookie_store::CookieDomain::HostOnly(s) = &cookie.domain { + domain = Some(s); + } + session_cookies.push(Cookie { name: cookie.name().into(), value: cookie.value().into(), @@ -306,7 +331,7 @@ impl Session { .map(|v| v.unix_timestamp()), path: cookie.path().map(Into::into), secure: cookie.secure(), - domain: cookie.domain().map(Into::into), + domain: domain.map(Into::into), }); } } @@ -357,6 +382,7 @@ mod tests { fn load_session_from_str(s: &str) -> Result { Ok(Session { + url: Url::parse("http://example.net").unwrap(), content: serde_json::from_str::(s)?.migrate(), path: PathBuf::new(), read_only: false, @@ -384,9 +410,10 @@ mod tests { session.headers()?.get("hello"), Some(&HeaderValue::from_static("world")), ); - assert_eq!(session.cookies()?[0].name_value(), ("baz", "quux")); - assert_eq!(session.cookies()?[0].path(), Some("/")); - assert_eq!(session.cookies()?[0].secure(), Some(false)); + let cookies = session.cookies()?; + assert_eq!(cookies[0].0.name_value(), ("baz", "quux")); + assert_eq!(cookies[0].0.path(), Some("/")); + assert_eq!(cookies[0].0.secure(), Some(false)); assert_eq!(session.content.auth, Auth::default()); Ok(()) @@ -412,9 +439,10 @@ mod tests { session.headers()?.get("hello"), Some(&HeaderValue::from_static("world")), ); - assert_eq!(session.cookies()?[0].name_value(), ("baz", "quux")); - assert_eq!(session.cookies()?[0].path(), Some("/")); - assert_eq!(session.cookies()?[0].secure(), Some(false)); + let cookies = session.cookies()?; + assert_eq!(cookies[0].0.name_value(), ("baz", "quux")); + assert_eq!(cookies[0].0.path(), Some("/")); + assert_eq!(cookies[0].0.secure(), Some(false)); assert_eq!( session.content.auth, Auth { @@ -507,15 +535,15 @@ mod tests { let cookies = session.cookies()?; - assert_eq!(cookies[0].name_value(), ("baz", "quux")); - assert_eq!(cookies[0].path(), Some("/")); - assert_eq!(cookies[0].secure(), Some(false)); - assert_eq!(cookies[0].domain(), Some("example.com")); + assert_eq!(cookies[0].0.name_value(), ("baz", "quux")); + assert_eq!(cookies[0].0.path(), Some("/")); + assert_eq!(cookies[0].0.secure(), Some(false)); + assert_eq!(cookies[0].0.domain(), Some("example.com")); - assert_eq!(cookies[1].name_value(), ("foo", "bar")); - assert_eq!(cookies[1].path(), Some("/")); - assert_eq!(cookies[1].secure(), Some(false)); - assert_eq!(cookies[1].domain(), None); + assert_eq!(cookies[1].0.name_value(), ("foo", "bar")); + assert_eq!(cookies[1].0.path(), Some("/")); + assert_eq!(cookies[1].0.secure(), Some(false)); + assert_eq!(cookies[1].0.domain(), None); Ok(()) } From 5833cd9a8e68f4d7690233f1d7bef9b8d8ff6a03 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sat, 24 Jun 2023 20:10:06 +0100 Subject: [PATCH 22/28] simplify loading cookies into cookie_store --- src/main.rs | 9 ++--- src/session.rs | 95 ++++++++++++++++++++++++-------------------------- 2 files changed, 48 insertions(+), 56 deletions(-) diff --git a/src/main.rs b/src/main.rs index 29adab3e..8b9fb7cc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,6 +28,7 @@ use std::sync::Arc; use anyhow::{anyhow, Context, Result}; use atty::Stream; use cli::FormatOptions; +use cookie_store::CookieStore; use cookie_store::RawCookie; use network_interface::{NetworkInterface, NetworkInterfaceConfig}; use redirect::RedirectFollower; @@ -333,12 +334,8 @@ fn run(args: Cli) -> Result { s.save_headers(&headers)?; let mut cookie_jar = cookie_jar.lock().unwrap(); - for (cookie, cookie_url) in s.cookies()? { - match cookie_jar.insert_raw(&cookie, &cookie_url) { - Ok(..) | Err(CookieError::Expired) | Err(CookieError::DomainMismatch) => {} - Err(err) => return Err(err.into()), - } - } + *cookie_jar = CookieStore::from_cookies(s.cookies(), false)?; + if let Some(cookie) = headers.remove(COOKIE) { for cookie in RawCookie::split_parse(cookie.to_str()?) { cookie_jar.insert_raw(&cookie?, &url)?; diff --git a/src/session.rs b/src/session.rs index 1bed1dff..6cff8c2e 100644 --- a/src/session.rs +++ b/src/session.rs @@ -263,45 +263,39 @@ impl Session { } } - pub fn cookies(&self) -> Result> { + pub fn cookies(&self) -> impl Iterator>> + '_ { match &self.content.cookies { Cookies::Map(_) => unreachable!(), - Cookies::List(cookies) => cookies - .iter() - .map(|cookie| { - let mut cookie_builder = - cookie_store::RawCookie::build(cookie.name.clone(), cookie.value.clone()); + Cookies::List(cookies) => cookies.iter().map(|cookie| { + let mut cookie_builder = + cookie_store::RawCookie::build(cookie.name.clone(), cookie.value.clone()); - if let Some(expires) = cookie.expires { - cookie_builder = cookie_builder - .expires(time::OffsetDateTime::from_unix_timestamp(expires)?); - } - if let Some(path) = &cookie.path { - cookie_builder = cookie_builder.path(path); - } - if let Some(secure) = cookie.secure { - cookie_builder = cookie_builder.secure(secure); - } + if let Some(expires) = cookie.expires { + cookie_builder = + cookie_builder.expires(time::OffsetDateTime::from_unix_timestamp(expires)?); + } + if let Some(path) = &cookie.path { + cookie_builder = cookie_builder.path(path.clone()); + } + if let Some(secure) = cookie.secure { + cookie_builder = cookie_builder.secure(secure); + } - let mut cookie_url = self.url.clone(); - if let Some(domain) = &cookie.domain { - cookie_url = format!("http://{domain}").parse()?; - // The cookie's domain attribute cannot be an IP address. - // See https://stackoverflow.com/a/30676300/5915221 - if let Some(url::Host::Domain(_)) = cookie_url.host() { - cookie_builder = cookie_builder.domain(domain.clone()); - } + let mut cookie_url = self.url.clone(); + if let Some(domain) = &cookie.domain { + cookie_url = format!("http://{domain}").parse()?; + // The cookie's domain attribute cannot be an IP address. + // See https://stackoverflow.com/a/30676300/5915221 + if let Some(url::Host::Domain(_)) = cookie_url.host() { + cookie_builder = cookie_builder.domain(domain.clone()); } + } - Ok(( - cookie_store::Cookie::try_from_raw_cookie( - &cookie_builder.finish(), - &cookie_url, - )?, - cookie_url, - )) - }) - .collect(), + Ok(cookie_store::Cookie::try_from_raw_cookie( + &cookie_builder.finish(), + &cookie_url, + )?) + }), } } @@ -410,10 +404,11 @@ mod tests { session.headers()?.get("hello"), Some(&HeaderValue::from_static("world")), ); - let cookies = session.cookies()?; - assert_eq!(cookies[0].0.name_value(), ("baz", "quux")); - assert_eq!(cookies[0].0.path(), Some("/")); - assert_eq!(cookies[0].0.secure(), Some(false)); + + let cookies = session.cookies().collect::>>()?; + assert_eq!(cookies[0].name_value(), ("baz", "quux")); + assert_eq!(cookies[0].path(), Some("/")); + assert_eq!(cookies[0].secure(), Some(false)); assert_eq!(session.content.auth, Auth::default()); Ok(()) @@ -439,10 +434,10 @@ mod tests { session.headers()?.get("hello"), Some(&HeaderValue::from_static("world")), ); - let cookies = session.cookies()?; - assert_eq!(cookies[0].0.name_value(), ("baz", "quux")); - assert_eq!(cookies[0].0.path(), Some("/")); - assert_eq!(cookies[0].0.secure(), Some(false)); + let cookies = session.cookies().collect::>>()?; + assert_eq!(cookies[0].name_value(), ("baz", "quux")); + assert_eq!(cookies[0].path(), Some("/")); + assert_eq!(cookies[0].secure(), Some(false)); assert_eq!( session.content.auth, Auth { @@ -533,17 +528,17 @@ mod tests { } "#})?; - let cookies = session.cookies()?; + let cookies = session.cookies().collect::>>()?; - assert_eq!(cookies[0].0.name_value(), ("baz", "quux")); - assert_eq!(cookies[0].0.path(), Some("/")); - assert_eq!(cookies[0].0.secure(), Some(false)); - assert_eq!(cookies[0].0.domain(), Some("example.com")); + assert_eq!(cookies[0].name_value(), ("baz", "quux")); + assert_eq!(cookies[0].path(), Some("/")); + assert_eq!(cookies[0].secure(), Some(false)); + assert_eq!(cookies[0].domain(), Some("example.com")); - assert_eq!(cookies[1].0.name_value(), ("foo", "bar")); - assert_eq!(cookies[1].0.path(), Some("/")); - assert_eq!(cookies[1].0.secure(), Some(false)); - assert_eq!(cookies[1].0.domain(), None); + assert_eq!(cookies[1].name_value(), ("foo", "bar")); + assert_eq!(cookies[1].path(), Some("/")); + assert_eq!(cookies[1].secure(), Some(false)); + assert_eq!(cookies[1].domain(), None); Ok(()) } From 3263ee2fa9617b9acce98bc9d9c05c3d8bd12d67 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sat, 24 Jun 2023 21:08:54 +0100 Subject: [PATCH 23/28] store url ref in session --- src/main.rs | 2 +- src/session.rs | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/main.rs b/src/main.rs index 8b9fb7cc..fae08944 100644 --- a/src/main.rs +++ b/src/main.rs @@ -315,7 +315,7 @@ fn run(args: Cli) -> Result { let mut session = match &args.session { Some(name_or_path) => Some( - Session::load_session(url.clone(), name_or_path.clone(), args.is_session_read_only) + Session::load_session(&url, name_or_path.clone(), args.is_session_read_only) .with_context(|| { format!("couldn't load session {:?}", name_or_path.to_string_lossy()) })?, diff --git a/src/session.rs b/src/session.rs index 6cff8c2e..0f02508d 100644 --- a/src/session.rs +++ b/src/session.rs @@ -146,21 +146,21 @@ impl Content { } } -pub struct Session { - url: Url, +pub struct Session<'a> { + url: &'a Url, pub path: PathBuf, read_only: bool, content: Content, } -impl Session { - pub fn load_session(url: Url, mut name_or_path: OsString, read_only: bool) -> Result { +impl<'a> Session<'a> { + pub fn load_session(url: &'a Url, mut name_or_path: OsString, read_only: bool) -> Result { let path = if is_path(&name_or_path) { PathBuf::from(name_or_path) } else { let mut path = config_dir() .context("couldn't get config directory")? - .join::(["sessions", &path_from_url(&url)?].iter().collect()); + .join::(["sessions", &path_from_url(url)?].iter().collect()); name_or_path.push(".json"); path.push(name_or_path); path @@ -299,9 +299,9 @@ impl Session { } } - pub fn save_cookies<'a, I>(&mut self, cookies: I) + pub fn save_cookies<'b, I>(&mut self, cookies: I) where - I: Iterator>, + I: Iterator>, { let session_cookies = match self.content.cookies { Cookies::Map(_) => unreachable!(), @@ -374,9 +374,12 @@ mod tests { use anyhow::Result; use reqwest::header::HeaderValue; + static SESSION_URL: once_cell::sync::Lazy = + once_cell::sync::Lazy::new(|| Url::parse("http://example.net").unwrap()); + fn load_session_from_str(s: &str) -> Result { Ok(Session { - url: Url::parse("http://example.net").unwrap(), + url: &SESSION_URL, content: serde_json::from_str::(s)?.migrate(), path: PathBuf::new(), read_only: false, From 08ad9c056ad541604ec9549bd158db20cd02a4be Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sun, 16 Jul 2023 15:46:06 +0100 Subject: [PATCH 24/28] pass cloned url to session --- src/main.rs | 2 +- src/session.rs | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/main.rs b/src/main.rs index fae08944..8b9fb7cc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -315,7 +315,7 @@ fn run(args: Cli) -> Result { let mut session = match &args.session { Some(name_or_path) => Some( - Session::load_session(&url, name_or_path.clone(), args.is_session_read_only) + Session::load_session(url.clone(), name_or_path.clone(), args.is_session_read_only) .with_context(|| { format!("couldn't load session {:?}", name_or_path.to_string_lossy()) })?, diff --git a/src/session.rs b/src/session.rs index 0f02508d..cf05865e 100644 --- a/src/session.rs +++ b/src/session.rs @@ -146,21 +146,21 @@ impl Content { } } -pub struct Session<'a> { - url: &'a Url, +pub struct Session { + url: Url, pub path: PathBuf, read_only: bool, content: Content, } -impl<'a> Session<'a> { - pub fn load_session(url: &'a Url, mut name_or_path: OsString, read_only: bool) -> Result { +impl Session { + pub fn load_session(url: Url, mut name_or_path: OsString, read_only: bool) -> Result { let path = if is_path(&name_or_path) { PathBuf::from(name_or_path) } else { let mut path = config_dir() .context("couldn't get config directory")? - .join::(["sessions", &path_from_url(url)?].iter().collect()); + .join::(["sessions", &path_from_url(&url)?].iter().collect()); name_or_path.push(".json"); path.push(name_or_path); path @@ -374,12 +374,9 @@ mod tests { use anyhow::Result; use reqwest::header::HeaderValue; - static SESSION_URL: once_cell::sync::Lazy = - once_cell::sync::Lazy::new(|| Url::parse("http://example.net").unwrap()); - fn load_session_from_str(s: &str) -> Result { Ok(Session { - url: &SESSION_URL, + url: Url::parse("http://example.net")?, content: serde_json::from_str::(s)?.migrate(), path: PathBuf::new(), read_only: false, From 73cf5040c7acc57986f51f8f04f7916397e71905 Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sun, 6 Aug 2023 21:25:06 +0100 Subject: [PATCH 25/28] replace path array with multiple joins Co-authored-by: Jan Verbeek --- src/session.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/session.rs b/src/session.rs index cf05865e..8d97d776 100644 --- a/src/session.rs +++ b/src/session.rs @@ -160,7 +160,8 @@ impl Session { } else { let mut path = config_dir() .context("couldn't get config directory")? - .join::(["sessions", &path_from_url(&url)?].iter().collect()); + .join("sessions") + .join(path_from_url(&url)?); name_or_path.push(".json"); path.push(name_or_path); path From 159dec7f9d2b23f3920d5e558c0d18eb68f222cb Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sat, 12 Aug 2023 23:03:10 +0100 Subject: [PATCH 26/28] migrate cookies using localhost.local as domain --- src/session.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/session.rs b/src/session.rs index 8d97d776..2c798ade 100644 --- a/src/session.rs +++ b/src/session.rs @@ -142,6 +142,16 @@ impl Content { ) } + // HTTPie appends .local to cookies from localhost. + // See https://github.com/psf/requests/issues/5388 + if let Cookies::List(ref mut cookies) = self.cookies { + for mut cookie in cookies { + if cookie.domain.as_deref() == Some("localhost.local") { + cookie.domain = Some("localhost".to_string()); + } + } + } + self } } @@ -523,6 +533,14 @@ mod tests { "path": "/", "secure": false, "domain": null + }, + { + "domain": "localhost.local", + "expires": null, + "name": "hello", + "path": "/cookies", + "secure": false, + "value": "world" } ], "headers": [] @@ -541,6 +559,11 @@ mod tests { assert_eq!(cookies[1].secure(), Some(false)); assert_eq!(cookies[1].domain(), None); + assert_eq!(cookies[2].name_value(), ("hello", "world")); + assert_eq!(cookies[2].path(), Some("/cookies")); + assert_eq!(cookies[2].secure(), Some(false)); + assert_eq!(cookies[2].domain(), Some("localhost")); + Ok(()) } } From 1551869f7891414015943db935d95342ef7ec87c Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sat, 12 Aug 2023 23:05:15 +0100 Subject: [PATCH 27/28] fix clippy error --- src/session.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/session.rs b/src/session.rs index 2c798ade..9888111a 100644 --- a/src/session.rs +++ b/src/session.rs @@ -145,7 +145,7 @@ impl Content { // HTTPie appends .local to cookies from localhost. // See https://github.com/psf/requests/issues/5388 if let Cookies::List(ref mut cookies) = self.cookies { - for mut cookie in cookies { + for cookie in cookies { if cookie.domain.as_deref() == Some("localhost.local") { cookie.domain = Some("localhost".to_string()); } From cc2b4067148391b77f46bc1b4fe9d815c4c0957c Mon Sep 17 00:00:00 2001 From: Mohamed Daahir Date: Sat, 9 Sep 2023 19:15:10 +0100 Subject: [PATCH 28/28] add context to cookie load errors --- src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index ca07d5cf..fbb32030 100644 --- a/src/main.rs +++ b/src/main.rs @@ -334,7 +334,8 @@ fn run(args: Cli) -> Result { s.save_headers(&headers)?; let mut cookie_jar = cookie_jar.lock().unwrap(); - *cookie_jar = CookieStore::from_cookies(s.cookies(), false)?; + *cookie_jar = CookieStore::from_cookies(s.cookies(), false) + .context("Failed to load cookies from session file")?; if let Some(cookie) = headers.remove(COOKIE) { for cookie in RawCookie::split_parse(cookie.to_str()?) {