From cffdfa90e3300c84116dfd48145e2779a831af35 Mon Sep 17 00:00:00 2001 From: Jeremy Andrews Date: Sat, 4 Dec 2021 10:41:02 +0100 Subject: [PATCH 1/4] initial re-implement drupal memcache with isahc --- Cargo.toml | 1 + examples/alternative_client/main.rs | 214 ++++++++++++++++++++++++++++ examples/alternative_client/util.rs | 111 +++++++++++++++ src/goose.rs | 5 +- 4 files changed, 330 insertions(+), 1 deletion(-) create mode 100644 examples/alternative_client/main.rs create mode 100644 examples/alternative_client/util.rs diff --git a/Cargo.toml b/Cargo.toml index 8c1a25ec..f792ffd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,3 +67,4 @@ httpmock = "0.6" serial_test = "0.5" native-tls = "0.2" rustls = "0.19" +isahc = "1.6" diff --git a/examples/alternative_client/main.rs b/examples/alternative_client/main.rs new file mode 100644 index 00000000..88cd90fe --- /dev/null +++ b/examples/alternative_client/main.rs @@ -0,0 +1,214 @@ +//! Conversion of drupal_memcache example to use the Isahc http client instead +//! of Reqwest. +//! +//! To run, you must set up the load test environment as described in the +//! drupal_memcache example. +//! +//! ## License +//! +//! Copyright 2021 Jeremy Andrews +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +mod util; + +use goose::metrics::{GooseRawRequest, GooseRequestMetric}; +use goose::prelude::*; + +use isahc::prelude::*; +use rand::Rng; +use regex::Regex; + +#[tokio::main] +async fn main() -> Result<(), GooseError> { + GooseAttack::initialize()? + .register_taskset( + taskset!("AnonBrowsingUser") + .set_weight(4)? + .register_task( + task!(drupal_memcache_front_page) + .set_weight(15)? + .set_name("(Anon) front page"), + ) + .register_task( + task!(drupal_memcache_node_page) + .set_weight(10)? + .set_name("(Anon) node page"), + ), /* + .register_task( + task!(drupal_memcache_profile_page) + .set_weight(3)? + .set_name("(Anon) user page"), + ), + */ + ) + /* + .register_taskset( + taskset!("AuthBrowsingUser") + .set_weight(1)? + .register_task( + task!(drupal_memcache_login) + .set_on_start() + .set_name("(Auth) login"), + ) + .register_task( + task!(drupal_memcache_front_page) + .set_weight(15)? + .set_name("(Auth) front page"), + ) + .register_task( + task!(drupal_memcache_node_page) + .set_weight(10)? + .set_name("(Auth) node page"), + ) + .register_task( + task!(drupal_memcache_profile_page) + .set_weight(3)? + .set_name("(Auth) user page"), + ) + .register_task( + task!(drupal_memcache_post_comment) + .set_weight(3)? + .set_name("(Auth) comment form"), + ), + ) + */ + .execute() + .await? + .print(); + + Ok(()) +} + +/// View the front page. +async fn drupal_memcache_front_page(user: &mut GooseUser) -> GooseTaskResult { + let started = std::time::Instant::now(); + let url = user.build_url("/").unwrap(); + let response = isahc::get_async(&url).await; + + match response { + Ok(mut r) => { + // Copy the headers so we have them for logging if there are errors. + let headers = &r.headers().clone(); + match r.text().await { + Ok(t) => { + let status = r.status(); + let mut request_metric = util::build_request_metric( + user, + GooseMethod::Get, + &url, + Some(headers), + "", + started, + status, + ); + request_metric.name = "/".to_string(); + user.send_request_metric_to_parent(request_metric)?; + + // Load all static assets. + util::load_static_assets(user, Some(headers), &t).await; + } + Err(e) => { + let status = r.status(); + let mut request_metric = util::build_request_metric( + user, + GooseMethod::Get, + &url, + Some(headers), + &e.to_string(), + started, + status, + ); + request_metric.name = "/".to_string(); + user.send_request_metric_to_parent(request_metric.clone())?; + return user.set_failure( + &format!("front_page: failed to parse page: {}", e), + &mut request_metric, + Some(headers), + None, + ); + } + } + } + Err(e) => { + let mut request_metric = util::build_request_metric( + user, + GooseMethod::Get, + &url, + None, + &e.to_string(), + started, + http::StatusCode::from_u16(500).unwrap(), + ); + request_metric.name = "/".to_string(); + user.send_request_metric_to_parent(request_metric.clone())?; + return user.set_failure( + &format!("front_page: no response from server: {}", e), + &mut request_metric, + None, + None, + ); + } + } + + Ok(()) +} + +/// View a node from 1 to 10,000, created by preptest.sh. +async fn drupal_memcache_node_page(user: &mut GooseUser) -> GooseTaskResult { + let started = std::time::Instant::now(); + let nid = rand::thread_rng().gen_range(1..10_000); + let url = user.build_url(format!("/node/{}", &nid).as_str()).unwrap(); + let response = isahc::get_async(&url).await; + + match response { + Ok(mut r) => { + // Copy the headers so we have them for logging if there are errors. + let headers = &r.headers().clone(); + let status = r.status(); + let mut request_metric = util::build_request_metric( + user, + GooseMethod::Get, + &url, + Some(headers), + "", + started, + status, + ); + request_metric.name = "/node/{}".to_string(); + r.consume().await.unwrap(); + user.send_request_metric_to_parent(request_metric)?; + } + Err(e) => { + let mut request_metric = util::build_request_metric( + user, + GooseMethod::Get, + &url, + None, + &e.to_string(), + started, + http::StatusCode::from_u16(500).unwrap(), + ); + request_metric.name = "/node/{}".to_string(); + user.send_request_metric_to_parent(request_metric.clone())?; + return user.set_failure( + &format!("front_page: no response from server: {}", e), + &mut request_metric, + None, + None, + ); + } + } + + Ok(()) +} diff --git a/examples/alternative_client/util.rs b/examples/alternative_client/util.rs new file mode 100644 index 00000000..d70ac558 --- /dev/null +++ b/examples/alternative_client/util.rs @@ -0,0 +1,111 @@ +//! Helper functions to simplify the use of the Isahc HTTP client. +//! +//! ## License +//! +//! Copyright 2021 Jeremy Andrews +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +use goose::metrics::{GooseRawRequest, GooseRequestMetric}; +use goose::prelude::*; + +use isahc::prelude::*; +//use rand::Rng; +use regex::Regex; + +pub(crate) fn build_request_metric( + user: &mut GooseUser, + method: GooseMethod, + url: &str, + headers: Option<&http::HeaderMap>, + error: &str, + started: std::time::Instant, + status: http::StatusCode, +) -> GooseRequestMetric { + let mut h: Vec = Vec::new(); + for header in headers { + h.push(format!("{:?}", header)); + } + + // Record information about the request. + let raw_request = GooseRawRequest { + method, + url: url.to_string(), + headers: h, + //@TODO + body: "".to_string(), + }; + + GooseRequestMetric { + elapsed: user.started.elapsed().as_millis() as u64, + raw: raw_request, + name: url.to_string(), + //@TODO + final_url: url.to_string(), + //@TODO + redirected: false, + response_time: started.elapsed().as_millis() as u64, + status_code: status.as_u16(), + success: status.is_success(), + update: false, + user: user.weighted_users_index, + error: error.to_string(), + // Coordinated Omission is disabled when using a custom client. + coordinated_omission_elapsed: 0, + user_cadence: 0, + } +} + +pub async fn load_static_assets( + user: &mut GooseUser, + headers: Option<&http::HeaderMap>, + text: &str, +) { + let re = Regex::new(r#"src="(.*?)""#).unwrap(); + for url in re.captures_iter(text) { + if url[1].contains("/misc") || url[1].contains("/themes") { + let started = std::time::Instant::now(); + let url = user.build_url("/").unwrap(); + match isahc::get_async(&url).await { + Ok(mut r) => { + let status = r.status(); + let mut request_metric = build_request_metric( + user, + GooseMethod::Get, + &url, + headers, + "", + started, + status, + ); + r.consume().await.unwrap(); + request_metric.name = "static asset".to_string(); + let _ = user.send_request_metric_to_parent(request_metric); + } + Err(e) => { + let mut request_metric = build_request_metric( + user, + GooseMethod::Get, + &url, + headers, + &e.to_string(), + started, + http::StatusCode::from_u16(500).unwrap(), + ); + request_metric.name = "static asset".to_string(); + let _ = user.send_request_metric_to_parent(request_metric.clone()); + } + }; + } + } +} diff --git a/src/goose.rs b/src/goose.rs index 2a42870e..048769cb 100644 --- a/src/goose.rs +++ b/src/goose.rs @@ -1774,7 +1774,10 @@ impl GooseUser { } } - fn send_request_metric_to_parent(&self, request_metric: GooseRequestMetric) -> GooseTaskResult { + pub fn send_request_metric_to_parent( + &self, + request_metric: GooseRequestMetric, + ) -> GooseTaskResult { // If requests-file is enabled, send a copy of the raw request to the logger thread. if !self.config.request_log.is_empty() { if let Some(logger) = self.logger.as_ref() { From 8fd093d5daaddf693c0e3fa363933ec206b21cc1 Mon Sep 17 00:00:00 2001 From: Jeremy Andrews Date: Sat, 4 Dec 2021 13:04:37 +0100 Subject: [PATCH 2/4] fix clippy errors --- .github/workflows/CI.yml | 2 +- Cargo.toml | 4 ++-- examples/alternative_client/main.rs | 4 +--- examples/alternative_client/util.rs | 6 ++++-- examples/closure.rs | 2 +- examples/drupal_memcache.rs | 4 ++-- examples/session.rs | 2 +- examples/simple.rs | 4 ++-- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5c9e347f..2511469e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -30,6 +30,6 @@ jobs: - name: Build run: cargo build --verbose --all-features - name: Docs - run: cargo rustdoc --lib --all-features --examples + run: cargo rustdoc --lib --all-features - name: Run tests run: cargo test --verbose --all-features diff --git a/Cargo.toml b/Cargo.toml index f792ffd5..bda32c6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ rustc_version = "0.4" [dev-dependencies] httpmock = "0.6" -serial_test = "0.5" +isahc = "1.6" native-tls = "0.2" rustls = "0.19" -isahc = "1.6" +serial_test = "0.5" diff --git a/examples/alternative_client/main.rs b/examples/alternative_client/main.rs index 88cd90fe..ddc0472e 100644 --- a/examples/alternative_client/main.rs +++ b/examples/alternative_client/main.rs @@ -12,7 +12,7 @@ //! you may not use this file except in compliance with the License. //! You may obtain a copy of the License at //! -//! http://www.apache.org/licenses/LICENSE-2.0 +//! //! //! Unless required by applicable law or agreed to in writing, software //! distributed under the License is distributed on an "AS IS" BASIS, @@ -22,12 +22,10 @@ mod util; -use goose::metrics::{GooseRawRequest, GooseRequestMetric}; use goose::prelude::*; use isahc::prelude::*; use rand::Rng; -use regex::Regex; #[tokio::main] async fn main() -> Result<(), GooseError> { diff --git a/examples/alternative_client/util.rs b/examples/alternative_client/util.rs index d70ac558..f67d5282 100644 --- a/examples/alternative_client/util.rs +++ b/examples/alternative_client/util.rs @@ -33,8 +33,10 @@ pub(crate) fn build_request_metric( status: http::StatusCode, ) -> GooseRequestMetric { let mut h: Vec = Vec::new(); - for header in headers { - h.push(format!("{:?}", header)); + if let Some(hs) = headers { + for header in hs { + h.push(format!("{:?}", header)); + } } // Record information about the request. diff --git a/examples/closure.rs b/examples/closure.rs index 3519ba5e..6c450060 100644 --- a/examples/closure.rs +++ b/examples/closure.rs @@ -8,7 +8,7 @@ //! you may not use this file except in compliance with the License. //! You may obtain a copy of the License at //! -//! http://www.apache.org/licenses/LICENSE-2.0 +//! //! //! Unless required by applicable law or agreed to in writing, software //! distributed under the License is distributed on an "AS IS" BASIS, diff --git a/examples/drupal_memcache.rs b/examples/drupal_memcache.rs index bad120e5..503e9725 100644 --- a/examples/drupal_memcache.rs +++ b/examples/drupal_memcache.rs @@ -1,5 +1,5 @@ //! Conversion of Locust load test used for the Drupal memcache module, from -//! https://github.com/tag1consulting/drupal-loadtest/ +//! //! //! To run, you must set up the load test environment as described in the above //! repository, and then run the example. You'll need to set --host and may want @@ -14,7 +14,7 @@ //! you may not use this file except in compliance with the License. //! You may obtain a copy of the License at //! -//! http://www.apache.org/licenses/LICENSE-2.0 +//! //! //! Unless required by applicable law or agreed to in writing, software //! distributed under the License is distributed on an "AS IS" BASIS, diff --git a/examples/session.rs b/examples/session.rs index 31b60215..3ff28f0a 100644 --- a/examples/session.rs +++ b/examples/session.rs @@ -9,7 +9,7 @@ //! you may not use this file except in compliance with the License. //! You may obtain a copy of the License at //! -//! http://www.apache.org/licenses/LICENSE-2.0 +//! //! //! Unless required by applicable law or agreed to in writing, software //! distributed under the License is distributed on an "AS IS" BASIS, diff --git a/examples/simple.rs b/examples/simple.rs index 168d1123..beca2b87 100644 --- a/examples/simple.rs +++ b/examples/simple.rs @@ -1,5 +1,5 @@ //! Simple Goose load test example. Duplicates the simple example on the -//! Locust project page (https://locust.io/). +//! Locust project page (). //! //! ## License //! @@ -9,7 +9,7 @@ //! you may not use this file except in compliance with the License. //! You may obtain a copy of the License at //! -//! http://www.apache.org/licenses/LICENSE-2.0 +//! //! //! Unless required by applicable law or agreed to in writing, software //! distributed under the License is distributed on an "AS IS" BASIS, From dd76d025d4cd7861c81cf3341158efe6b7f92885 Mon Sep 17 00:00:00 2001 From: Jeremy Andrews Date: Sun, 5 Dec 2021 07:55:45 +0100 Subject: [PATCH 3/4] complete anonymous portion --- examples/alternative_client/main.rs | 68 ++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/examples/alternative_client/main.rs b/examples/alternative_client/main.rs index ddc0472e..10a42e68 100644 --- a/examples/alternative_client/main.rs +++ b/examples/alternative_client/main.rs @@ -42,13 +42,12 @@ async fn main() -> Result<(), GooseError> { task!(drupal_memcache_node_page) .set_weight(10)? .set_name("(Anon) node page"), - ), /* - .register_task( - task!(drupal_memcache_profile_page) - .set_weight(3)? - .set_name("(Anon) user page"), - ), - */ + ) + .register_task( + task!(drupal_memcache_profile_page) + .set_weight(3)? + .set_name("(Anon) user page"), + ), ) /* .register_taskset( @@ -130,7 +129,7 @@ async fn drupal_memcache_front_page(user: &mut GooseUser) -> GooseTaskResult { request_metric.name = "/".to_string(); user.send_request_metric_to_parent(request_metric.clone())?; return user.set_failure( - &format!("front_page: failed to parse page: {}", e), + &format!("front page: failed to parse page: {}", e), &mut request_metric, Some(headers), None, @@ -151,7 +150,7 @@ async fn drupal_memcache_front_page(user: &mut GooseUser) -> GooseTaskResult { request_metric.name = "/".to_string(); user.send_request_metric_to_parent(request_metric.clone())?; return user.set_failure( - &format!("front_page: no response from server: {}", e), + &format!("front page: no response from server: {}", e), &mut request_metric, None, None, @@ -200,7 +199,56 @@ async fn drupal_memcache_node_page(user: &mut GooseUser) -> GooseTaskResult { request_metric.name = "/node/{}".to_string(); user.send_request_metric_to_parent(request_metric.clone())?; return user.set_failure( - &format!("front_page: no response from server: {}", e), + &format!("node page: no response from server: {}", e), + &mut request_metric, + None, + None, + ); + } + } + + Ok(()) +} + +/// View a profile from 2 to 5,001, created by preptest.sh. +async fn drupal_memcache_profile_page(user: &mut GooseUser) -> GooseTaskResult { + let started = std::time::Instant::now(); + let uid = rand::thread_rng().gen_range(2..5_001); + let url = user.build_url(format!("/user/{}", &uid).as_str()).unwrap(); + let response = isahc::get_async(&url).await; + + match response { + Ok(mut r) => { + // Copy the headers so we have them for logging if there are errors. + let headers = &r.headers().clone(); + let status = r.status(); + let mut request_metric = util::build_request_metric( + user, + GooseMethod::Get, + &url, + Some(headers), + "", + started, + status, + ); + request_metric.name = "/user/{}".to_string(); + r.consume().await.unwrap(); + user.send_request_metric_to_parent(request_metric)?; + } + Err(e) => { + let mut request_metric = util::build_request_metric( + user, + GooseMethod::Get, + &url, + None, + &e.to_string(), + started, + http::StatusCode::from_u16(500).unwrap(), + ); + request_metric.name = "/user/{}".to_string(); + user.send_request_metric_to_parent(request_metric.clone())?; + return user.set_failure( + &format!("user page: no response from server: {}", e), &mut request_metric, None, None, From 0f5d7cd0917d47fe7d767035f4c47c08831db943 Mon Sep 17 00:00:00 2001 From: Jeremy Andrews Date: Sun, 5 Dec 2021 08:22:11 +0100 Subject: [PATCH 4/4] add log in and authenticated users --- examples/alternative_client/main.rs | 156 ++++++++++++++++++++++++++-- 1 file changed, 149 insertions(+), 7 deletions(-) diff --git a/examples/alternative_client/main.rs b/examples/alternative_client/main.rs index 10a42e68..9e607f16 100644 --- a/examples/alternative_client/main.rs +++ b/examples/alternative_client/main.rs @@ -26,6 +26,7 @@ use goose::prelude::*; use isahc::prelude::*; use rand::Rng; +use regex::Regex; #[tokio::main] async fn main() -> Result<(), GooseError> { @@ -49,7 +50,6 @@ async fn main() -> Result<(), GooseError> { .set_name("(Anon) user page"), ), ) - /* .register_taskset( taskset!("AuthBrowsingUser") .set_weight(1)? @@ -72,14 +72,15 @@ async fn main() -> Result<(), GooseError> { task!(drupal_memcache_profile_page) .set_weight(3)? .set_name("(Auth) user page"), - ) - .register_task( - task!(drupal_memcache_post_comment) - .set_weight(3)? - .set_name("(Auth) comment form"), ), + /* + .register_task( + task!(drupal_memcache_post_comment) + .set_weight(3)? + .set_name("(Auth) comment form"), + ), + */ ) - */ .execute() .await? .print(); @@ -258,3 +259,144 @@ async fn drupal_memcache_profile_page(user: &mut GooseUser) -> GooseTaskResult { Ok(()) } + +/// Log in. +async fn drupal_memcache_login(user: &mut GooseUser) -> GooseTaskResult { + let started = std::time::Instant::now(); + let url = user.build_url("/user").unwrap(); + let response = isahc::get_async(&url).await; + + match response { + Ok(mut r) => { + // Copy the headers so we have them for logging if there are errors. + let headers = &r.headers().clone(); + let status = r.status(); + let mut request_metric = util::build_request_metric( + user, + GooseMethod::Get, + &url, + Some(headers), + "", + started, + status, + ); + request_metric.name = "/user".to_string(); + user.send_request_metric_to_parent(request_metric.clone())?; + + match r.text().await { + Ok(html) => { + let re = Regex::new(r#"name="form_build_id" value=['"](.*?)['"]"#).unwrap(); + let form_build_id = match re.captures(&html) { + Some(f) => f, + None => { + // This will automatically get written to the error log if enabled, and will + // be displayed to stdout if `-v` is enabled when running the load test. + return user.set_failure( + "login: no form_build_id on page: /user page", + &mut request_metric, + Some(headers), + Some(&html), + ); + } + }; + + // Log the user in. + let uid: usize = rand::thread_rng().gen_range(3..5_002); + let username = format!("user{}", uid); + let params = format!( + r#"{{ + "name": "{}", + "pass": "12345", + "form_build_id": "{}", + "form_id": "user_login", + "op": "Log+in", + }}"#, + username.as_str(), + &form_build_id[1], + ); + let started = std::time::Instant::now(); + let response = isahc::post_async(&url, params.as_str()).await; + match response { + Ok(mut r) => { + // Copy the headers so we have them for logging if there are errors. + let headers = &r.headers().clone(); + let status = r.status(); + let mut request_metric = util::build_request_metric( + user, + GooseMethod::Post, + &url, + Some(headers), + "", + started, + status, + ); + request_metric.name = "/user".to_string(); + r.consume().await.unwrap(); + user.send_request_metric_to_parent(request_metric)?; + } + Err(e) => { + let mut request_metric = util::build_request_metric( + user, + GooseMethod::Post, + &url, + None, + &e.to_string(), + started, + http::StatusCode::from_u16(500).unwrap(), + ); + request_metric.name = "/user".to_string(); + user.send_request_metric_to_parent(request_metric.clone())?; + return user.set_failure( + &format!("user page: no response from server: {}", e), + &mut request_metric, + None, + None, + ); + } + } + } + Err(e) => { + let status = r.status(); + let mut request_metric = util::build_request_metric( + user, + GooseMethod::Get, + &url, + Some(headers), + &e.to_string(), + started, + status, + ); + request_metric.name = "/".to_string(); + user.send_request_metric_to_parent(request_metric.clone())?; + return user.set_failure( + &format!("login: failed to parse page: {}", e), + &mut request_metric, + Some(headers), + None, + ); + } + } + } + Err(e) => { + let mut request_metric = util::build_request_metric( + user, + GooseMethod::Get, + &url, + None, + &e.to_string(), + started, + http::StatusCode::from_u16(500).unwrap(), + ); + request_metric.name = "/user".to_string(); + user.send_request_metric_to_parent(request_metric.clone())?; + return user.set_failure( + &format!("login: no response from server: {}", e), + &mut request_metric, + None, + None, + ); + } + } + + Ok(()) +}