Skip to content

Commit

Permalink
addressing review comments, and updating test values
Browse files Browse the repository at this point in the history
  • Loading branch information
@igor-aptos
igor-aptos committed Jul 12, 2024
1 parent 525ae5c commit 389126d
Show file tree
Hide file tree
Showing 5 changed files with 33 additions and 31 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/forge-stable.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ jobs:
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-env-workload-sweep-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 1600 # Run for 26 minutes (4 tests, each for 400 seconds)
FORGE_RUNNER_DURATION_SECS: 2000 # Run for 33 minutes (5 tests, each for 400 seconds)
FORGE_TEST_SUITE: realistic_env_workload_sweep
POST_TO_SLACK: true

Expand Down
22 changes: 11 additions & 11 deletions testsuite/forge-cli/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ use aptos_testcases::{
fullnode_reboot_stress_test::FullNodeRebootStressTest,
generate_traffic,
load_vs_perf_benchmark::{
ContinuousTraffic, LoadVsPerfBenchmark, TransactionWorkload, Workloads,
BackgroundTraffic, LoadVsPerfBenchmark, TransactionWorkload, Workloads,
},
modifiers::{CpuChaosTest, ExecutionDelayConfig, ExecutionDelayTest},
multi_region_network_test::{
Expand Down Expand Up @@ -1076,7 +1076,7 @@ fn realistic_env_load_sweep_test() -> ForgeConfig {
},
)
.collect(),
continuous_traffic: None,
background_traffic: None,
})
}

Expand All @@ -1086,11 +1086,11 @@ fn realistic_env_workload_sweep_test() -> ForgeConfig {
workloads: Workloads::TRANSACTIONS(vec![
TransactionWorkload::new(TransactionTypeArg::CoinTransfer, 20000),
TransactionWorkload::new(TransactionTypeArg::NoOp, 20000).with_num_modules(100),
TransactionWorkload::new(TransactionTypeArg::ModifyGlobalResource, 20000)
TransactionWorkload::new(TransactionTypeArg::ModifyGlobalResource, 10000)
.with_transactions_per_account(1),
TransactionWorkload::new(TransactionTypeArg::TokenV2AmbassadorMint, 30000)
TransactionWorkload::new(TransactionTypeArg::TokenV2AmbassadorMint, 20000)
.with_unique_senders(),
TransactionWorkload::new(TransactionTypeArg::PublishPackage, 1000)
TransactionWorkload::new(TransactionTypeArg::PublishPackage, 500)
.with_transactions_per_account(1),
]),
// Investigate/improve to make latency more predictable on different workloads
Expand All @@ -1099,7 +1099,7 @@ fn realistic_env_workload_sweep_test() -> ForgeConfig {
(7000, 100, 0.3, 0.5, 0.5, 0.5),
(2000, 300, 0.3, 1.0, 0.6, 1.0),
(3200, 500, 0.3, 1.5, 0.7, 0.7),
(150, 30, 0.5, 1.0, 1.5, 1.0),
(30, 5, 0.5, 1.0, 1.5, 1.0),
]
.into_iter()
.map(
Expand Down Expand Up @@ -1129,7 +1129,7 @@ fn realistic_env_workload_sweep_test() -> ForgeConfig {
},
)
.collect(),
continuous_traffic: None,
background_traffic: None,
})
}

Expand All @@ -1148,7 +1148,7 @@ fn realistic_env_fairness_workload_sweep() -> ForgeConfig {
.with_transactions_per_account(1),
]),
criteria: Vec::new(),
continuous_traffic: Some(ContinuousTraffic {
background_traffic: Some(BackgroundTraffic {
traffic: EmitJobRequest::default()
.num_accounts_mode(NumAccountsMode::TransactionsPerAccount(1))
.mode(EmitJobMode::ConstTps { tps: 40 }),
Expand Down Expand Up @@ -1187,7 +1187,7 @@ fn realistic_env_graceful_workload_sweep() -> ForgeConfig {
TransactionWorkload::new_const_tps(TransactionTypeArg::ModifyGlobalFlagAggV2, 3 * 3500),
]),
criteria: Vec::new(),
continuous_traffic: Some(ContinuousTraffic {
background_traffic: Some(BackgroundTraffic {
traffic: EmitJobRequest::default()
.num_accounts_mode(NumAccountsMode::TransactionsPerAccount(1))
.mode(EmitJobMode::ConstTps { tps: 10 })
Expand Down Expand Up @@ -1234,7 +1234,7 @@ fn load_vs_perf_benchmark() -> ForgeConfig {
200, 1000, 3000, 5000, 7000, 7500, 8000, 9000, 10000, 12000, 15000,
]),
criteria: Vec::new(),
continuous_traffic: None,
background_traffic: None,
})
.with_genesis_helm_config_fn(Arc::new(|helm_values| {
// no epoch change.
Expand Down Expand Up @@ -1276,7 +1276,7 @@ fn workload_vs_perf_benchmark() -> ForgeConfig {
.with_unique_senders(),
]),
criteria: Vec::new(),
continuous_traffic: None,
background_traffic: None,
})
.with_genesis_helm_config_fn(Arc::new(|helm_values| {
// no epoch change.
Expand Down
1 change: 0 additions & 1 deletion testsuite/forge.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,6 @@ def with_context(
)
)
except Exception as e:
print(e)
result.set_state(ForgeState.FAIL)
result.set_debugging_output(
"{}\n{}\n".format(
Expand Down
2 changes: 1 addition & 1 deletion testsuite/testcases/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -488,7 +488,7 @@ impl dyn NetworkLoadTest + '_ {
duration: Duration,
warmup_duration_fraction: f32,
cooldown_duration_fraction: f32,
mut synchronized_with_job: Option<&mut EmitJob>,
synchronized_with_job: Option<&mut EmitJob>,
) -> Result<Vec<LoadTestPhaseStats>> {
let destination = self.setup(ctx).await.context("setup NetworkLoadTest")?;
let nodes_to_send_load_to = destination.get_destination_nodes(ctx.swarm.clone()).await;
Expand Down
37 changes: 20 additions & 17 deletions testsuite/testcases/src/load_vs_perf_benchmark.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ impl TransactionWorkload {
}
}

pub struct ContinuousTraffic {
pub struct BackgroundTraffic {
pub traffic: EmitJobRequest,
pub criteria: Vec<SuccessCriteria>,
}
Expand All @@ -247,12 +247,15 @@ pub struct LoadVsPerfBenchmark {
pub workloads: Workloads,
pub criteria: Vec<SuccessCriteria>,

pub continuous_traffic: Option<ContinuousTraffic>,
pub background_traffic: Option<BackgroundTraffic>,
}

impl Test for LoadVsPerfBenchmark {
fn name(&self) -> &'static str {
"continuous progress test"
match self.workloads {
Workloads::TPS(_) => "load vs perf test",
Workloads::TRANSACTIONS(_) => "workload vs perf test",
}
}
}

Expand Down Expand Up @@ -306,14 +309,14 @@ impl NetworkTest for LoadVsPerfBenchmark {
let mut ctx_locker = ctx.ctx.lock().await;
let ctx = ctx_locker.deref_mut();

let mut continous_job = if let Some(continuous_traffic) = &self.continuous_traffic {
let mut background_job = if let Some(background_traffic) = &self.background_traffic {
let nodes_to_send_load_to = LoadDestination::FullnodesOtherwiseValidators
.get_destination_nodes(ctx.swarm.clone())
.await;
let rng = SeedableRng::from_rng(ctx.core().rng())?;
let (mut emitter, emit_job_request) = create_emitter_and_request(
ctx.swarm.clone(),
continuous_traffic.traffic.clone(),
background_traffic.traffic.clone(),
&nodes_to_send_load_to,
rng,
)
Expand Down Expand Up @@ -351,7 +354,7 @@ impl NetworkTest for LoadVsPerfBenchmark {
phase_duration
.checked_mul(self.workloads.num_phases(index) as u32)
.unwrap(),
continous_job.as_mut(),
background_job.as_mut(),
)
.await
.inspect_err(|e| {
Expand All @@ -369,11 +372,11 @@ impl NetworkTest for LoadVsPerfBenchmark {
info!("{}", line);
}

if let Some(job) = &continous_job {
if let Some(job) = &background_job {
let stats_by_phase = job.peek_and_accumulate();
for line in to_table_continuous(
for line in to_table_background(
"background traffic".to_string(),
&extract_continuous_stats(stats_by_phase),
&extract_background_stats(stats_by_phase),
) {
info!("{}", line);
}
Expand All @@ -389,13 +392,13 @@ impl NetworkTest for LoadVsPerfBenchmark {
ctx.report.report_text(line);
}

let continuous_results = match continous_job {
let background_results = match background_job {
Some(job) => {
let stats_by_phase = job.stop_job().await;

let result = extract_continuous_stats(stats_by_phase);
let result = extract_background_stats(stats_by_phase);

for line in to_table_continuous("continuous traffic".to_string(), &result) {
for line in to_table_background("background traffic".to_string(), &result) {
ctx.report.report_text(line);
}
Some(result)
Expand All @@ -418,11 +421,11 @@ impl NetworkTest for LoadVsPerfBenchmark {
}
}

if let Some(results) = continuous_results {
if let Some(results) = background_results {
for (index, (name, stats)) in results.into_iter().enumerate() {
let rate = stats.rate();
if let Some(criteria) = self
.continuous_traffic
.background_traffic
.as_ref()
.unwrap()
.criteria
Expand All @@ -443,12 +446,12 @@ impl NetworkTest for LoadVsPerfBenchmark {
}
}

fn extract_continuous_stats(stats_by_phase: Vec<TxnStats>) -> Vec<(String, TxnStats)> {
fn extract_background_stats(stats_by_phase: Vec<TxnStats>) -> Vec<(String, TxnStats)> {
let mut result = vec![];
for (phase, phase_stats) in stats_by_phase.into_iter().enumerate() {
if phase % 2 != 0 {
result.push((
format!("continuous with traffic {}", phase / 2),
format!("background with traffic {}", phase / 2),
phase_stats,
));
}
Expand Down Expand Up @@ -504,7 +507,7 @@ fn to_table(type_name: String, results: &[Vec<SingleRunStats>]) -> Vec<String> {
table
}

fn to_table_continuous(type_name: String, results: &[(String, TxnStats)]) -> Vec<String> {
fn to_table_background(type_name: String, results: &[(String, TxnStats)]) -> Vec<String> {
let mut table = Vec::new();
table.push(format!(
"{: <40} | {: <12} | {: <12} | {: <12} | {: <12} | {: <12} | {: <12} | {: <12} | {: <12}",
Expand Down

0 comments on commit 389126d

Please sign in to comment.