Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simulated Crucible Agent: Use CLI-configured IP address #649

Merged
merged 4 commits into from
Jan 27, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion nexus/src/db/datastore.rs
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ impl DataStore {
.do_update()
.set((
dsl::time_modified.eq(Utc::now()),
dsl::pool_id.eq(excluded(dsl::id)),
dsl::pool_id.eq(excluded(dsl::pool_id)),
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think this was the manifestation of the problem, but this was definitely a bug.

dsl::ip.eq(excluded(dsl::ip)),
dsl::port.eq(excluded(dsl::port)),
dsl::kind.eq(excluded(dsl::kind)),
Expand Down
2 changes: 1 addition & 1 deletion nexus/src/nexus.rs
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ impl Nexus {
address: SocketAddr,
kind: DatasetKind,
) -> Result<(), Error> {
info!(self.log, "upserting dataset"; "zpool_id" => zpool_id.to_string(), "dataset_id" => id.to_string());
info!(self.log, "upserting dataset"; "zpool_id" => zpool_id.to_string(), "dataset_id" => id.to_string(), "address" => address.to_string());
let dataset = db::model::Dataset::new(id, zpool_id, address, kind);
self.db_datastore.dataset_upsert(dataset).await?;
Ok(())
Expand Down
7 changes: 5 additions & 2 deletions nexus/test-utils/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use oximeter_collector::Oximeter;
use oximeter_producer::Server as ProducerServer;
use slog::o;
use slog::Logger;
use std::net::SocketAddr;
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::path::Path;
use std::time::Duration;
use uuid::Uuid;
Expand Down Expand Up @@ -183,7 +183,10 @@ pub async fn start_sled_agent(
},
/* TODO-cleanup this is unused */
log: ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Debug },
storage: sim::ConfigStorage::default(),
storage: sim::ConfigStorage {
zpools: vec![],
ip: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
},
};

sim::Server::start(&config, &log).await
Expand Down
1 change: 1 addition & 0 deletions sled-agent/src/bin/sled-agent-sim.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ async fn do_run() -> Result<(), CmdError> {
storage: ConfigStorage {
// Create 10 "virtual" U.2s, with 1 TB of storage.
zpools: vec![ConfigZpool { size: 1 << 40 }; 10],
ip: args.sled_agent_addr.ip(),
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

At @david-crespo 's suggestion, I'm just re-using whatever IP we supplied for the sled agent here when allocating ports for Crucible too.

},
};

Expand Down
5 changes: 3 additions & 2 deletions sled-agent/src/sim/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use dropshot::ConfigDropshot;
use dropshot::ConfigLogging;
use serde::Deserialize;
use serde::Serialize;
use std::net::SocketAddr;
use std::net::{IpAddr, SocketAddr};
use uuid::Uuid;

/**
Expand Down Expand Up @@ -43,9 +43,10 @@ pub struct ConfigZpool {
}

/// Configuration describing simulated storage.
#[derive(Clone, Default, Debug, PartialEq, Deserialize, Serialize)]
#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
pub struct ConfigStorage {
pub zpools: Vec<ConfigZpool>,
pub ip: IpAddr,
}

/**
Expand Down
3 changes: 1 addition & 2 deletions sled-agent/src/sim/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,7 @@ impl Server {
"server" => config.id.clone().to_string()
));
let sled_agent = Arc::new(SledAgent::new_simulated_with_id(
&config.id,
config.sim_mode,
&config,
sa_log,
Arc::clone(&nexus_client),
));
Expand Down
10 changes: 6 additions & 4 deletions sled-agent/src/sim/sled_agent.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use std::sync::Arc;
use uuid::Uuid;

use super::collection::SimCollection;
use super::config::SimMode;
use super::config::Config;
use super::disk::SimDisk;
use super::instance::SimInstance;
use super::storage::{CrucibleData, Storage};
Expand Down Expand Up @@ -48,11 +48,12 @@ impl SledAgent {
*/
/** Constructs a simulated SledAgent with the given uuid. */
pub fn new_simulated_with_id(
id: &Uuid,
sim_mode: SimMode,
config: &Config,
log: Logger,
nexus_client: Arc<NexusClient>,
) -> SledAgent {
let id = config.id;
let sim_mode = config.sim_mode;
info!(&log, "created simulated sled agent"; "sim_mode" => ?sim_mode);

let instance_log = log.new(o!("kind" => "instances"));
Expand All @@ -71,8 +72,9 @@ impl SledAgent {
sim_mode,
)),
storage: Mutex::new(Storage::new(
*id,
id,
Arc::clone(&nexus_client),
config.storage.ip,
storage_log,
)),
}
Expand Down
21 changes: 12 additions & 9 deletions sled-agent/src/sim/storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use nexus_client::types::{
use nexus_client::Client as NexusClient;
use slog::Logger;
use std::collections::HashMap;
use std::net::SocketAddr;
use std::net::{IpAddr, SocketAddr};
use std::str::FromStr;
use std::sync::Arc;
use uuid::Uuid;
Expand Down Expand Up @@ -137,10 +137,10 @@ pub struct CrucibleServer {
}

impl CrucibleServer {
fn new(log: &Logger) -> Self {
fn new(log: &Logger, crucible_ip: IpAddr) -> Self {
let data = Arc::new(CrucibleData::new());
let config = dropshot::ConfigDropshot {
bind_address: SocketAddr::new("127.0.0.1".parse().unwrap(), 0),
bind_address: SocketAddr::new(crucible_ip, 0),
..Default::default()
};
let dropshot_log = log
Expand All @@ -163,21 +163,22 @@ impl CrucibleServer {
}
}

pub struct Zpool {
struct Zpool {
datasets: HashMap<Uuid, CrucibleServer>,
}

impl Zpool {
pub fn new() -> Self {
fn new() -> Self {
Zpool { datasets: HashMap::new() }
}

pub fn insert_dataset(
fn insert_dataset(
&mut self,
log: &Logger,
id: Uuid,
crucible_ip: IpAddr,
) -> &CrucibleServer {
self.datasets.insert(id, CrucibleServer::new(log));
self.datasets.insert(id, CrucibleServer::new(log, crucible_ip));
self.datasets
.get(&id)
.expect("Failed to get the dataset we just inserted")
Expand All @@ -190,15 +191,17 @@ pub struct Storage {
nexus_client: Arc<NexusClient>,
log: Logger,
zpools: HashMap<Uuid, Zpool>,
crucible_ip: IpAddr,
}

impl Storage {
pub fn new(
sled_id: Uuid,
nexus_client: Arc<NexusClient>,
crucible_ip: IpAddr,
log: Logger,
) -> Self {
Self { sled_id, nexus_client, log, zpools: HashMap::new() }
Self { sled_id, nexus_client, log, zpools: HashMap::new(), crucible_ip }
}

/// Adds a Zpool to the sled's simulated storage and notifies Nexus.
Expand All @@ -221,7 +224,7 @@ impl Storage {
.zpools
.get_mut(&zpool_id)
.expect("Zpool does not exist")
.insert_dataset(&self.log, dataset_id);
.insert_dataset(&self.log, dataset_id, self.crucible_ip);

// Notify Nexus
let request = DatasetPutRequest {
Expand Down