feat: gateway restores removes containers

docker-compose.yml

@@ -8,7 +8,7 @@ networks:
     ipam:
       driver: default
       config:
-        - subnet: 10.99.0.0/24
+        - subnet: 10.99.0.0/16
 services:
   gateway:
     image: "${CONTAINER_REGISTRY}/gateway:${BACKEND_TAG}"

gateway/Cargo.toml

@@ -24,7 +24,6 @@ hyper = { version = "0.14.19", features = [ "stream" ] }
 # not great, but waiting for WebSocket changes to be merged
 hyper-reverse-proxy = { git = "https://github.com/chesedo/hyper-reverse-proxy", branch = "bug/host_header" }
 instant-acme = "0.1.0"
-
 lazy_static = "1.4.0"
 once_cell = "1.14.0"
 opentelemetry = { version = "0.18.0", features = ["rt-tokio"] }
@@ -58,3 +57,4 @@ colored = "2"
 portpicker = "0.1"
 snailquote = "0.3"
 tempfile = "3.3.0"
+

gateway/src/project.rs

@@ -9,6 +9,7 @@ use bollard::container::{
 use bollard::errors::Error as DockerError;
 use bollard::models::{ContainerConfig, ContainerInspectResponse, ContainerStateStatusEnum};
 use bollard::system::EventsOptions;
+use fqdn::FQDN;
 use futures::prelude::*;
 use http::uri::InvalidUri;
 use http::Uri;
@@ -81,6 +82,46 @@ where
     }
 }
 
+pub trait ContainerInspectResponseExt {
+    fn container(&self) -> &ContainerInspectResponse;
+
+    fn project_name(&self) -> Result<ProjectName, ProjectError> {
+        // This version can't be enabled while there are active
+        // deployers before v0.8.0 since the don't have this label
+        // TODO: switch to this version when you notice all deployers
+        // are greater than v0.8.0
+        // let name = safe_unwrap!(container.config.labels.get("project.name")).to_string();
+
+        let container = self.container();
+        let container_name = safe_unwrap!(container.name.strip_prefix("/")).to_string();
+        let prefix = safe_unwrap!(container.config.labels.get("shuttle_prefix")).to_string();
+        safe_unwrap!(container_name.strip_prefix(&prefix).strip_suffix("_run"))
+            .parse::<ProjectName>()
+            .map_err(|_| ProjectError::internal("invalid project name"))
+    }
+
+    fn args(&self) -> Result<&Vec<String>, ProjectError> {
+        let container = self.container();
+        Ok(safe_unwrap!(container.args))
+    }
+
+    fn fqdn(&self) -> Result<FQDN, ProjectError> {
+        let mut args = self.args()?.iter();
+        (&mut args)
+            .find(|arg| arg.as_str() == "--proxy-fqdn")
+            .ok_or_else(|| ProjectError::internal("no such argument: --proxy-fqdn"))?;
+        args.next()
+            .and_then(|arg| arg.parse().ok())
+            .ok_or_else(|| ProjectError::internal("argument to --proxy-fqdn is malformed"))
+    }
+}
+
+impl ContainerInspectResponseExt for ContainerInspectResponse {
+    fn container(&self) -> &ContainerInspectResponse {
+        self
+    }
+}
+
 impl From<DockerError> for Error {
     fn from(err: DockerError) -> Self {
         error!(error = %err, "internal Docker error");
@@ -185,9 +226,9 @@ impl Project {
         }
     }
 
-    pub fn initial_key(&self) -> Option<&String> {
-        if let Self::Creating(ProjectCreating { initial_key, .. }) = self {
-            Some(initial_key)
+    pub fn initial_key(&self) -> Option<&str> {
+        if let Self::Creating(creating) = self {
+            Some(creating.initial_key())
         } else {
             None
         }
@@ -293,21 +334,18 @@ where
     /// project into the wrong state if the docker is transitioning
     /// the state of its resources under us
     async fn refresh(self, ctx: &Ctx) -> Result<Self, Self::Error> {
-        let _container = if let Some(container_id) = self.container_id() {
-            Some(ctx.docker().inspect_container(&container_id, None).await?)
-        } else {
-            None
-        };
-
         let refreshed = match self {
             Self::Creating(creating) => Self::Creating(creating),
             Self::Starting(ProjectStarting { container })
             | Self::Started(ProjectStarted { container, .. })
             | Self::Ready(ProjectReady { container, .. })
             | Self::Stopping(ProjectStopping { container })
-            | Self::Stopped(ProjectStopped { container }) => {
-                let container = container.refresh(ctx).await?;
-                match container.state.as_ref().unwrap().status.as_ref().unwrap() {
+            | Self::Stopped(ProjectStopped { container }) => match container
+                .clone()
+                .refresh(ctx)
+                .await
+            {
+                Ok(container) => match container.state.as_ref().unwrap().status.as_ref().unwrap() {
                     ContainerStateStatusEnum::RUNNING => {
                         let service = Service::from_container(container.clone())?;
                         Self::Started(ProjectStarted { container, service })
@@ -322,8 +360,21 @@ where
                             "container resource has drifted out of sync: cannot recover",
                         ))
                     }
+                },
+                Err(DockerError::DockerResponseServerError {
+                    status_code: 404, ..
+                }) => {
+                    // container not found, let's try to recreate it
+                    // with the same image
+                    let project_name = container.project_name()?;
+                    let fqdn = container.fqdn()?;
+                    let creating = ProjectCreating::new_with_random_initial_key(project_name)
+                        .with_image(container.image.unwrap())
+                        .with_fqdn(fqdn.to_string());
+                    Self::Creating(creating)
                 }
-            }
+                Err(err) => return Err(err.into()),
+            },
             Self::Destroying(destroying) => Self::Destroying(destroying),
             Self::Destroyed(destroyed) => Self::Destroyed(destroyed),
             Self::Errored(err) => Self::Errored(err),
@@ -337,6 +388,7 @@ pub struct ProjectCreating {
     project_name: ProjectName,
     initial_key: String,
     fqdn: Option<String>,
+    image: Option<String>,
 }
 
 impl ProjectCreating {
@@ -345,6 +397,7 @@ impl ProjectCreating {
             project_name,
             initial_key,
             fqdn: None,
+            image: None,
         }
     }
 
@@ -358,10 +411,19 @@ impl ProjectCreating {
         Self::new(project_name, initial_key)
     }
 
+    pub fn with_image(mut self, image: String) -> Self {
+        self.image = Some(image);
+        self
+    }
+
     pub fn project_name(&self) -> &ProjectName {
         &self.project_name
     }
 
+    pub fn initial_key(&self) -> &str {
+        &self.initial_key
+    }
+
     fn container_name<C: DockerContext>(&self, ctx: &C) -> String {
         let prefix = &ctx.container_settings().prefix;
 
@@ -375,7 +437,7 @@ impl ProjectCreating {
         ctx: &C,
     ) -> (CreateContainerOptions<String>, Config<String>) {
         let ContainerSettings {
-            image,
+            image: default_image,
             prefix,
             provisioner_host,
             network_name,
@@ -388,14 +450,16 @@ impl ProjectCreating {
             initial_key,
             project_name,
             fqdn,
+            image,
+            ..
         } = &self;
 
         let create_container_options = CreateContainerOptions {
             name: self.container_name(ctx),
         };
 
         let container_config: ContainerConfig = deserialize_json!({
-            "Image": image,
+            "Image": image.as_ref().unwrap_or(default_image),
             "Hostname": format!("{prefix}{project_name}"),
             "Labels": {
                 "shuttle_prefix": prefix,
@@ -589,7 +653,7 @@ where
 }
 
 impl ProjectReady {
-    pub fn name(&self) -> &str {
+    pub fn name(&self) -> &ProjectName {
         &self.service.name
     }
 
@@ -619,20 +683,14 @@ impl HealthCheckRecord {
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct Service {
-    name: String,
+    name: ProjectName,
     target: IpAddr,
     last_check: Option<HealthCheckRecord>,
 }
 
 impl Service {
     pub fn from_container(container: ContainerInspectResponse) -> Result<Self, ProjectError> {
-        // This version can't be enabled while there are active deployers before v0.8.0 since the don't have this label
-        // TODO: switch to this version when you notice all deployers are greater than v0.8.0
-        // let name = safe_unwrap!(container.config.labels.get("project.name")).to_string();
-        let container_name = safe_unwrap!(container.name.strip_prefix("/")).to_string();
-        let prefix = safe_unwrap!(container.config.labels.get("shuttle_prefix")).to_string();
-        let resource_name =
-            safe_unwrap!(container_name.strip_prefix(&prefix).strip_suffix("_run")).to_string();
+        let resource_name = container.project_name()?;
 
         let network = safe_unwrap!(container.network_settings.networks)
             .values()
@@ -962,6 +1020,7 @@ pub mod tests {
                 project_name: "my-project-test".parse().unwrap(),
                 initial_key: "test".to_string(),
                 fqdn: None,
+                image: None,
             }),
             #[assertion = "Container created, assigned an `id`"]
             Ok(Project::Starting(ProjectStarting {

gateway/src/service.rs

@@ -265,11 +265,18 @@ impl GatewayService {
         project_name: &ProjectName,
         project: &Project,
     ) -> Result<(), Error> {
-        query("UPDATE projects SET project_state = ?1 WHERE project_name = ?2")
+        let query = match project {
+            Project::Creating(state) => query(
+                "UPDATE projects SET initial_key = ?1, project_state = ?2 WHERE project_name = ?3",
+            )
+            .bind(state.initial_key())
             .bind(SqlxJson(project))
-            .bind(project_name)
-            .execute(&self.db)
-            .await?;
+            .bind(project_name),
+            _ => query("UPDATE projects SET project_state = ?1 WHERE project_name = ?2")
+                .bind(SqlxJson(project))
+                .bind(project_name),
+        };
+        query.execute(&self.db).await?;
         Ok(())
     }
 
@@ -411,14 +418,9 @@ impl GatewayService {
             let project = row.get::<SqlxJson<Project>, _>("project_state").0;
             if project.is_destroyed() {
                 // But is in `::Destroyed` state, recreate it
-                let project = SqlxJson(Project::create(project_name.clone()));
-                query("UPDATE projects SET project_state = ?1, initial_key = ?2 WHERE project_name = ?3")
-                    .bind(&project)
-                    .bind(project.initial_key().unwrap())
-                    .bind(&project_name)
-                    .execute(&self.db)
-                    .await?;
-                Ok(project.0)
+                let project = Project::create(project_name.clone());
+                self.update_project(&project_name, &project).await?;
+                Ok(project)
             } else {
                 // Otherwise it already exists
                 Err(Error::from_kind(ErrorKind::ProjectAlreadyExists))