diff --git a/changes/2786.fix.md b/changes/2786.fix.md new file mode 100644 index 0000000000..11c7c98764 --- /dev/null +++ b/changes/2786.fix.md @@ -0,0 +1 @@ +Handle container port mismatch when creating kernel. diff --git a/src/ai/backend/agent/docker/agent.py b/src/ai/backend/agent/docker/agent.py index 349698cdf9..5cba83547d 100644 --- a/src/ai/backend/agent/docker/agent.py +++ b/src/ai/backend/agent/docker/agent.py @@ -731,7 +731,9 @@ async def start_container( advertised_kernel_host = self.local_config["container"].get("advertised-host") repl_ports = [2000, 2001] if len(service_ports) + len(repl_ports) > len(self.port_pool): - raise RuntimeError("Container ports are not sufficiently available.") + raise RuntimeError( + f"Container ports are not sufficiently available. (remaining ports: {self.port_pool})" + ) exposed_ports = repl_ports host_ports = [self.port_pool.pop() for _ in repl_ports] for sport in service_ports: @@ -977,6 +979,12 @@ async def _rollback_container_creation() -> None: container_id=cid, message="Container port not found" ) host_port = int(ports[0]["HostPort"]) + if host_port != host_ports[idx]: + await _rollback_container_creation() + raise ContainerCreationError( + container_id=cid, + message=f"Port mapping mismatch. {host_port = }, {host_ports[idx] = }", + ) assert host_port == host_ports[idx] if port == 2000: # intrinsic repl_in_port = host_port