Skip to content

Commit

Permalink
fix: Handle container port mismatch when creating kernel (#2786) (#2788)
Browse files Browse the repository at this point in the history
Co-authored-by: Sanghun Lee <sanghun@lablup.com>
  • Loading branch information
lablup-octodog and fregataa authored Aug 29, 2024
1 parent 5f377bd commit 43dc77b
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 1 deletion.
1 change: 1 addition & 0 deletions changes/2786.fix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Handle container port mismatch when creating kernel.
10 changes: 9 additions & 1 deletion src/ai/backend/agent/docker/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -731,7 +731,9 @@ async def start_container(
advertised_kernel_host = self.local_config["container"].get("advertised-host")
repl_ports = [2000, 2001]
if len(service_ports) + len(repl_ports) > len(self.port_pool):
raise RuntimeError("Container ports are not sufficiently available.")
raise RuntimeError(
f"Container ports are not sufficiently available. (remaining ports: {self.port_pool})"
)
exposed_ports = repl_ports
host_ports = [self.port_pool.pop() for _ in repl_ports]
for sport in service_ports:
Expand Down Expand Up @@ -977,6 +979,12 @@ async def _rollback_container_creation() -> None:
container_id=cid, message="Container port not found"
)
host_port = int(ports[0]["HostPort"])
if host_port != host_ports[idx]:
await _rollback_container_creation()
raise ContainerCreationError(
container_id=cid,
message=f"Port mapping mismatch. {host_port = }, {host_ports[idx] = }",
)
assert host_port == host_ports[idx]
if port == 2000: # intrinsic
repl_in_port = host_port
Expand Down

0 comments on commit 43dc77b

Please sign in to comment.