Complete VM support in virt plugin (#15151)

src/middlewared/middlewared/api/v25_04_0/__init__.py

@@ -51,5 +51,6 @@
 from .virt_device import *  # noqa
 from .virt_global import *  # noqa
 from .virt_instance import *  # noqa
+from .virt_volume import *  # noqa
 from .vm import *  # noqa
 from .vm_device import *  # noqa

src/middlewared/middlewared/api/v25_04_0/virt_device.py

@@ -1,6 +1,6 @@
 from typing import Annotated, Literal, TypeAlias
 
-from pydantic import Field
+from pydantic import Field, field_validator
 
 from middlewared.api.base import BaseModel, LocalGID, LocalUID, NonEmptyString
 
@@ -23,8 +23,28 @@ class Device(BaseModel):
 
 class Disk(Device):
     dev_type: Literal['DISK']
-    source: str | None = None
+    source: NonEmptyString | None = None
+    '''
+    For CONTAINER instances, this would be a valid pool path. For VM instances, it
+    can be a valid zvol path or an incus storage volume name
+    '''
     destination: str | None = None
+    boot_priority: int | None = Field(default=None, ge=0)
+
+    @field_validator('source')
+    @classmethod
+    def validate_source(cls, source):
+        if source is None or '/' not in source:
+            return source
+
+        # Source must be an absolute path now
+        if not source.startswith(('/dev/zvol/', '/mnt/')):
+            raise ValueError('Only pool paths are allowed')
+
+        if source.startswith('/mnt/.ix-apps'):
+            raise ValueError('Invalid source')
+
+        return source
 
 
 NicType: TypeAlias = Literal['BRIDGED', 'MACVLAN']
@@ -94,16 +114,15 @@ class USBChoice(BaseModel):
     product_id: str
     bus: int
     dev: int
-    product: str
-    manufacturer: str
+    product: str | None
+    manufacturer: str | None
 
 
 class VirtDeviceUSBChoicesResult(BaseModel):
     result: dict[str, USBChoice]
 
 
 class VirtDeviceGPUChoicesArgs(BaseModel):
-    instance_type: InstanceType
     gpu_type: GPUType
 
 

src/middlewared/middlewared/api/v25_04_0/virt_instance.py

@@ -1,6 +1,6 @@
 from typing import Annotated, Literal, TypeAlias
 
-from pydantic import Field, StringConstraints
+from pydantic import Field, model_validator, StringConstraints
 
 from middlewared.api.base import BaseModel, ForUpdateMetaclass, NonEmptyString, single_argument_args
 
@@ -15,7 +15,6 @@
     'VirtInstanceImageChoicesResult', 'VirtInstanceDeviceListArgs', 'VirtInstanceDeviceListResult',
     'VirtInstanceDeviceAddArgs', 'VirtInstanceDeviceAddResult', 'VirtInstanceDeviceUpdateArgs',
     'VirtInstanceDeviceUpdateResult', 'VirtInstanceDeviceDeleteArgs', 'VirtInstanceDeviceDeleteResult',
-
 ]
 
 
@@ -49,7 +48,9 @@ class VirtInstanceEntry(BaseModel):
     environment: dict[str, str]
     aliases: list[VirtInstanceAlias]
     image: Image
-    raw: dict
+    raw: dict | None
+    vnc_enabled: bool
+    vnc_port: int | None
 
 
 # Lets require at least 32MiB of reserved memory
@@ -62,14 +63,33 @@ class VirtInstanceEntry(BaseModel):
 @single_argument_args('virt_instance_create')
 class VirtInstanceCreateArgs(BaseModel):
     name: Annotated[NonEmptyString, StringConstraints(max_length=200)]
-    image: Annotated[NonEmptyString, StringConstraints(max_length=200)]
+    source_type: Literal[None, 'IMAGE'] = 'IMAGE'
+    image: Annotated[NonEmptyString, StringConstraints(max_length=200)] | None = None
     remote: REMOTE_CHOICES = 'LINUX_CONTAINERS'
     instance_type: InstanceType = 'CONTAINER'
     environment: dict[str, str] | None = None
     autostart: bool | None = True
     cpu: str | None = None
     devices: list[DeviceType] | None = None
     memory: MemoryType | None = None
+    enable_vnc: bool = False
+    vnc_port: int | None = Field(ge=5900, le=65535, default=None)
+
+    @model_validator(mode='after')
+    def validate_attrs(self):
+        if self.instance_type == 'CONTAINER':
+            if self.source_type != 'IMAGE':
+                raise ValueError('Source type must be set to "IMAGE" when instance type is CONTAINER')
+            if self.enable_vnc:
+                raise ValueError('VNC is not supported for containers and `enable_vnc` should be unset')
+        else:
+            if self.enable_vnc and self.vnc_port is None:
+                raise ValueError('VNC port must be set when VNC is enabled')
+
+        if self.source_type == 'IMAGE' and self.image is None:
+            raise ValueError('Image must be set when source type is "IMAGE"')
+
+        return self
 
 
 class VirtInstanceCreateResult(BaseModel):
@@ -81,6 +101,7 @@ class VirtInstanceUpdate(BaseModel, metaclass=ForUpdateMetaclass):
     autostart: bool | None = None
     cpu: str | None = None
     memory: MemoryType | None = None
+    vnc_port: int | None = Field(ge=5900, le=65535)
 
 
 class VirtInstanceUpdateArgs(BaseModel):
@@ -115,7 +136,7 @@ class StopArgs(BaseModel):
 
 class VirtInstanceStopArgs(BaseModel):
     id: str
-    stop_args: StopArgs
+    stop_args: StopArgs = StopArgs()
 
 
 class VirtInstanceStopResult(BaseModel):
@@ -124,18 +145,21 @@ class VirtInstanceStopResult(BaseModel):
 
 class VirtInstanceRestartArgs(BaseModel):
     id: str
-    stop_args: StopArgs
+    stop_args: StopArgs = StopArgs()
 
 
 class VirtInstanceRestartResult(BaseModel):
     result: bool
 
 
-@single_argument_args('virt_instances_image_choices')
-class VirtInstanceImageChoicesArgs(BaseModel):
+class VirtInstanceImageChoices(BaseModel):
     remote: REMOTE_CHOICES = 'LINUX_CONTAINERS'
 
 
+class VirtInstanceImageChoicesArgs(BaseModel):
+    virt_instances_image_choices: VirtInstanceImageChoices = VirtInstanceImageChoices()
+
+
 class ImageChoiceItem(BaseModel):
     label: str
     os: str

src/middlewared/middlewared/api/v25_04_0/virt_volume.py

@@ -0,0 +1,76 @@
+import os
+from typing import Literal
+
+from pydantic import Field, field_validator
+
+from middlewared.api.base import (
+    BaseModel, ForUpdateMetaclass, NonEmptyString, single_argument_args,
+)
+
+__all__ = [
+    'VirtVolumeEntry', 'VirtVolumeCreateArgs', 'VirtVolumeCreateResult',
+    'VirtVolumeUpdateArgs', 'VirtVolumeUpdateResult', 'VirtVolumeDeleteArgs',
+    'VirtVolumeDeleteResult', 'VirtVolumeImportISOArgs', 'VirtVolumeImportISOResult',
+]
+
+
+class VirtVolumeEntry(BaseModel):
+    id: NonEmptyString
+    name: NonEmptyString
+    content_type: NonEmptyString
+    created_at: str
+    type: NonEmptyString
+    config: dict
+    used_by: list[NonEmptyString]
+
+
+@single_argument_args('virt_volume_create')
+class VirtVolumeCreateArgs(BaseModel):
+    name: NonEmptyString
+    content_type: Literal['BLOCK'] = 'BLOCK'
+    size: int = Field(ge=512, default=1024)  # 1 gb default
+    '''Size of volume in MB and it should at least be 512 MB'''
+
+
+class VirtVolumeCreateResult(BaseModel):
+    result: VirtVolumeEntry
+
+
+class VirtVolumeUpdate(BaseModel, metaclass=ForUpdateMetaclass):
+    size: int = Field(ge=512)
+
+
+class VirtVolumeUpdateArgs(BaseModel):
+    id: NonEmptyString
+    virt_volume_update: VirtVolumeUpdate
+
+
+class VirtVolumeUpdateResult(BaseModel):
+    result: VirtVolumeEntry
+
+
+class VirtVolumeDeleteArgs(BaseModel):
+    id: NonEmptyString
+
+
+class VirtVolumeDeleteResult(BaseModel):
+    result: Literal[True]
+
+
+@single_argument_args('virt_volume_import_iso')
+class VirtVolumeImportISOArgs(BaseModel):
+    name: NonEmptyString
+    '''Specify name of the newly created volume from the ISO specified'''
+    iso_location: NonEmptyString | None = None
+    upload_iso: bool = False
+
+    @field_validator('iso_location')
+    @classmethod
+    def validate_iso_location(cls, v):
+        if v and not os.path.exists(v):
+            raise ValueError('Specified ISO location does not exist')
+        return v
+
+
+class VirtVolumeImportISOResult(BaseModel):
+    result: VirtVolumeEntry

src/middlewared/middlewared/logger.py

@@ -38,6 +38,8 @@
 # Prevent debug docker logs
 logging.getLogger('docker.utils.config').setLevel(logging.ERROR)
 logging.getLogger('docker.auth').setLevel(logging.ERROR)
+# Prevent httpx debug spam
+logging.getLogger('httpx._client').setLevel(logging.ERROR)
 
 # /usr/lib/python3/dist-packages/pydantic/json_schema.py:2158: PydanticJsonSchemaWarning:
 # Default value <object object at 0x7fa8ac040d30> is not JSON serializable; excluding default from JSON schema

src/middlewared/middlewared/plugins/virt/attachments.py

@@ -1,4 +1,6 @@
+from itertools import product
 from typing import TYPE_CHECKING
+
 from middlewared.common.attachment import FSAttachmentDelegate
 from middlewared.common.ports import PortDelegate
 
@@ -58,24 +60,18 @@ async def start(self, attachments):
 
 class VirtPortDelegate(PortDelegate):
 
-    name = 'virt devices'
-    namespace = 'virt.device'
+    name = 'virt instances'
+    namespace = 'virt'
     title = 'Virtualization Device'
 
     async def get_ports(self):
         ports = []
-        for instance in await self.middleware.call('virt.instance.query'):
-            instance_ports = []
-            for device in await self.middleware.call('virt.instance.device_list', instance['id']):
-                if device['dev_type'] != 'PROXY':
-                    continue
-                instance_ports.append(('0.0.0.0', device['source_port']))
-                instance_ports.append(('::', device['source_port']))
-            if instance_ports:
+        for instance_id, instance_ports in (await self.middleware.call('virt.instance.get_ports_mapping')).items():
+            if instance_ports := list(product(['0.0.0.0', '::'], instance_ports)):
                 ports.append({
-                    'description': f'{instance["id"]!r} instance',
+                    'description': f'{instance_id!r} instance',
                     'ports': instance_ports,
-                    'instance': instance['id'],
+                    'instance': instance_id,
                 })
         return ports
 

src/middlewared/middlewared/plugins/virt/device.py

@@ -36,7 +36,7 @@ def usb_choices(self):
         return choices
 
     @api_method(VirtDeviceGPUChoicesArgs, VirtDeviceGPUChoicesResult, roles=['VIRT_INSTANCE_READ'])
-    async def gpu_choices(self, instance_type, gpu_type):
+    async def gpu_choices(self, gpu_type):
         """
         Provide choices for GPU devices.
         """
@@ -45,9 +45,6 @@ async def gpu_choices(self, instance_type, gpu_type):
         if gpu_type != 'PHYSICAL':
             raise CallError('Only PHYSICAL type is supported for now.')
 
-        if instance_type != 'CONTAINER':
-            raise CallError('Only CONTAINER supported for now.')
-
         for i in await self.middleware.call('device.get_gpus'):
             if not i['available_to_host'] or i['uses_system_critical_devices']:
                 continue