Adding sushy tools to the ocp-on-libvirt solution

roles/ocp_on_libvirt/README.md

@@ -1,3 +1,27 @@
+# OCP_on_libvirt role
+
+Role to setup a libvirt cluster ready to be provisioned with an OpenShift Container Platform release using the Installer Provisioned Infrastructure (IPI) method.
+
+## Parameters
+
+Name                                    | Required  | Default  | Description
+----------------------------------------|-----------|----------|-------------
+enable_conserver                        | No        | false    | Conserver is used to write the console output to a file for later monitoring. Disabling it may be required to monitor the VMs console during ongoing jobs.
+enable_redfish                          | No        | false    | By default, VMs are created with a Virtual Baseboard Management Controller (vBMC) that can be operated using the IPMI protocol. This parameter enables support for the Redfish protocol based on the sushy_tool Redfish Emulator.
+enable_virtualmedia                     | Only with enable_redfish=true | false | The Redfish protocol uses PXE as the default protocol to provide the hosts with the boot image. Currently we don't support the Redfish/PXE combination so, if you intend to use redfish in your setup, you must also enable virtualmedia.
+vbmc_user                               | Yes       | -        | User name to authenticate with the Redfish API.
+vbmc_pass                               | Yes       | -        | User password to authenticate with the Redfish API.
+externalMACAddress                      | Yes       | -        | MAC address to be assigned to the bootstrap VM, required for DHCP managed static addressing.
+bootstrapProvisioningIP                 | Yes       | -        | The IP address on the bootstrap VM where the provisioning services run while the installer is deploying the control plane (master) nodes
+redfish_port                            | No        | 8082     | Port the Redfish service is listening on.
+ocp_on_libvirt_repo_root_path           | No        | /tmp     | Path to the directory where the Assisted Installer artifacts, including the sushy-tools Redfish emulator, are created.
+ocp_on_libvirt_cert_country             | No        | US       | Settings for the TLS certificate in the sushy-tools Redfish emulator.
+ocp_on_libvirt_cert_state               | No        | MA       | see *cert_country*
+ocp_on_libvirt_cert_locality            | No        | Westford | see *cert_country*
+ocp_on_libvirt_cert_organization        | No        | DCI      | see *cert_country*
+ocp_on_libvirt_cert_organizational_unit | No        | Lab      | see *cert_country*
+
+
 ## PCI passthrough example configuration
 
 If you would like to add the following PCI passthrough to the worker node:
@@ -20,4 +44,23 @@ Please add the following variables to the libvirt cluster configuration:
     bus: "0x86"
     slot: "0x00"
     function: "0x0"
-```
+```
+
+## How to create a libvirt cluster based on Virtualmedia over Redfish
+
+### Example of inventory file
+
+```
+all:
+  children:
+    vm_host:
+      hosts:
+        libvirthost:
+          vbmc_user: MY_USER
+          vbmc_pass: MY_PASSWORD
+    nodes:
+      hosts:
+        libvirthost
+  vars:
+    bootstrapProvisioningIP: 192.168.168.168
+    externalMACAddress: 01:23:45:67:89:ab

roles/ocp_on_libvirt/defaults/main.yml

@@ -1,9 +1,22 @@
 ---
 libvirt_image_path: /var/lib/libvirt/images
 enable_conserver: false
+bootmode: "{{ 'uefi' if enable_redfish else 'legacy' }}"
 enable_legacy_vga_mode: false
 do_dns_config: true
 apps_ip_address: 192.168.123.10
 api_ip_address: 192.168.123.5
 dns_vip_address: 192.168.123.6
+
+# REDFISH
+enable_redfish: false
+enable_virtualmedia: false
+redfish_port: 8082
+ocp_on_libvirt_repo_root_path: /tmp
+ocp_on_libvirt_cert_country: US
+ocp_on_libvirt_cert_state: MA
+ocp_on_libvirt_cert_locality: Westford
+ocp_on_libvirt_cert_organization: DCI
+ocp_on_libvirt_cert_organizational_unit: Lab
+
 ...

roles/ocp_on_libvirt/tasks/main.yml

@@ -30,6 +30,9 @@
       vars:
         vbmc_host: "{{ vbmc_host_provided }}"
         vbmc_nodes: "{{ resources }}"
+    - name: Redfish Setup
+      ansible.builtin.include_tasks: redfish_setup.yml
+      when: enable_redfish | bool
     - name: DCI Setup
       ansible.builtin.include_tasks: dci_setup.yml
     - name: Setup conserver

roles/ocp_on_libvirt/tasks/redfish_setup.yml

@@ -0,0 +1,34 @@
+- name: Set hardware vendor to "kvm"
+  set_fact:
+    vendor: kvm
+    bmc_user: "{{ vbmc_user }}"
+    bmc_password: "{{ vbmc_pass }}"
+  loop: "{{ groups['nodes'] }}"
+  delegate_to: "{{ item }}"
+
+- name: Install sushy-tools
+  ansible.builtin.include_role:
+    name: setup_sushy_tools
+  vars:
+    sushy_tools_port: "{{ redfish_port }}"
+    repo_root_path: "{{ ocp_on_libvirt_repo_root_path }}"
+    cert_country: "{{ ocp_on_libvirt_cert_country }}"
+    cert_state: "{{ ocp_on_libvirt_cert_state }}"
+    cert_locality: "{{ ocp_on_libvirt_cert_locality }}"
+    cert_organization: "{{ ocp_on_libvirt_cert_organization }}"
+    cert_organizational_unit: "{{ ocp_on_libvirt_cert_organizational_unit }}"
+    sushy_ignore_boot_device: false
+    inventory_validated: true
+
+- name: Get KVM hosts UUID
+  shell: >
+    set -o pipefail; virsh list --all --name --uuid |
+    sed -e 's/^\([^ ]*\) \([^ ]*\)$/"\2": "\1",/g' |
+    tr -d '\n' |
+    sed -e 's/^\(.*\),$/{\1}/g'
+  register: all_vms
+  become: true
+
+- name: Store KVM hosts UUID
+  set_fact:
+    redfish_kvm_uuid: "{{ all_vms.stdout | from_json }}"

roles/ocp_on_libvirt/templates/hosts.j2

@@ -16,6 +16,10 @@ customize_extramanifests_path={{ customize_extramanifests_path }}
 
 #webserver_url="http://{{ ansible_fqdn }}:8080"
 
+{% if externalMACAddress is defined %}
+externalMACAddress={{ externalMACAddress }}
+{% endif %}
+
 {% if enable_lso | default(false) | bool %}
 ocs_install_type=internal
 local_storage_devices=["/dev/sdb"]
@@ -31,7 +35,7 @@ labels={"cluster.ocs.openshift.io/openshift-storage": ""}
 [masters]
 {% for key, value in ironic_nodes.items() -%}
 {% if 'master' in key -%}
-{{ key }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} role=master ipmi_user={{ value.ipmi_user }} ipmi_password={{ value.ipmi_pass }} ipmi_address={{ value.ipmi_address }} ipmi_port={{ value.ipmi_port }} provision_mac={{ value.mac_address }} hardware_profile=default socket_console={{ enable_conserver }}{% if value.root_device_hint is defined %} root_device_hint={{ value.root_device_hint }}{% endif %}{% if value.root_device_hint_value is defined %} root_device_hint_value={{ value.root_device_hint_value }}{% endif %}
+{{ key }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} role=master ipmi_user={{ value.ipmi_user }} ipmi_password={{ value.ipmi_pass }} ipmi_address={{ value.ipmi_address }} ipmi_port={{ value.ipmi_port }} provision_mac={{ value.mac_address }} hardware_profile=default socket_console={{ enable_conserver }}{% if value.root_device_hint is defined %} root_device_hint={{ value.root_device_hint }}{% endif %}{% if value.root_device_hint_value is defined %} root_device_hint_value={{ value.root_device_hint_value }}{% endif %}{% if key in redfish_kvm_uuid | default({}) %} kvm_uuid={{ redfish_kvm_uuid[key] }} redfish_port={{ redfish_port }}{% endif %}
 {% endif %}
 {%- endfor %}
 
@@ -43,7 +47,7 @@ ansible_ssh_extra_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/n
 [workers]
 {% for key, value in ironic_nodes.items() -%}
 {% if 'worker' in key -%}
-{{ key }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} role=worker ipmi_user={{ value.ipmi_user }} ipmi_password={{ value.ipmi_pass }} ipmi_address={{ value.ipmi_address }} ipmi_port={{ value.ipmi_port }} provision_mac={{ value.mac_address }} hardware_profile=unknown socket_console={{ enable_conserver }}{% if value.root_device_hint is defined %} root_device_hint={{ value.root_device_hint }}{% endif %}{% if value.root_device_hint_value is defined %} root_device_hint_value={{ value.root_device_hint_value }}{% endif %}
+{{ key }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} role=worker ipmi_user={{ value.ipmi_user }} ipmi_password={{ value.ipmi_pass }} ipmi_address={{ value.ipmi_address }} ipmi_port={{ value.ipmi_port }} provision_mac={{ value.mac_address }} hardware_profile=unknown socket_console={{ enable_conserver }}{% if value.root_device_hint is defined %} root_device_hint={{ value.root_device_hint }}{% endif %}{% if value.root_device_hint_value is defined %} root_device_hint_value={{ value.root_device_hint_value }}{% endif %}{% if key in redfish_kvm_uuid | default({}) %} kvm_uuid={{ redfish_kvm_uuid[key] }} redfish_port={{ redfish_port }}{% endif %}
 {% endif %}
 {%- endfor %}
 
@@ -55,7 +59,7 @@ ansible_ssh_extra_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/n
 [provisioner]
 {% for key, value in ironic_nodes.items() -%}
 {% if 'provision' in key -%}
-{{ key }}{{ "." + ansible_fqdn if not do_dns_config|bool else "" }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} ansible_user={{ provisionhost_user }} prov_nic=eth0 pub_nic=eth1 ansible_ssh_common_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+{{ key }}{{ "." + ansible_fqdn if not do_dns_config|bool else "" }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} ansible_user={{ provisionhost_user }} prov_nic=eth0 pub_nic=eth1 ansible_ssh_common_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"{% if enable_virtualmedia %} bootstrapProvisioningIP={{ bootstrapProvisioningIP }}{% endif %}
 {% endif %}
 {%- endfor %}
 
@@ -76,3 +80,10 @@ ansible_ssh_extra_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/n
 # The following mirror entries are the default ones. If you want to add more mirror
 #   you can uncomment this parameter and add it here.
 #registry_source_mirrors=["quay.io/openshift-release-dev/ocp-v4.0-art-dev", "registry.svc.ci.openshift.org/ocp/release", "quay.io/openshift-release-dev/ocp-release"]
+
+{% if enable_redfish | bool -%}
+[kvm_hosts_redfish]
+{% for key, value in ironic_nodes.items() -%}
+{{ key }}
+{% endfor %}
+{%- endif %}

roles/ocp_on_libvirt/templates/libvirt_node.xml.j2

@@ -3,8 +3,16 @@
   <memory unit='KiB'>{{ host['memory'] * 1024 | int }}</memory>
   <vcpu placement='static'>{{ host['vcpus'] }}</vcpu>
   <os>
-    <type arch='{{ host['arch'] }}'>hvm</type>
+    <type arch='{{ host['arch'] }}'{{ " machine='pc-q35-rhel8.6.0'" if bootmode == "uefi" else "" }}>hvm</type>
     <boot dev='{{ host['boot_dev'] }}'/>
+{% if bootmode == "uefi" %}
+    <loader readonly='yes' secure='no' type='pflash'>/usr/share/edk2/ovmf/OVMF_CODE.secboot.fd</loader>
+    <nvram template='/usr/share/edk2/ovmf/OVMF_VARS.secboot.fd'>/var/lib/libvirt/qemu/nvram/{{ host['name'] }}_VARS.fd</nvram>
+    <firmware>
+      <feature enabled='no' name='secure-boot'/>
+      <feature enabled='yes' name='enrolled-keys'/>
+    </firmware>
+{% endif %}
 {% if enable_conserver %}
     <bios useserial='yes' rebootTimeout='0'/>
 {% endif %}
@@ -13,6 +21,9 @@
     <acpi/>
     <apic/>
     <pae/>
+{% if bootmode == "uefi" %}
+    <smm state="on"/>
+{% endif %}
   </features>
   <cpu mode='{{ host['cpu_mode'] }}'>
     <model fallback='allow' />
@@ -31,7 +42,7 @@
       <driver name='qemu' type='raw'/>
       <source file='/tmp/vm-{{ host['name'] }}.iso'/>
       <backingStore/>
-      <target dev='hda' bus='ide'/>
+      <target dev='hda' bus='{{ "sata" if bootmode == "uefi" else "ide" }}'/>
       <readonly/>
     </disk>
     <disk type='file' device='disk'>
@@ -79,7 +90,7 @@
     </interface>
 {% endfor %}
     <controller type='usb' index='0'/>
-    <controller type='pci' index='0' model='pci-root'/>
+    <controller type='pci' index='0' model='{{ "pcie-root" if bootmode == "uefi" else "pci-root" }}'/>
 {% if enable_conserver %}
     <serial type='unix'>
       <source mode='bind' path='/var/lib/libvirt/consoles/{{ host['name'] }}.console'/>

roles/setup_sushy_tools/defaults/main.yml

@@ -6,6 +6,7 @@ sushy_auth_dir: "{{ sushy_dir }}/auth"
 sushy_cert_dir: "{{ sushy_dir }}/cert"
 sushy_auth_file: "{{ sushy_auth_dir }}/htpasswd"
 sushy_data_dir: "{{ sushy_dir }}/data"
+sushy_ignore_boot_device: true
 sushy_libvirt_uri: "qemu:///system"
 sushy_packages_rhel9:
   - python3-devel

roles/setup_sushy_tools/templates/sushy-emulator.conf.j2

@@ -43,7 +43,7 @@ SUSHY_EMULATOR_LIBVIRT_URI = u'{{ sushy_libvirt_uri }}'
 # Instruct the libvirt driver to ignore any instructions to
 # set the boot device. Allowing the UEFI firmware to instead
 # rely on the EFI Boot Manager
-SUSHY_EMULATOR_IGNORE_BOOT_DEVICE = True
+SUSHY_EMULATOR_IGNORE_BOOT_DEVICE = {{ (sushy_ignore_boot_device | bool) | ternary("True", "False") }}
 
 # The map of firmware loaders dependant on the boot mode and
 # system architecture