Adding sushy tools to the ocp-on-libvirt solution

Currently ocp-on-libvirt installs the conserver service along with the
VMs to allow for IPMI remote management of the hosts, which in turn
allows for PXE image boot up.

In order to support ZTP deployments, virtual media image boot up is
needed, and to manage it remotely, the target server BMCs must suport
the redfish protocol.

Sushy tools provides support for the redfish protocol for libvirt VMs.

This change installs sushy tools, as implemented in the
assisted-installer ansible project,  when running the ocp-on-libvirt
role.

Depends-On: dci-labs/bos2-ci-config#72
Depends-On: openshift-kni/baremetal-deploy#975
Depends-On: redhat-partner-solutions/crucible#272
Test-Args-Hints: -e enable_redfish=true -e enable_virtualmedia=true

roles/ocp_on_libvirt/defaults/main.yml

@@ -1,9 +1,18 @@
 ---
 libvirt_image_path: /var/lib/libvirt/images
+bootmode: uefi
 enable_conserver: false
 enable_legacy_vga_mode: false
 do_dns_config: true
 apps_ip_address: 192.168.123.10
 api_ip_address: 192.168.123.5
 dns_vip_address: 192.168.123.6
+
+# REDFISH
+enable_redfish: false
+enable_virtualmedia: false
+redfish_cache_dir: /var/lib/dci-openshift-agent
+redfish_assisted_deploy_repo: https://github.com/redhat-partner-solutions/crucible.git
+redfish_assisted_deploy_version: 92864fc8be3f83a80666d3c1e98d1740f17dfc30
+
 ...

roles/ocp_on_libvirt/tasks/main.yml

@@ -30,6 +30,9 @@
       vars:
         vbmc_host: "{{ vbmc_host_provided }}"
         vbmc_nodes: "{{ resources }}"
+    - name: Redfish Setup
+      ansible.builtin.include_tasks: redfish_setup.yml
+      when: enable_redfish | bool
     - name: DCI Setup
       ansible.builtin.include_tasks: dci_setup.yml
     - name: Setup conserver

roles/ocp_on_libvirt/tasks/redfish_setup.yml

@@ -0,0 +1,54 @@
+- name: "Get Ansible roles path"
+  shell: "set -o pipefail && ansible-config dump|grep DEFAULT_ROLES_PATH|sed -e 's/.*=\\s*//'"
+  register: roles_path_cmd
+  delegate_to: localhost
+
+- name: "Set roles_path"
+  set_fact:
+    roles_path: "{{ roles_path_cmd.stdout }}"
+
+- name: "Clone/update assisted-deploy repo"
+  vars:
+    git_repo: "{{ redfish_assisted_deploy_repo }}"
+    git_ref: "{{ redfish_assisted_deploy_version }}"
+  git:
+    version: "{{ git_ref }}"
+    repo: "{{ git_repo }}"
+    dest: "{{ redfish_cache_dir }}/assisted_deploy_repo"
+    #force: true
+  # On RHEL8 git clone can sporadically fail with OpenSSL SSL_read:
+  # SSL_ERROR_SYSCALL, errno 104.
+  delegate_to: localhost
+  register: git_clone
+  retries: 3
+  delay: 10
+  until: not git_clone.failed
+  when:
+    - "redfish_cache_dir + '/assisted_deploy_repo/roles' in roles_path"
+  tags:
+    - clone_upstream_repos
+
+- name: Install sushy-tools
+  vars:
+    repo_root_path: "{{ playbook_dir | dirname }}"
+    cert_country: US
+    cert_state: MA
+    cert_locality: Westford
+    cert_organization: DCI
+    cert_organizational_unit: Lab
+    sushy_ignore_boot_device: false
+  include_role:
+    name: setup_sushy_tools
+
+- name: Get KVM hosts UUID
+  shell: >
+    virsh list --all --name --uuid |
+    sed -e 's/^\([^ ]*\) \([^ ]*\)$/"\2": "\1",/g' |
+    tr -d '\n' |
+    sed -e 's/^\(.*\),$/{\1}/g'
+  register: all_vms
+  become: yes
+
+- name: Store KVM hosts UUID
+  set_fact:
+    redfish_kvm_uuid: "{{ all_vms.stdout | from_json }}"

roles/ocp_on_libvirt/templates/hosts.j2

@@ -13,6 +13,10 @@ dir="{{ '{{' }} ansible_user_dir {{ '}}' }}/clusterconfigs"
 
 #webserver_url="http://{{ ansible_fqdn }}:8080"
 
+{% if externalMACAddress is defined %}
+externalMACAddress={{ externalMACAddress }}
+{% endif %}
+
 {% if enable_lso | default(false) | bool %}
 ocs_install_type=internal
 local_storage_devices=["/dev/sdb"]
@@ -28,7 +32,7 @@ labels={"cluster.ocs.openshift.io/openshift-storage": ""}
 [masters]
 {% for key, value in ironic_nodes.items() -%}
 {% if 'master' in key -%}
-{{ key }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} role=master ipmi_user={{ value.ipmi_user }} ipmi_password={{ value.ipmi_pass }} ipmi_address={{ value.ipmi_address }} ipmi_port={{ value.ipmi_port }} provision_mac={{ value.mac_address }} hardware_profile=default socket_console={{ enable_conserver }}{% if value.root_device_hint is defined %} root_device_hint={{ value.root_device_hint }}{% endif %}{% if value.root_device_hint_value is defined %} root_device_hint_value={{ value.root_device_hint_value }}{% endif %}
+{{ key }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} role=master ipmi_user={{ value.ipmi_user }} ipmi_password={{ value.ipmi_pass }} ipmi_address={{ value.ipmi_address }} ipmi_port={{ value.ipmi_port }} provision_mac={{ value.mac_address }} hardware_profile=default socket_console={{ enable_conserver }}{% if value.root_device_hint is defined %} root_device_hint={{ value.root_device_hint }}{% endif %}{% if value.root_device_hint_value is defined %} root_device_hint_value={{ value.root_device_hint_value }}{% endif %}{% if key in redfish_kvm_uuid | default({}) %} kvm_uuid={{ redfish_kvm_uuid[key] }} redfish_port={{ redfish_port }}{% endif %}
 {% endif %}
 {%- endfor %}
 
@@ -40,7 +44,7 @@ ansible_ssh_extra_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/n
 [workers]
 {% for key, value in ironic_nodes.items() -%}
 {% if 'worker' in key -%}
-{{ key }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} role=worker ipmi_user={{ value.ipmi_user }} ipmi_password={{ value.ipmi_pass }} ipmi_address={{ value.ipmi_address }} ipmi_port={{ value.ipmi_port }} provision_mac={{ value.mac_address }} hardware_profile=unknown socket_console={{ enable_conserver }}{% if value.root_device_hint is defined %} root_device_hint={{ value.root_device_hint }}{% endif %}{% if value.root_device_hint_value is defined %} root_device_hint_value={{ value.root_device_hint_value }}{% endif %}
+{{ key }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} role=worker ipmi_user={{ value.ipmi_user }} ipmi_password={{ value.ipmi_pass }} ipmi_address={{ value.ipmi_address }} ipmi_port={{ value.ipmi_port }} provision_mac={{ value.mac_address }} hardware_profile=unknown socket_console={{ enable_conserver }}{% if value.root_device_hint is defined %} root_device_hint={{ value.root_device_hint }}{% endif %}{% if value.root_device_hint_value is defined %} root_device_hint_value={{ value.root_device_hint_value }}{% endif %}{% if key in redfish_kvm_uuid | default({}) %} kvm_uuid={{ redfish_kvm_uuid[key] }} redfish_port={{ redfish_port }}{% endif %}
 {% endif %}
 {%- endfor %}
 
@@ -52,7 +56,7 @@ ansible_ssh_extra_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/n
 [provisioner]
 {% for key, value in ironic_nodes.items() -%}
 {% if 'provision' in key -%}
-{{ key }}{{ "." + ansible_fqdn if not do_dns_config|bool else "" }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} ansible_user={{ provisionhost_user }} prov_nic=eth0 pub_nic=eth1 ansible_ssh_common_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+{{ key }}{{ "." + ansible_fqdn if not do_dns_config|bool else "" }} {{ "ansible_host=" + key + "." + ansible_fqdn if not do_dns_config|bool else "" }} name={{ key }} ansible_user={{ provisionhost_user }} prov_nic=eth0 pub_nic=eth1 ansible_ssh_common_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"{{ enable_virtualmedia | ternary(" bootstrapProvisioningIP=" + bootstrapProvisioningIP, "") }}
 {% endif %}
 {%- endfor %}
 
@@ -73,3 +77,10 @@ ansible_ssh_extra_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/n
 # The following mirror entries are the default ones. If you want to add more mirror
 #   you can uncomment this parameter and add it here.
 #registry_source_mirrors=["quay.io/openshift-release-dev/ocp-v4.0-art-dev", "registry.svc.ci.openshift.org/ocp/release", "quay.io/openshift-release-dev/ocp-release"]
+
+{% if enable_redfish | bool -%}
+[kvm_hosts_redfish]
+{% for key, value in ironic_nodes.items() -%}
+{{ key }}
+{% endfor %}
+{%- endif %}

roles/ocp_on_libvirt/templates/libvirt_node.xml.j2

@@ -3,8 +3,12 @@
   <memory unit='KiB'>{{ host['memory'] * 1024 | int }}</memory>
   <vcpu placement='static'>{{ host['vcpus'] }}</vcpu>
   <os>
-    <type arch='{{ host['arch'] }}'>hvm</type>
+    <type arch='{{ host['arch'] }}'{{ " machine='pc-q35-rhel8.6.0'" if bootmode == "uefi" else "" }}>hvm</type>
     <boot dev='{{ host['boot_dev'] }}'/>
+{% if bootmode == "uefi" %}
+    <loader readonly='yes' secure='yes' type='pflash'>/usr/share/edk2/ovmf/OVMF_CODE.secboot.fd</loader>
+    <nvram template='/usr/share/edk2/ovmf/OVMF_VARS.secboot.fd'>/var/lib/libvirt/qemu/nvram/{{ host['name'] }}_VARS.fd</nvram>
+{% endif %}
 {% if enable_conserver %}
     <bios useserial='yes' rebootTimeout='0'/>
 {% endif %}
@@ -13,6 +17,9 @@
     <acpi/>
     <apic/>
     <pae/>
+{% if bootmode == "uefi" %}
+    <smm state="on"/>
+{% endif %}
   </features>
   <cpu mode='{{ host['cpu_mode'] }}'>
     <model fallback='allow' />
@@ -31,7 +38,7 @@
       <driver name='qemu' type='raw'/>
       <source file='/tmp/vm-{{ host['name'] }}.iso'/>
       <backingStore/>
-      <target dev='hda' bus='ide'/>
+      <target dev='hda' bus='{{ "sata" if bootmode == "uefi" else "ide" }}'/>
       <readonly/>
     </disk>
     <disk type='file' device='disk'>
@@ -79,7 +86,7 @@
     </interface>
 {% endfor %}
     <controller type='usb' index='0'/>
-    <controller type='pci' index='0' model='pci-root'/>
+    <controller type='pci' index='0' model='{{ "pcie-root" if bootmode == "uefi" else "pci-root" }}'/>
 {% if enable_conserver %}
     <serial type='unix'>
       <source mode='bind' path='/var/lib/libvirt/consoles/{{ host['name'] }}.console'/>