Enable XSAVE feature in CPUID

core/cpu.c

@@ -33,7 +33,9 @@
 #include "hax.h"
 
 #include "cpuid.h"
+#include "driver.h"
 #include "dump.h"
+#include "fpu.h"
 #include "ia32_defs.h"
 #include "intr.h"
 #include "name.h"
@@ -68,6 +70,7 @@ void cpu_init_vmx(void *arg)
 {
     struct info_t vmx_info;
     struct per_cpu_data *cpu_data;
+    struct hstate *hstate;
     uint32_t fc_msr;
     vmcs_t *vmxon;
     int nx_enable = 0, vt_enable = 0;
@@ -143,6 +146,16 @@ void cpu_init_vmx(void *arg)
     vmx_read_info(&cpu_data->vmx_info);
 
     cpu_data->cpu_features |= HAX_CPUF_INITIALIZED;
+
+    if (cpu_has_feature(X86_FEATURE_XSAVE)) {
+        hstate = &cpu_data->hstate;
+
+        hstate->xcr0 = ia32_xgetbv(XCR_XFEATURE_ENABLED_MASK);
+        hax->supported_xcr0 = hstate->xcr0 & HAX_SUPPORTED_XCR0;
+
+        hax_log(HAX_LOGI, "%s: host xcr0 = 0x%llx, HAXM supported xcr0 = 0x%llx"
+                "\n", __func__, hstate->xcr0, hax->supported_xcr0);
+    }
 }
 
 void cpu_exit_vmx(void *arg)

core/cpuid.c

@@ -31,11 +31,14 @@
 
 #include "cpuid.h"
 
+#include "cpu.h"
 #include "driver.h"
+#include "fpu.h"
 #include "ia32.h"
+#include "ia32_defs.h"
 
-#define CPUID_CACHE_SIZE        6
-#define CPUID_FEATURE_SET_SIZE  3
+#define CPUID_CACHE_SIZE        7
+#define CPUID_FEATURE_SET_SIZE  4
 #define MAX_BASIC_CPUID         0x16
 #define MAX_EXTENDED_CPUID      0x80000008
 
@@ -94,6 +97,8 @@ static hax_cpuid_entry * find_cpuid_entry(hax_cpuid_entry *features,
                                           uint32_t size, uint32_t function,
                                           uint32_t index);
 static void dump_features(hax_cpuid_entry *features, uint32_t size);
+static uint32_t calc_xstate_required_size(uint64_t xstate_bv,
+                                          bool is_compacted);
 
 static void adjust_0000_0001(cpuid_args_t *args);
 static void adjust_8000_0001(cpuid_args_t *args);
@@ -102,6 +107,7 @@ static void execute_0000_0001(cpuid_args_t *args);
 static void execute_0000_0002(cpuid_args_t *args);
 static void execute_0000_0007(cpuid_args_t *args);
 static void execute_0000_000a(cpuid_args_t *args);
+static void execute_0000_000d(cpuid_args_t *args);
 static void execute_4000_0000(cpuid_args_t *args);
 static void execute_8000_0000(cpuid_args_t *args);
 static void execute_8000_0001(cpuid_args_t *args);
@@ -133,6 +139,8 @@ static const cpuid_manager_t kCpuidManager[] = {
     {0x00000007, 0, execute_0000_0007},  // Structured Extended Feature Flags
     {0x00000007, 1, execute_0000_0007},  // Structured Extended Feature Flags
     {0x0000000a, 0, execute_0000_000a},  // Architectural Performance Monitoring
+    {0x0000000d, 0, execute_0000_000d},  // Processor Extended State
+    {0x0000000d, 1, execute_0000_000d},  // Processor Extended State
     {0x00000015, 0, NULL},               // Time Stamp Counter and Nominal
                                          // Core Crystal Clock Information
     {0x00000016, 0, NULL},               // Processor Frequency Information
@@ -216,9 +224,12 @@ void cpuid_host_init(void)
     data[2] = res.ecx;
     data[3] = res.ebx;
 
+    cpuid_query_subleaf(&res, 0x0000000d, 0x01);
+    data[4] = res.eax;
+
     cpuid_query_leaf(&res, 0x80000001);
-    data[4] = res.ecx;
-    data[5] = res.edx;
+    data[5] = res.ecx;
+    data[6] = res.edx;
 
     cache.initialized = true;
 }
@@ -277,9 +288,14 @@ void cpuid_init_supported_features(void)
         .ecx = cache.data[2]
     };
     host_supported[2] = (hax_cpuid_entry){
+        .function = 0x0d,
+        .index = 1,
+        .eax = cache.data[4]
+    };
+    host_supported[3] = (hax_cpuid_entry){
         .function = 0x80000001,
-        .ecx = cache.data[4],
-        .edx = cache.data[5]
+        .ecx = cache.data[5],
+        .edx = cache.data[6]
     };
 
     hax_log(HAX_LOGI, "%s: host supported features:\n", __func__);
@@ -299,6 +315,7 @@ void cpuid_init_supported_features(void)
             FEATURE(AESNI)      |
             FEATURE(PCLMULQDQ)  |
             FEATURE(POPCNT)     |
+            FEATURE(XSAVE)      |
             FEATURE(AVX)        |
             FEATURE(F16C),
         .edx =
@@ -334,6 +351,14 @@ void cpuid_init_supported_features(void)
             FEATURE(INVPCID)
     };
     hax_supported[2] = (hax_cpuid_entry){
+        .function = 0x0d,
+        .index = 1,
+        .eax =
+            FEATURE(XSAVEOPT)   |
+            FEATURE(XSAVEC)     |
+            FEATURE(XGETBV1)
+    };
+    hax_supported[3] = (hax_cpuid_entry){
         .function = 0x80000001,
         .edx =
             FEATURE(NX)         |
@@ -486,6 +511,28 @@ void cpuid_execute(hax_cpuid_t *cpuid, cpuid_args_t *args)
             leaf, subleaf, args->eax, args->ebx, args->ecx, args->edx);
 }
 
+void cpuid_update(hax_cpuid_t *cpuid, vcpu_state_t *state)
+{
+    hax_cpuid_entry *entry;
+
+    entry = find_cpuid_entry(cpuid->features, CPUID_TOTAL_LEAVES, 0x01, 0);
+    if (entry != NULL && cpu_has_feature(X86_FEATURE_XSAVE)) {
+        // Update OSXSAVE bit
+        update_feature(entry, X86_FEATURE_OSXSAVE,
+                       !!(state->_cr4 & CR4_OSXSAVE));
+    }
+
+    entry = find_cpuid_entry(cpuid->features, CPUID_TOTAL_LEAVES, 0x0d, 0);
+    if (entry != NULL) {
+        entry->ebx = calc_xstate_required_size(state->_xcr0, false);
+    }
+
+    entry = find_cpuid_entry(cpuid->features, CPUID_TOTAL_LEAVES, 0x0d, 1);
+    if (entry != NULL && is_feature_set(entry, X86_FEATURE_XSAVEC)) {
+        entry->ebx = calc_xstate_required_size(state->_xcr0, true);
+    }
+}
+
 void cpuid_get_features_mask(hax_cpuid_t *cpuid, uint64_t *features_mask)
 {
     *features_mask = cpuid->features_mask;
@@ -542,6 +589,16 @@ int cpuid_set_guest_features(hax_cpuid_t *cpuid, hax_cpuid *cpuid_info)
     return 0;
 }
 
+void cpuid_post_set_features(hax_cpuid_t *cpuid, struct vm_t *vm)
+{
+    hax_cpuid_entry *entry;
+
+    entry = find_cpuid_entry(cpuid->features, CPUID_TOTAL_LEAVES, 0x0d, 0);
+
+    vm->valid_xcr0 = (entry == NULL) ? 0 : (entry->eax |
+            ((uint64_t)entry->edx << 32)) & hax->supported_xcr0;
+}
+
 static uint32_t feature_leaf(uint32_t feature_key)
 {
     cpuid_feature_t feature;
@@ -681,6 +738,31 @@ static void dump_features(hax_cpuid_entry *features, uint32_t size)
     }
 }
 
+static uint32_t calc_xstate_required_size(uint64_t xstate_bv, bool is_compacted)
+{
+    uint32_t size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
+    uint32_t bit, offset;
+    cpuid_args_t args;
+
+    for (xstate_bv &= XFEATURE_MASK_EXTENDED, bit = 0;
+         xstate_bv != 0;
+         xstate_bv >>= 1, ++bit) {
+        if (!(xstate_bv & 0x1))
+            continue;
+
+        args.eax = 0x0d;
+        args.ecx = bit;
+        asm_cpuid(&args);
+
+        offset = is_compacted ? size : args.ebx;
+        size = max(size, offset + args.eax);
+    }
+
+    hax_log(HAX_LOGI, "%s: size = %ld\n", __func__, size);
+
+    return size;
+}
+
 static void adjust_0000_0001(cpuid_args_t *args)
 {
 #define VIRT_FAMILY    0x06
@@ -762,7 +844,7 @@ static void adjust_8000_0001(cpuid_args_t *args)
     if (args == NULL)
         return;
 
-    hax_supported = &cache.hax_supported[2];
+    hax_supported = &cache.hax_supported[3];
 
     args->eax = args->ebx = 0;
     // Report only the features specified but turn off any features this
@@ -841,6 +923,56 @@ static void execute_0000_000a(cpuid_args_t *args)
     args->edx = pmu_info->cpuid_edx;
 }
 
+static void execute_0000_000d(cpuid_args_t *args)
+{
+    if (args == NULL)
+        return;
+
+    if (args->ecx > 63)
+        return;
+
+    switch (args->ecx) {
+        case 0: {  // Processor Extended State Enumeration Main Leaf
+            asm_cpuid(args);
+
+            args->eax &= hax->supported_xcr0;
+            args->ebx = calc_xstate_required_size(hax->supported_xcr0, false);
+            args->ecx = args->ebx;
+            args->edx &= hax->supported_xcr0 >> 32;
+            break;
+        }
+        case 1: {  // Processor Extended State Enumeration Sub-leaf
+            hax_cpuid_entry *hax_supported = &cache.hax_supported[2];
+
+            args->eax = hax_supported->eax;
+            args->ebx = (args->eax & FEATURE(XSAVEC))
+                        ? calc_xstate_required_size(hax->supported_xcr0, true)
+                        : 0;
+            args->ecx = args->edx = 0;
+            break;
+        }
+        default: {  // 2 ... 63: Processor Extended State Enumeration Sub-leaves 
+            if (!(hax->supported_xcr0 & (1ULL << args->ecx))) {
+                args->eax = args->ebx = args->ecx = args->edx = 0;
+                break;
+            }
+
+            asm_cpuid(args);
+
+            // The size of the save area for an extended state should never be
+            // 0. Furthermore, it should be clear in ECX[0] as currently HAXM
+            // only supports XCR0-managed area.
+            if (args->eax == 0 || (args->ecx & 0x1)) {
+                hax_log(HAX_LOGW, "%s: returned exceptional size (eax: %d) or "
+                        "state (ecx: %08lx)\n", __func__, args->eax, args->ecx);
+            }
+
+            args->edx = 0;
+            break;
+        }
+    }
+}
+
 static void execute_4000_0000(cpuid_args_t *args)
 {
     static const char kSignature[13] = "HAXMHAXMHAXM";

core/ia32.c

@@ -38,10 +38,14 @@ struct qword_val {
 #ifdef HAX_ARCH_X86_32
 extern void ASMCALL asm_rdmsr(uint32_t reg, struct qword_val *qv);
 extern void ASMCALL asm_wrmsr(uint32_t reg, struct qword_val *qv);
+extern void ASMCALL asm_xgetbv(uint32_t reg, struct qword_val *qv);
+extern void ASMCALL asm_xsetbv(uint32_t reg, struct qword_val *qv);
 extern void ASMCALL asm_rdtsc(struct qword_val *qv);
 #else  // !HAX_ARCH_X86_32
 extern uint64_t ASMCALL asm_rdmsr(uint32_t reg);
 extern void ASMCALL asm_wrmsr(uint32_t reg, uint64_t val);
+extern uint64_t ASMCALL asm_xgetbv(uint32_t reg);
+extern void ASMCALL asm_xsetbv(uint32_t reg, uint64_t val);
 extern uint64_t ASMCALL asm_rdtsc(void);
 #endif  // HAX_ARCH_X86_32
 
@@ -70,6 +74,33 @@ void ia32_wrmsr(uint32_t reg, uint64_t val)
 #endif
 }
 
+uint64_t ia32_xgetbv(uint32_t reg)
+{
+#ifdef HAX_ARCH_X86_32
+    struct qword_val value = {0};
+
+    asm_xgetbv(reg, &value);
+
+    return ((uint64_t)value.low | (uint64_t)value.high << 32);
+#else
+    return asm_xgetbv(reg);
+#endif
+}
+
+void ia32_xsetbv(uint32_t reg, uint64_t val)
+{
+#ifdef HAX_ARCH_X86_32
+    struct qword_val value = {0};
+
+    value.high = (uint32_t)(val >> 32);
+    value.low = (uint32_t)val;
+
+    asm_xsetbv(reg, &value);
+#else
+    asm_xsetbv(reg, val);
+#endif
+}
+
 uint64_t ia32_rdtsc(void)
 {
 #ifdef HAX_ARCH_X86_32

core/ia32_ops.asm

@@ -289,6 +289,48 @@ function asm_wrmsr, 2
     %error "Unimplemented function"
 %endif
 
+function asm_xgetbv, 2
+%ifidn __BITS__, 64
+    mov rcx, reg_arg1
+    xgetbv
+    shl rdx, 32
+    or reg_ret, rdx
+    ret
+%elifidn __CONV__, x32_cdecl
+    push ebx
+    mov ebx, reg_arg2
+    xgetbv
+    mov [ebx + qword_struct.lo], eax
+    mov [ebx + qword_struct.hi], edx
+    pop ebx
+    ret
+%else
+    %error "Unimplemented function"
+%endif
+
+function asm_xsetbv, 2
+%ifidn __BITS__, 64
+    mov rcx, reg_arg1
+    mov rax, reg_arg2
+    mov rdx, reg_arg2
+    shr rdx, 32
+    xsetbv
+    ret
+%elifidn __CONV__, x32_cdecl
+    push edi
+    push esi
+    mov edi, [reg_arg2 + qword_struct.lo]
+    mov esi, [reg_arg2 + qword_struct.hi]
+    mov eax, edi
+    mov edx, esi
+    xsetbv
+    pop esi
+    pop edi
+    ret
+%else
+    %error "Unimplemented function"
+%endif
+
 function get_kernel_tr_selector, 0
     str reg_ret_16
     ret

core/include/cpu.h

@@ -84,6 +84,8 @@ struct hstate {
     uint64_t dr7;
     // CR0
     bool cr0_ts;
+    // XCR0
+    uint64_t xcr0;
     uint64_t _pat;
 };