LinuxAllocator: Fixes bug with old kernels and hint allocation

In the face of an application using MAP_FIXED_NOREPLACE AND the host
linux kernel doesn't understand this flag. Then we were falling down the
hint allocation path which would allocate a pointer in 64-bit space,
returning this pointer to a 32-bit userspace and breaking things.

Now when the hint fails with this flag, we know that it intersecting a
range and can early exit.

Source/Tests/LinuxSyscalls/LinuxAllocator.cpp

@@ -139,8 +139,10 @@ void *MemAllocator32Bit::mmap(void *addr, size_t length, int prot, int flags, in
   uintptr_t Addr = reinterpret_cast<uintptr_t>(addr);
   uintptr_t PageAddr = Addr >> PAGE_SHIFT;
 
+  // Define MAP_FIXED_NOREPLACE ourselves to ensure we always parse this flag
+  constexpr int FEX_MAP_FIXED_NOREPLACE = 0x100000;
   bool Fixed = ((flags & MAP_FIXED) ||
-      (flags & MAP_FIXED_NOREPLACE));
+      (flags & FEX_MAP_FIXED_NOREPLACE));
 
   // Both Addr and length must be page aligned
   if (Addr & PAGE_MASK) {
@@ -170,8 +172,7 @@ void *MemAllocator32Bit::mmap(void *addr, size_t length, int prot, int flags, in
 
   bool Map32Bit = flags & FEX::HLE::X86_64_MAP_32BIT;
 
-  // Find a region that fits our address
-  if (Addr == 0) {
+  auto AllocateNoHint = [&]() -> void*{
     bool Wrapped = false;
     uint64_t BottomPage = Map32Bit && (LastScanLocation >= LastKeyLocation32Bit) ? LastKeyLocation32Bit : LastScanLocation;
 restart:
@@ -194,7 +195,7 @@ void *MemAllocator32Bit::mmap(void *addr, size_t length, int prot, int flags, in
           reinterpret_cast<void*>(LowerPage<< PAGE_SHIFT),
           length,
           prot,
-          flags | MAP_FIXED_NOREPLACE,
+          flags | FEX_MAP_FIXED_NOREPLACE,
           fd,
           offset);
 
@@ -244,6 +245,11 @@ void *MemAllocator32Bit::mmap(void *addr, size_t length, int prot, int flags, in
         }
       }
     }
+  };
+
+  // Find a region that fits our address
+  if (Addr == 0) {
+    return AllocateNoHint();
   }
   else {
     void *MappedPtr = ::mmap(
@@ -254,7 +260,15 @@ void *MemAllocator32Bit::mmap(void *addr, size_t length, int prot, int flags, in
       fd,
       offset);
 
-    if (MappedPtr != MAP_FAILED) {
+    if (MappedPtr >= reinterpret_cast<void*>(TOP_KEY << PAGE_SHIFT) &&
+        (flags & FEX_MAP_FIXED_NOREPLACE)) {
+      // Handles the case where MAP_FIXED_NOREPLACE isn't handled by the host system's
+      // kernel and returns the wrong pointer
+      // Make sure to munmap this so we don't leak memory
+      ::munmap(MappedPtr, length);
+      return reinterpret_cast<void*>(-EEXIST);
+    }
+    else if (MappedPtr != MAP_FAILED) {
       SetUsedPages(PageAddr, PagesLength);
       return MappedPtr;
     }