Return NULL for mmap fail case on Unix (#78069)

* Return NULL for mmap fail case on Unix If mmap failed, "MAP_FAILED" is returned not "NULL". The windows implememtation of GetRWMapping returns "NULL" for fail case, and the caller function is also checking "NULL". So, change Unix implementation to return "NULL" for fail case. * call memset when mmap succeeds * check MAP_FAILED instead of NULL * return false for failing mmap * Call g_fatalErrorHandler if releasing failed * Update src/coreclr/utilcode/executableallocator.cpp Co-authored-by: Jan Vorlicek <jan.vorlicek@volny.cz> * Fix wrong condition check Co-authored-by: Jan Vorlicek <jan.vorlicek@volny.cz>
dotnet · Nov 10, 2022 · 39e9c6b · 39e9c6b
1 parent db21365
commit 39e9c6b
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 4 deletions.
diff --git a/src/coreclr/minipal/Unix/doublemapping.cpp b/src/coreclr/minipal/Unix/doublemapping.cpp
@@ -198,7 +198,10 @@ bool VMToOSInterface::ReleaseDoubleMappedMemory(void *mapperHandle, void* pStart
 {
 #ifndef TARGET_OSX
     int fd = (int)(size_t)mapperHandle;
-    mmap(pStart, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, offset);
+    if (mmap(pStart, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, offset) == MAP_FAILED)
+    {
+        return false;
+    }
     memset(pStart, 0, size);
 #endif // TARGET_OSX
     return munmap(pStart, size) != -1;
@@ -208,7 +211,12 @@ void* VMToOSInterface::GetRWMapping(void *mapperHandle, void* pStart, size_t off
 {
 #ifndef TARGET_OSX
     int fd = (int)(size_t)mapperHandle;
-    return mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, offset);
+    void* result = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, offset);
+    if (result == MAP_FAILED)
+    {
+        result = NULL;
+    }
+    return result;
 #else // TARGET_OSX
 #ifdef TARGET_AMD64
     vm_address_t startRW;

diff --git a/src/coreclr/nativeaot/Runtime/unix/PalRedhawkUnix.cpp b/src/coreclr/nativeaot/Runtime/unix/PalRedhawkUnix.cpp
@@ -743,7 +743,7 @@ REDHAWK_PALEXPORT _Ret_maybenull_ _Post_writable_byte_size_(size) void* REDHAWK_
 
         void * pRetVal = mmap(pAddress, alignedSize, unixProtect, flags, -1, 0);
 
-        if (pRetVal != NULL)
+        if (pRetVal != MAP_FAILED)
         {
             void * pAlignedRetVal = (void *)(((size_t)pRetVal + (Alignment - 1)) & ~(Alignment - 1));
             size_t startPadding = (size_t)pAlignedRetVal - (size_t)pRetVal;

diff --git a/src/coreclr/utilcode/executableallocator.cpp b/src/coreclr/utilcode/executableallocator.cpp
@@ -453,7 +453,10 @@ void ExecutableAllocator::Release(void* pRX)
 
         if (pBlock != NULL)
         {
-            VMToOSInterface::ReleaseDoubleMappedMemory(m_doubleMemoryMapperHandle, pRX, pBlock->offset, pBlock->size);
+            if (!VMToOSInterface::ReleaseDoubleMappedMemory(m_doubleMemoryMapperHandle, pRX, pBlock->offset, pBlock->size))
+            {
+                g_fatalErrorHandler(COR_E_EXECUTIONENGINE, W("Releasing the double mapped memory failed"));
+            }
             // Put the released block into the free block list
             pBlock->baseRX = NULL;
             pBlock->next = m_pFirstFreeBlockRX;