[release/8.0] Disable W^X in Rosetta emulated x64 containers on macOS (#106353)

* Disable W^X on x64 in rosetta based container

The docker on macOS Arm64 uses Rosetta to run x64 containers. That
has an effect on the double mapping. The Rosetta is unable to detect
when an already executed code page is modified. So we cannot use double
mapping on those containers. To detect that case, this change adds check
that verifies that the double mapping works even when the code is
modified.

Close #102226

* Rework based on PR feedback

* Check only for Rosetta

* Enable the rosetta check for x86 too

This will help WINE running 32 bit code under rosetta emulation on
macOS.

---------

Co-authored-by: Jan Vorlicek <[email protected]>

Author: github-actions[bot]

src/coreclr/minipal/CMakeLists.txt

@@ -1,4 +1,5 @@
 include_directories(.)
+include_directories(${CLR_SRC_NATIVE_DIR})
 if (CLR_CMAKE_HOST_UNIX)
     add_subdirectory(Unix)
 else (CLR_CMAKE_HOST_UNIX)

src/coreclr/minipal/Unix/doublemapping.cpp

@@ -20,24 +20,13 @@
 #define memfd_create(...) syscall(__NR_memfd_create, __VA_ARGS__)
 #endif // TARGET_LINUX && !MFD_CLOEXEC
 #include "minipal.h"
+#include "minipal/cpufeatures.h"
 
-#if defined(TARGET_OSX) && defined(TARGET_AMD64)
-#include <mach/mach.h>
-#include <sys/sysctl.h>
+#ifdef TARGET_OSX
 
-bool IsProcessTranslated()
-{
-   int ret = 0;
-   size_t size = sizeof(ret);
-   if (sysctlbyname("sysctl.proc_translated", &ret, &size, NULL, 0) == -1)
-   {
-      return false;
-   }
-   return ret == 1;
-}
-#endif // TARGET_OSX && TARGET_AMD64
+#include <mach/mach.h>
 
-#ifndef TARGET_OSX
+#else // TARGET_OSX
 
 #ifdef TARGET_64BIT
 static const off_t MaxDoubleMappedSize = 2048ULL*1024*1024*1024;
@@ -49,6 +38,12 @@ static const off_t MaxDoubleMappedSize = UINT_MAX;
 
 bool VMToOSInterface::CreateDoubleMemoryMapper(void** pHandle, size_t *pMaxExecutableCodeSize)
 {
+    if (minipal_detect_rosetta())
+    {
+        // Rosetta doesn't support double mapping correctly
+        return false;
+    }
+
 #ifndef TARGET_OSX
 
 #ifdef TARGET_FREEBSD
@@ -78,14 +73,6 @@ bool VMToOSInterface::CreateDoubleMemoryMapper(void** pHandle, size_t *pMaxExecu
     *pHandle = (void*)(size_t)fd;
 #else // !TARGET_OSX
 
-#ifdef TARGET_AMD64
-    if (IsProcessTranslated())
-    {
-        // Rosetta doesn't support double mapping correctly
-        return false;
-    }
-#endif // TARGET_AMD64
-
     *pMaxExecutableCodeSize = SIZE_MAX;
     *pHandle = NULL;
 #endif // !TARGET_OSX

src/coreclr/minipal/Windows/doublemapping.cpp

@@ -6,6 +6,7 @@
 #include <inttypes.h>
 #include <assert.h>
 #include "minipal.h"
+#include "minipal/cpufeatures.h"
 
 #define HIDWORD(_qw)    ((ULONG)((_qw) >> 32))
 #define LODWORD(_qw)    ((ULONG)(_qw))
@@ -60,6 +61,12 @@ inline void *GetBotMemoryAddress(void)
 
 bool VMToOSInterface::CreateDoubleMemoryMapper(void **pHandle, size_t *pMaxExecutableCodeSize)
 {
+    if (minipal_detect_rosetta())
+    {
+        // Rosetta doesn't support double mapping correctly. WINE on macOS ARM64 can be running under Rosetta.
+        return false;
+    }
+
     *pMaxExecutableCodeSize = (size_t)MaxDoubleMappedSize;
     *pHandle = CreateFileMapping(
                  INVALID_HANDLE_VALUE,    // use paging file

src/native/minipal/cpufeatures.c

@@ -4,6 +4,8 @@
 #include <stdint.h>
 #include <stdbool.h>
 #include <stddef.h>
+#include <stdio.h>
+#include <string.h>
 
 #include "cpufeatures.h"
 #include "cpuid.h"
@@ -427,3 +429,36 @@ int minipal_getcpufeatures(void)
 
     return result;
 }
+
+// Detect if the current process is running under the Apple Rosetta x64 emulator
+bool minipal_detect_rosetta(void)
+{
+#if defined(HOST_AMD64) || defined(HOST_X86)
+    // Check for CPU brand indicating emulation
+    int regs[4];
+    char brand[49];
+
+    // Get the maximum value for extended function CPUID info
+    __cpuid(regs, (int)0x80000000);
+    if ((unsigned int)regs[0] < 0x80000004)
+    {
+        return false; // Extended CPUID not supported
+    }
+
+    // Retrieve the CPU brand string
+    for (unsigned int i = 0x80000002; i <= 0x80000004; ++i)
+    {
+        __cpuid(regs, (int)i);
+        memcpy(brand + (i - 0x80000002) * sizeof(regs), regs, sizeof(regs));
+    }
+    brand[sizeof(brand) - 1] = '\0';
+
+    // Check if CPU brand indicates emulation
+    if (strstr(brand, "VirtualApple") != NULL)
+    {
+        return true;
+    }
+#endif // HOST_AMD64 || HOST_X86
+
+    return false;
+}

src/native/minipal/cpufeatures.h

@@ -72,6 +72,7 @@ extern "C"
 #endif // __cplusplus
 
 int minipal_getcpufeatures(void);
+bool minipal_detect_rosetta(void);
 
 #ifdef __cplusplus
 }