Change the CryptoKit interop entrypoints to use the Swift calling convention (#99970)

Co-authored-by: Aleksey Kliger (λgeek) <[email protected]>

Author: jkoritzinsky

src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.Aead.cs

@@ -3,10 +3,13 @@
 
 using System;
 using System.Diagnostics;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Security.Cryptography;
 using System.Security.Cryptography.Apple;
 
+#pragma warning disable CS3016 // Arrays as attribute arguments are not CLS Compliant
+
 internal static partial class Interop
 {
     internal static partial class AppleCrypto
@@ -164,6 +167,7 @@ internal static unsafe void AesGcmDecrypt(
         }
 
         [LibraryImport(Libraries.AppleCryptoNative)]
+        [UnmanagedCallConv(CallConvs = [ typeof(CallConvSwift) ])]
         private static unsafe partial int AppleCryptoNative_ChaCha20Poly1305Encrypt(
             byte* keyPtr,
             int keyLength,
@@ -179,6 +183,7 @@ private static unsafe partial int AppleCryptoNative_ChaCha20Poly1305Encrypt(
             int aadLength);
 
         [LibraryImport(Libraries.AppleCryptoNative)]
+        [UnmanagedCallConv(CallConvs = [ typeof(CallConvSwift) ])]
         private static unsafe partial int AppleCryptoNative_ChaCha20Poly1305Decrypt(
             byte* keyPtr,
             int keyLength,
@@ -194,6 +199,7 @@ private static unsafe partial int AppleCryptoNative_ChaCha20Poly1305Decrypt(
             int aadLength);
 
         [LibraryImport(Libraries.AppleCryptoNative)]
+        [UnmanagedCallConv(CallConvs = [ typeof(CallConvSwift) ])]
         private static unsafe partial int AppleCryptoNative_AesGcmEncrypt(
             byte* keyPtr,
             int keyLength,
@@ -209,6 +215,7 @@ private static unsafe partial int AppleCryptoNative_AesGcmEncrypt(
             int aadLength);
 
         [LibraryImport(Libraries.AppleCryptoNative)]
+        [UnmanagedCallConv(CallConvs = [ typeof(CallConvSwift) ])]
         private static unsafe partial int AppleCryptoNative_AesGcmDecrypt(
             byte* keyPtr,
             int keyLength,

src/mono/mono/metadata/native-library-qcall.c

@@ -3,7 +3,7 @@
 
 static Entry mono_qcalls[] =
 {
-	DllImportEntry(NULL) // This NULL entry can be removed when a QCall is added to Mono (and added to this array)
+	{"NULL", NULL}, // This NULL entry can be removed when a QCall is added to Mono (and added to this array)
 };
 
 gpointer

src/native/libs/System.Security.Cryptography.Native.Apple/pal_swiftbindings.h

@@ -6,58 +6,7 @@
 #include "pal_types.h"
 #include "pal_compiler.h"
 
-PALEXPORT int32_t AppleCryptoNative_ChaCha20Poly1305Encrypt(
-    uint8_t* keyPtr,
-    int32_t keyLength,
-    uint8_t* noncePtr,
-    int32_t nonceLength,
-    uint8_t* plaintextPtr,
-    int32_t plaintextLength,
-    uint8_t* ciphertextBuffer,
-    int32_t ciphertextBufferLength,
-    uint8_t* tagBuffer,
-    int32_t tagBufferLength,
-    uint8_t* aadPtr,
-    int32_t aadLength);
-
-PALEXPORT int32_t AppleCryptoNative_ChaCha20Poly1305Decrypt(
-    uint8_t* keyPtr,
-    int32_t keyLength,
-    uint8_t* noncePtr,
-    int32_t nonceLength,
-    uint8_t* ciphertextPtr,
-    int32_t ciphertextLength,
-    uint8_t* tagPtr,
-    int32_t tagLength,
-    uint8_t* plaintextBuffer,
-    int32_t plaintextBufferLength,
-    uint8_t* aadPtr,
-    int32_t aadLength);
-
-PALEXPORT int32_t AppleCryptoNative_AesGcmEncrypt(
-    uint8_t* keyPtr,
-    int32_t keyLength,
-    uint8_t* noncePtr,
-    int32_t nonceLength,
-    uint8_t* plaintextPtr,
-    int32_t plaintextLength,
-    uint8_t* ciphertextBuffer,
-    int32_t ciphertextBufferLength,
-    uint8_t* tagBuffer,
-    int32_t tagBufferLength,
-    uint8_t* aadPtr,
-    int32_t aadLength);
-
-PALEXPORT int32_t AppleCryptoNative_AesGcmDecrypt(
-    uint8_t* keyPtr,
-    int32_t keyLength,
-    uint8_t* noncePtr,
-    int32_t nonceLength,
-    uint8_t* ciphertextPtr,
-    int32_t ciphertextLength,
-    uint8_t* tagPtr,
-    int32_t tagLength,
-    uint8_t* plaintextBuffer,
-    int32_t plaintextBufferLength,
-    uint8_t* aadPtr,
-    int32_t aadLength);
+EXTERN_C void* AppleCryptoNative_ChaCha20Poly1305Encrypt;
+EXTERN_C void* AppleCryptoNative_ChaCha20Poly1305Decrypt;
+EXTERN_C void* AppleCryptoNative_AesGcmEncrypt;
+EXTERN_C void* AppleCryptoNative_AesGcmDecrypt;

src/native/libs/System.Security.Cryptography.Native.Apple/pal_swiftbindings.swift

@@ -4,7 +4,7 @@
 import CryptoKit
 import Foundation
 
-@_cdecl("AppleCryptoNative_ChaCha20Poly1305Encrypt")
+@_silgen_name("AppleCryptoNative_ChaCha20Poly1305Encrypt")
 public func AppleCryptoNative_ChaCha20Poly1305Encrypt(
     keyPtr: UnsafeMutableRawPointer,
     keyLength: Int32,
@@ -41,7 +41,7 @@ public func AppleCryptoNative_ChaCha20Poly1305Encrypt(
     return 1
  }
 
-@_cdecl("AppleCryptoNative_ChaCha20Poly1305Decrypt")
+@_silgen_name("AppleCryptoNative_ChaCha20Poly1305Decrypt")
 public func AppleCryptoNative_ChaCha20Poly1305Decrypt(
     keyPtr: UnsafeMutableRawPointer,
     keyLength: Int32,
@@ -86,7 +86,7 @@ public func AppleCryptoNative_ChaCha20Poly1305Decrypt(
     }
 }
 
-@_cdecl("AppleCryptoNative_AesGcmEncrypt")
+@_silgen_name("AppleCryptoNative_AesGcmEncrypt")
 public func AppleCryptoNative_AesGcmEncrypt(
     keyPtr: UnsafeMutableRawPointer,
     keyLength: Int32,
@@ -123,7 +123,7 @@ public func AppleCryptoNative_AesGcmEncrypt(
     return 1
  }
 
-@_cdecl("AppleCryptoNative_AesGcmDecrypt")
+@_silgen_name("AppleCryptoNative_AesGcmDecrypt")
 public func AppleCryptoNative_AesGcmDecrypt(
     keyPtr: UnsafeMutableRawPointer,
     keyLength: Int32,

src/native/minipal/entrypoints.h

@@ -14,9 +14,9 @@ typedef struct
     const void* method;
 } Entry;
 
-// expands to:      {"impl", (void*)impl},
+// expands to:      {"impl", (void*)&impl},
 #define DllImportEntry(impl) \
-    {#impl, (void*)impl},
+    {#impl, (void*)&impl},
 
 static const void* minipal_resolve_dllimport(const Entry* resolutionTable, size_t tableLength, const char* name)
 {