Hash names of all the packaged DSOs (#6522)

[monodroid] Hash the names of all packaged DSOs (#6522)

Context: #6522
Context: https://sourceware.org/binutils/docs-2.37/as/index.html
Context: https://sourceware.org/binutils/docs-2.37/as/AArch64-Directives.html#AArch64-Directives
Context: https://sourceware.org/binutils/docs-2.37/as/ARM-Directives.html#ARM-Directives
Context: https://sourceware.org/binutils/docs-2.37/as/ARM-Directives.html#ARM-Directives
Context: https://sourceware.org/binutils/docs-2.37/as/Quad.html#Quad

Commit 000cf5a introduced hashing of native library names so that
we would only try to load specific Mono components which were
configured at app build time.

Expand this concept so that we support hashing any form of native
library name (DSO name) sent to use by Mono, and look for a cached
entry for that DSO.  The cache includes the actual native library
name, whether it should be ignored when requested (e.g. an empty AOT
`.so` file; see db161ae & df667b0), and the cached `dlopen()` value:

	struct DSOCacheEntry {
	    uint64_t    hash;
	    bool        ignore;
	    const char *name;
	    void       *handle;
	};
	DSOCacheEntry dso_cache[] = {…};

The `dso_cache` is computed at App build time, and stored in
`libxamarin-app.so`, and contains values sorted on
`DSOCacheEntry::hash`.

The use of `dso_cache` removes a bit of string processing from both
startup and the application run time, reducing app startup times in
some scenarios:

| Scenario                                      | Before ms |  After ms |        Δ |
| --------------------------------------------- | --------: | --------: | -------: |
| Plain Xamarin.Android (JIT, 64-bit)           |   311.500 |   311.600 | +0.03% ✗ |
| Plain Xamarin.Android (Profiled AOT, 64-bit)  |   253.500 |   247.000 | -2.56% ✓ |
| .NET MAUI Hello World (JIT, 64-bit)           |  1156.300 |  1159.500 | +0.28% ✗ |
| .NET MAUI Hello World (Profiled AOT, 64-bit)  |   868.900 |   831.700 | -4.28% ✓ |

(Times measured on a Pixel 3 XL.)

Above table is a subset of values from #6522;
see original PR for complete table information.  We believe that the
occasional increases are within the margin of error.

While implementing the above described `dso_cache` idea, I hit known
limitations of the native assembler generator code which, until this
commit, weren't a problem (mostly related to hard-coded structure and
array alignment).  Address these limitations by rewriting the assembly
generator.  It now fully implements the structure and symbol alignment
calculation for all the supported architectures.  Also added is the
ability to generate assembler code from managed structures, using
reflection, without the need of manually written code.  It also fixes
a previously unnoticed issue which made typemap structures not aligned
properly, which may have made them slightly slower than necessary.

Author: grendello

build-tools/scripts/XABuildConfig.cs.in

@@ -8,6 +8,9 @@ namespace Xamarin.Android.Tools
 		public const string SupportedABIs = "@XA_SUPPORTED_ABIS@";
 		public const string NDKRevision = "@NDK_REVISION@";
 		public const string NDKRelease = "@NDK_RELEASE@";
+		public const string XamarinAndroidVersion = "@XAMARIN_ANDROID_VERSION@";
+		public const string XamarinAndroidCommitHash = "@XAMARIN_ANDROID_COMMIT_HASH@";
+		public const string XamarinAndroidBranch = "@XAMARIN_ANDROID_BRANCH@";
 		public const int NDKMinimumApiAvailable = @NDK_MINIMUM_API_AVAILABLE@;
 		public const int AndroidLatestStableApiLevel = @ANDROID_LATEST_STABLE_API_LEVEL@;
 		public const int AndroidDefaultTargetDotnetApiLevel = @ANDROID_DEFAULT_TARGET_DOTNET_API_LEVEL@;

build-tools/xaprepare/xaprepare/Application/BuildInfo.cs

@@ -25,6 +25,9 @@ partial class BuildInfo : AppObject
 		public string MonoHash               { get; private set; } = String.Empty;
 		public string FullMonoHash           { get; private set; } = String.Empty;
 
+		public string XACommitHash           { get; private set; } = String.Empty;
+		public string XABranch               { get; private set; } = String.Empty;
+
 		public async Task GatherGitInfo (Context context)
 		{
 			if (context == null)
@@ -36,6 +39,14 @@ public async Task GatherGitInfo (Context context)
 			Log.StatusLine ();
 			DetermineMonoHash (context);
 			Log.StatusLine ();
+			DetermineXACommitInfo (context);
+		}
+
+		void DetermineXACommitInfo (Context context)
+		{
+			GitRunner git = CreateGitRunner (context);
+			XACommitHash = git.GetTopCommitHash (shortHash: false);
+			XABranch = git.GetBranchName ();
 		}
 
 		public bool GatherNDKInfo (Context context)

build-tools/xaprepare/xaprepare/Steps/Step_GenerateFiles.cs

@@ -168,6 +168,9 @@ GeneratedFile Get_XABuildConfig_cs (Context context)
 				{ "@XA_SUPPORTED_ABIS@",         context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedTargetJitAbis).Replace (':', ';') },
 				{ "@ANDROID_DEFAULT_TARGET_DOTNET_API_LEVEL@", context.Properties.GetRequiredValue (KnownProperties.AndroidDefaultTargetDotnetApiLevel) },
 				{ "@ANDROID_LATEST_STABLE_API_LEVEL@", context.Properties.GetRequiredValue (KnownProperties.AndroidLatestStableApiLevel) },
+				{ "@XAMARIN_ANDROID_VERSION@",   context.Properties.GetRequiredValue (KnownProperties.ProductVersion) },
+				{ "@XAMARIN_ANDROID_COMMIT_HASH@", context.BuildInfo.XACommitHash },
+				{ "@XAMARIN_ANDROID_BRANCH@", context.BuildInfo.XABranch },
 			};
 
 			return new GeneratedPlaceholdersFile (

build-tools/xaprepare/xaprepare/ToolRunners/GitRunner.cs

@@ -135,6 +135,33 @@ public async Task<bool> SubmoduleUpdate (string? workingDirectory = null, bool i
 			return await RunGit (runner, "submodule-update");
 		}
 
+		public string GetBranchName (string? workingDirectory = null)
+		{
+			string runnerWorkingDirectory = DetermineRunnerWorkingDirectory (workingDirectory);
+			var runner = CreateGitRunner (runnerWorkingDirectory);
+			runner
+				.AddArgument ("name-rev")
+				.AddArgument ("--name-only")
+				.AddArgument ("--exclude=tags/*")
+				.AddArgument ("HEAD");
+
+			string branchName = String.Empty;
+			using (var outputSink = (OutputSink)SetupOutputSink (runner)) {
+				outputSink.LineCallback = (string? line) => {
+					if (!String.IsNullOrEmpty (branchName)) {
+						return;
+					}
+					branchName = line?.Trim () ?? String.Empty;
+				};
+
+				if (!runner.Run ()) {
+					return String.Empty;
+				}
+
+				return branchName;
+			}
+		}
+
 		public string GetTopCommitHash (string? workingDirectory = null, bool shortHash = true)
 		{
 			string runnerWorkingDirectory = DetermineRunnerWorkingDirectory (workingDirectory);

src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs

@@ -75,13 +75,12 @@ void GenerateCompressedAssemblySources ()
 			void Generate (IDictionary<string, CompressedAssemblyInfo> dict)
 			{
 				foreach (string abi in SupportedAbis) {
-					NativeAssemblerTargetProvider asmTargetProvider = GeneratePackageManagerJava.GetAssemblyTargetProvider (abi);
 					string baseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"compressed_assemblies.{abi.ToLowerInvariant ()}");
 					string asmFilePath = $"{baseAsmFilePath}.s";
-					var asmgen = new CompressedAssembliesNativeAssemblyGenerator (dict, asmTargetProvider, baseAsmFilePath);
+					var asmgen = new CompressedAssembliesNativeAssemblyGenerator (GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), dict, baseAsmFilePath);
 
 					using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) {
-						asmgen.Write (sw);
+						asmgen.Write (sw, asmFilePath);
 						sw.Flush ();
 						if (Files.CopyIfStreamChanged (sw.BaseStream, asmFilePath)) {
 							Log.LogDebugMessage ($"File {asmFilePath} was regenerated");

src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs

@@ -28,6 +28,8 @@ public class GeneratePackageManagerJava : AndroidTask
 		[Required]
 		public ITaskItem[] ResolvedUserAssemblies { get; set; }
 
+		public ITaskItem[] NativeLibraries { get; set; }
+
 		public ITaskItem[] MonoComponents { get; set; }
 
 		public ITaskItem[] SatelliteAssemblies { get; set; }
@@ -142,20 +144,20 @@ public override bool RunTask ()
 			return !Log.HasLoggedErrors;
 		}
 
-		static internal NativeAssemblerTargetProvider GetAssemblyTargetProvider (string abi)
+		static internal AndroidTargetArch GetAndroidTargetArchForAbi (string abi)
 		{
 			switch (abi.Trim ()) {
 				case "armeabi-v7a":
-					return new ARMNativeAssemblerTargetProvider (false);
+					return AndroidTargetArch.Arm;
 
 				case "arm64-v8a":
-					return new ARMNativeAssemblerTargetProvider (true);
+					return AndroidTargetArch.Arm64;
 
 				case "x86":
-					return new X86NativeAssemblerTargetProvider (false);
+					return AndroidTargetArch.X86;
 
 				case "x86_64":
-					return new X86NativeAssemblerTargetProvider (true);
+					return AndroidTargetArch.X86_64;
 
 				default:
 					throw new InvalidOperationException ($"Unknown ABI {abi}");
@@ -351,15 +353,29 @@ void AddEnvironment ()
 				}
 			}
 
+			var uniqueNativeLibraries = new List<ITaskItem> ();
+			if (NativeLibraries != null) {
+				var seenNativeLibraryNames = new HashSet<string> (StringComparer.OrdinalIgnoreCase);
+				foreach (ITaskItem item in NativeLibraries) {
+					// We don't care about different ABIs here, just the file name
+					string name = Path.GetFileName (item.ItemSpec);
+					if (seenNativeLibraryNames.Contains (name)) {
+						continue;
+					}
+
+					seenNativeLibraryNames.Add (name);
+					uniqueNativeLibraries.Add (item);
+				}
+			}
+
 			bool haveRuntimeConfigBlob = !String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath);
 			var appConfState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<ApplicationConfigTaskState> (ApplicationConfigTaskState.RegisterTaskObjectKey, RegisteredTaskObjectLifetime.Build);
 
 			foreach (string abi in SupportedAbis) {
-				NativeAssemblerTargetProvider asmTargetProvider = GetAssemblyTargetProvider (abi);
 				string baseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"environment.{abi.ToLowerInvariant ()}");
 				string asmFilePath = $"{baseAsmFilePath}.s";
 
-				var asmgen = new ApplicationConfigNativeAssemblyGenerator (asmTargetProvider, baseAsmFilePath, environmentVariables, systemProperties) {
+				var asmgen = new ApplicationConfigNativeAssemblyGenerator (GetAndroidTargetArchForAbi (abi), environmentVariables, systemProperties, Log) {
 					IsBundledApp = IsBundledApplication,
 					UsesMonoAOT = usesMonoAOT,
 					UsesMonoLLVM = EnableLLVM,
@@ -379,11 +395,12 @@ void AddEnvironment ()
 									  // runtime, thus the number 2 here. All architecture specific stores contain assemblies with the same names
 									  // and in the same order.
 					MonoComponents = monoComponents,
+					NativeLibraries = uniqueNativeLibraries,
 					HaveAssemblyStore = UseAssemblyStore,
 				};
 
 				using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) {
-					asmgen.Write (sw);
+					asmgen.Write (sw, asmFilePath);
 					sw.Flush ();
 					Files.CopyIfStreamChanged (sw.BaseStream, asmFilePath);
 				}

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Globalization;
 using System.IO;
 using System.Text;
 using System.Text.RegularExpressions;
@@ -33,15 +34,16 @@ public sealed class ApplicationConfig
 			public uint   system_property_count;
 			public uint   number_of_assemblies_in_apk;
 			public uint   bundled_assembly_name_width;
-			public uint   number_of_assembly_blobs;
+			public uint   number_of_assembly_store_files;
+			public uint   number_of_dso_cache_entries;
 			public uint   mono_components_mask;
 			public string android_package_name;
 		};
-		const uint ApplicationConfigFieldCount = 18;
+		const uint ApplicationConfigFieldCount = 19;
 
 		static readonly object ndkInitLock = new object ();
 		static readonly char[] readElfFieldSeparator = new [] { ' ', '\t' };
-		static readonly Regex stringLabelRegex = new Regex ("^\\.L\\.env\\.str\\.[0-9]+:", RegexOptions.Compiled);
+		static readonly Regex stringLabelRegex = new Regex ("^\\.L\\.autostr\\.[0-9]+:", RegexOptions.Compiled);
 
 		static readonly HashSet <string> expectedPointerTypes = new HashSet <string> (StringComparer.Ordinal) {
 			".long",
@@ -102,7 +104,6 @@ static ApplicationConfig ReadApplicationConfig (string envFile)
 
 					line = lines [++i];
 					field = GetField (envFile, line, i);
-
 					AssertFieldType (envFile, ".asciz", field [0], i);
 					strings [label] = AssertIsAssemblerString (envFile, field [1], i);
 					continue;
@@ -196,17 +197,22 @@ static ApplicationConfig ReadApplicationConfig (string envFile)
 						ret.bundled_assembly_name_width = ConvertFieldToUInt32 ("bundled_assembly_name_width", envFile, i, field [1]);
 						break;
 
-					case 15: // number_of_assembly_blobs: uint32_t / .word | .long
+					case 15: // number_of_assembly_store_files: uint32_t / .word | .long
 						Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}");
-						ret.number_of_assembly_blobs = ConvertFieldToUInt32 ("number_of_assembly_blobs", envFile, i, field [1]);
+						ret.number_of_assembly_store_files = ConvertFieldToUInt32 ("number_of_assembly_store_files", envFile, i, field [1]);
 						break;
 
-					case 16: // mono_components_mask: uint32_t / .word | .long
+					case 16: // number_of_dso_cache_entries: uint32_t / .word | .long
+						Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}");
+						ret.number_of_dso_cache_entries = ConvertFieldToUInt32 ("number_of_dso_cache_entries", envFile, i, field [1]);
+						break;
+
+					case 17: // mono_components_mask: uint32_t / .word | .long
 						Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}");
 						ret.mono_components_mask = ConvertFieldToUInt32 ("mono_components_mask", envFile, i, field [1]);
 						break;
 
-					case 17: // android_package_name: string / [pointer type]
+					case 18: // android_package_name: string / [pointer type]
 						Assert.IsTrue (expectedPointerTypes.Contains (field [0]), $"Unexpected pointer field type in '{envFile}:{i}': {field [0]}");
 						pointers.Add (field [1].Trim ());
 						break;
@@ -216,7 +222,7 @@ static ApplicationConfig ReadApplicationConfig (string envFile)
 				if (String.Compare (".size", field [0], StringComparison.Ordinal) == 0) {
 					fieldCount--;
 					Assert.IsTrue (field [1].StartsWith ("application_config", StringComparison.Ordinal), $"Mismatched .size directive in '{envFile}:{i}'");
-					break; // We've reached the end of the application_config structure
+					gatherFields = false; // We've reached the end of the application_config structure
 				}
 			}
 			Assert.AreEqual (ApplicationConfigFieldCount, fieldCount, $"Invalid 'application_config' field count in environment file '{envFile}'");
@@ -280,7 +286,8 @@ static Dictionary<string, string> ReadEnvironmentVariables (string envFile)
 				field = GetField (envFile, line, i);
 				if (String.Compare (".size", field [0], StringComparison.Ordinal) == 0) {
 					Assert.IsTrue (field [1].StartsWith ("app_environment_variables", StringComparison.Ordinal), $"Mismatched .size directive in '{envFile}:{i}'");
-					break; // We've reached the end of the environment variable array
+					gatherPointers = false; // We've reached the end of the environment variable array
+					continue;
 				}
 
 				Assert.IsTrue (expectedPointerTypes.Contains (field [0]), $"Unexpected pointer field type in '{envFile}:{i}': {field [0]}");
@@ -311,13 +318,20 @@ static Dictionary<string, string> ReadEnvironmentVariables (string envFile)
 		static bool IsCommentLine (string line)
 		{
 			string l = line?.Trim ();
-			return !String.IsNullOrEmpty (l) && l.StartsWith ("/*", StringComparison.Ordinal);
+			if (String.IsNullOrEmpty (l)) {
+				return false;
+			}
+
+			return l.StartsWith ("/*", StringComparison.Ordinal) ||
+				l.StartsWith ("//", StringComparison.Ordinal) ||
+				l.StartsWith ("#", StringComparison.Ordinal) ||
+				l.StartsWith ("@", StringComparison.Ordinal);
 		}
 
 		static string[] GetField (string file, string line, int lineNumber)
 		{
 			string[] ret = line?.Trim ()?.Split ('\t');
-			Assert.AreEqual (2, ret.Length, $"Invalid assembler field format in file '{file}:{lineNumber}': '{line}'");
+			Assert.IsTrue (ret.Length >= 2, $"Invalid assembler field format in file '{file}:{lineNumber}': '{line}'");
 
 			return ret;
 		}
@@ -497,10 +511,11 @@ static void DumpLines (string streamName, List <string> lines)
 
 		static bool ConvertFieldToBool (string fieldName, string envFile, int fileLine, string value)
 		{
-			Assert.AreEqual (1, value.Length, $"Field '{fieldName}' in {envFile}:{fileLine} is not a valid boolean value (too long)");
+			// Allow both decimal and hexadecimal values
+			Assert.IsTrue (value.Length > 0 && value.Length <= 3, $"Field '{fieldName}' in {envFile}:{fileLine} is not a valid boolean value (length not between 1 and 3)");
 
 			uint fv;
-			Assert.IsTrue (UInt32.TryParse (value, out fv), $"Field '{fieldName}' in {envFile}:{fileLine} is not a valid boolean value (not a valid integer)");
+			Assert.IsTrue (TryParseInteger (value, out fv), $"Field '{fieldName}' in {envFile}:{fileLine} is not a valid boolean value (not a valid integer)");
 			Assert.IsTrue (fv == 0 || fv == 1, $"Field '{fieldName}' in {envFile}:{fileLine} is not a valid boolean value (not a valid boolean value 0 or 1)");
 
 			return fv == 1;
@@ -511,7 +526,7 @@ static uint ConvertFieldToUInt32 (string fieldName, string envFile, int fileLine
 			Assert.IsTrue (value.Length > 0, $"Field '{fieldName}' in {envFile}:{fileLine} is not a valid uint32_t value (not long enough)");
 
 			uint fv;
-			Assert.IsTrue (UInt32.TryParse (value, out fv), $"Field '{fieldName}' in {envFile}:{fileLine} is not a valid uint32_t value (not a valid integer)");
+			Assert.IsTrue (TryParseInteger (value, out fv), $"Field '{fieldName}' in {envFile}:{fileLine} is not a valid uint32_t value (not a valid integer)");
 
 			return fv;
 		}
@@ -521,9 +536,27 @@ static byte ConvertFieldToByte (string fieldName, string envFile, int fileLine,
 			Assert.IsTrue (value.Length > 0, $"Field '{fieldName}' in {envFile}:{fileLine} is not a valid uint8_t value (not long enough)");
 
 			byte fv;
-			Assert.IsTrue (Byte.TryParse (value, out fv), $"Field '{fieldName}' in {envFile}:{fileLine} is not a valid uint8_t value (not a valid integer)");
+			Assert.IsTrue (TryParseInteger (value, out fv), $"Field '{fieldName}' in {envFile}:{fileLine} is not a valid uint8_t value (not a valid integer)");
 
 			return fv;
 		}
+
+		static bool TryParseInteger (string value, out uint fv)
+		{
+			if (value.StartsWith ("0x", StringComparison.Ordinal)) {
+				return UInt32.TryParse (value.Substring (2), NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out fv);
+			}
+
+			return UInt32.TryParse (value, out fv);
+		}
+
+		static bool TryParseInteger (string value, out byte fv)
+		{
+			if (value.StartsWith ("0x", StringComparison.Ordinal)) {
+				return Byte.TryParse (value.Substring (2), NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out fv);
+			}
+
+			return Byte.TryParse (value, out fv);
+		}
 	}
 }

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj

@@ -6,6 +6,8 @@
     <OutputPath>..\..\..\..\bin\Test$(Configuration)</OutputPath>
     <SignAssembly>true</SignAssembly>
     <AssemblyOriginatorKeyFile>..\..\..\..\product.snk</AssemblyOriginatorKeyFile>
+    <DebugSymbols>True</DebugSymbols>
+    <DebugType>portable</DebugType>
   </PropertyGroup>
 
   <Import Project="..\..\..\..\Configuration.props" />