[XABT] Scan for JLO needed wrappers in LinkAssembliesNoShrink. (#9893)

Today, application builds use Cecil to scan through all Android assemblies at least twice:

- Debug: `_LinkAssembliesNoShrink`, `_GenerateJavaStubs`
- Release: `_ILLink`, `_GenerateJavaStubs`, `_RemoveRegisterAttribute`

This is a costly operation that we would like to only perform once to help speed up application builds.  The long-term goal is to move the (many) steps currently performed by `_GenerateJavaStubs` into "linker" steps.

This PR moves the Cecil scanning required for the first step: "generating Java Callable Wrappers".

## Implementation

This does not move generating JCW Java code to the linker, it only moves scanning for the JLO information needed to generate JCWs to the linker.  This information is persisted as an XML file in `/obj` next to the processed assembly.

Example:

- `obj/Debug/net10.0-android/android/assets/MyApplication.dll`
- `obj/Debug/net10.0-android/android/assets/MyApplication.jlo.xml`

Doing it this way has two benefits: it helps keep the linker from getting too complex (eg: generating Java code) and it helps incremental builds.  If an assembly has not changed since the last build, the JLO information in the `.jlo.xml` is still valid and will be reused.  The assembly does not need to be re-scanned.  (For example, the ~70 AndroidX assemblies that the MAUI template uses do not need to be re-scanned on every Debug build.)

The process of actually generating the JCWs is done by a new `GenerateJavaCallableWrappers` task in the `_GenerateJavaStubs` target that consumes the `.jlo.xml` files and outputs the `.java` files.

## Release Builds

In an ideal Release build, we would (probably?) run this new step as part of the `ILLink` invocation.  However there are some tricky limitations to running in the linker pipeline, chiefly being that the linker cannot load external assemblies like `Java.Interop.Tool.JavaCallableWrappers.dll` (and dependencies) that we need.  This can be worked around by including the needed source files in the linker assembly, but this will require additional work.

Instead, for now, we will add an invocation a new `_LinkAssembliesAdditionalSteps` target to linked builds that runs after `ILLink`. This calls the existing `LinkAssembliesNoShrink` task to scan for JLOs but does not execute the other linker tasks that have already been run. This allows us to move forward with converting to "linker steps" for making Debug builds faster without the extra work required to move these steps into `ILLink`.  A future enhancement can perform this work if desired.  (There is also [some question](dotnet/runtime#107211) as to whether we actually want to create new `ILLink` linker steps.)

Note this temporarily leaves the `$(_AndroidJLOCheckedBuild)` flag that compares the generated Java files to the previous ones.  This will likely be useful ensuring future moved steps are correct as well.

Author: jpobst

external/Java.Interop

@@ -0,0 +1,98 @@
+#nullable enable
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Java.Interop.Tools.JavaCallableWrappers;
+using Java.Interop.Tools.JavaCallableWrappers.Adapters;
+using Java.Interop.Tools.JavaCallableWrappers.CallableWrapperMembers;
+using Microsoft.Android.Build.Tasks;
+using Microsoft.Build.Utilities;
+using Mono.Cecil;
+using Mono.Linker;
+using Mono.Linker.Steps;
+using Xamarin.Android.Tasks;
+
+namespace MonoDroid.Tuner;
+
+/// <summary>
+/// Scans an assembly for JLOs that need JCWs generated and writes them to an XML file.
+/// </summary>
+public class FindJavaObjectsStep : BaseStep
+{
+	public string ApplicationJavaClass { get; set; } = "";
+
+	public bool ErrorOnCustomJavaObject { get; set; }
+
+	public bool UseMarshalMethods { get; set; }
+
+	public TaskLoggingHelper Log { get; set; }
+
+	public FindJavaObjectsStep (TaskLoggingHelper log) => Log = log;
+
+	public bool ProcessAssembly (AssemblyDefinition assembly, string destinationJLOXml)
+	{
+		var action = Annotations.HasAction (assembly) ? Annotations.GetAction (assembly) : AssemblyAction.Skip;
+
+		if (action == AssemblyAction.Delete)
+			return false;
+
+		var types = ScanForJavaTypes (assembly);
+		var initial_count = types.Count;
+
+		// Filter out Java types we don't care about
+		types = types.Where (t => !t.IsInterface && !JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (t, Context)).ToList ();
+
+		Log.LogDebugMessage ($"{assembly.Name.Name} - Found {initial_count} Java types, filtered to {types.Count}");
+
+		var wrappers = ConvertToCallableWrappers (types);
+
+		using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) {
+			XmlExporter.Export (sw, wrappers, true);
+			Files.CopyIfStreamChanged (sw.BaseStream, destinationJLOXml);
+		}
+
+		return true;
+	}
+
+	public static void WriteEmptyXmlFile (string destination)
+	{
+		XmlExporter.Export (destination, [], false);
+	}
+
+	List<TypeDefinition> ScanForJavaTypes (AssemblyDefinition assembly)
+	{
+		var types = new List<TypeDefinition> ();
+
+		var scanner = new XAJavaTypeScanner (Xamarin.Android.Tools.AndroidTargetArch.None, Log, Context) {
+			ErrorOnCustomJavaObject = ErrorOnCustomJavaObject
+		};
+
+		foreach (ModuleDefinition md in assembly.Modules) {
+			foreach (TypeDefinition td in md.Types) {
+				scanner.AddJavaType (td, types);
+			}
+		}
+
+		return types;
+	}
+
+	List<CallableWrapperType> ConvertToCallableWrappers (List<TypeDefinition> types)
+	{
+		var wrappers = new List<CallableWrapperType> ();
+
+		var reader_options = new CallableWrapperReaderOptions {
+			DefaultApplicationJavaClass = ApplicationJavaClass,
+			DefaultMonoRuntimeInitialization = "mono.MonoPackageManager.LoadApplication (context);",
+		};
+
+		if (UseMarshalMethods)
+			reader_options.MethodClassifier = new MarshalMethodsClassifier (Context, Context.Resolver, Log);
+
+		foreach (var type in types) {
+			var wrapper = CecilImporter.CreateType (type, Context, reader_options);
+			wrappers.Add (wrapper);
+		}
+
+		return wrappers;
+	}
+}

src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/FindJavaObjectsStep.cs

@@ -198,6 +198,7 @@ _ResolveAssemblies MSBuild target.
       _ResolveSatellitePaths;
       _CreatePackageWorkspace;
       _LinkAssemblies;
+      _AfterILLinkAdditionalSteps;
     </_PrepareAssembliesDependsOnTargets>
   </PropertyGroup>
 

src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets

@@ -67,6 +67,7 @@ properties that determine build ordering.
       _ResolveSatellitePaths;
       _CreatePackageWorkspace;
       _LinkAssemblies;
+      _AfterILLinkAdditionalSteps;
       _GenerateJavaStubs;
       _ManifestMerger;
       _ConvertCustomView;

src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets

@@ -0,0 +1,248 @@
+#nullable enable
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Java.Interop.Tools.Cecil;
+using Java.Interop.Tools.JavaCallableWrappers;
+using Java.Interop.Tools.TypeNameMappings;
+using Microsoft.Android.Build.Tasks;
+using Microsoft.Build.Framework;
+using Mono.Cecil;
+using MonoDroid.Tuner;
+using Xamarin.Android.Tools;
+using PackageNamingPolicyEnum = Java.Interop.Tools.TypeNameMappings.PackageNamingPolicy;
+
+namespace Xamarin.Android.Tasks;
+
+/// <summary>
+/// This task runs additional "linker steps" that are not part of ILLink. These steps
+/// are run *after* the linker has run. Additionally, this task is run by
+/// LinkAssembliesNoShrink to modify assemblies when ILLink is not used.
+/// </summary>
+public class AssemblyModifierPipeline : AndroidTask
+{
+	// Names of assemblies which don't have Mono.Android.dll references, or are framework assemblies, but which must
+	// be scanned for Java types.
+	static readonly HashSet<string> SpecialAssemblies = new HashSet<string> (StringComparer.OrdinalIgnoreCase) {
+			"Java.Interop.dll",
+			"Mono.Android.dll",
+			"Mono.Android.Runtime.dll",
+		};
+
+	public override string TaskPrefix => "AMP";
+
+	public string ApplicationJavaClass { get; set; } = "";
+
+	public string CodeGenerationTarget { get; set; } = "";
+
+	public bool Debug { get; set; }
+
+	[Required]
+	public ITaskItem [] DestinationFiles { get; set; } = [];
+
+	public bool Deterministic { get; set; }
+
+	public bool EnableMarshalMethods { get; set; }
+
+	public bool ErrorOnCustomJavaObject { get; set; }
+
+	public string? PackageNamingPolicy { get; set; }
+
+	/// <summary>
+	/// Defaults to false, enables Mono.Cecil to load symbols
+	/// </summary>
+	public bool ReadSymbols { get; set; }
+
+	/// <summary>
+	/// These are used so we have the full list of SearchDirectories
+	/// </summary>
+	[Required]
+	public ITaskItem [] ResolvedAssemblies { get; set; } = [];
+
+	[Required]
+	public ITaskItem [] ResolvedUserAssemblies { get; set; } = [];
+
+	[Required]
+	public ITaskItem [] SourceFiles { get; set; } = [];
+
+	protected JavaPeerStyle codeGenerationTarget;
+
+	public override bool RunTask ()
+	{
+		codeGenerationTarget = MonoAndroidHelper.ParseCodeGenerationTarget (CodeGenerationTarget);
+		JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out PackageNamingPolicyEnum pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64;
+
+		if (SourceFiles.Length != DestinationFiles.Length)
+			throw new ArgumentException ("source and destination count mismatch");
+
+		var readerParameters = new ReaderParameters {
+			ReadSymbols = ReadSymbols,
+		};
+
+		var writerParameters = new WriterParameters {
+			DeterministicMvid = Deterministic,
+		};
+
+		Dictionary<AndroidTargetArch, Dictionary<string, ITaskItem>> perArchAssemblies = MonoAndroidHelper.GetPerArchAssemblies (ResolvedAssemblies, Array.Empty<string> (), validate: false);
+
+		RunState? runState = null;
+		var currentArch = AndroidTargetArch.None;
+
+		for (int i = 0; i < SourceFiles.Length; i++) {
+			ITaskItem source = SourceFiles [i];
+			AndroidTargetArch sourceArch = GetValidArchitecture (source);
+			ITaskItem destination = DestinationFiles [i];
+			AndroidTargetArch destinationArch = GetValidArchitecture (destination);
+
+			if (sourceArch != destinationArch) {
+				throw new InvalidOperationException ($"Internal error: assembly '{sourceArch}' targets architecture '{sourceArch}', while destination assembly '{destination}' targets '{destinationArch}' instead");
+			}
+
+			// Each architecture must have a different set of context classes, or otherwise only the first instance of the assembly may be rewritten.
+			if (currentArch != sourceArch) {
+				currentArch = sourceArch;
+				runState?.Dispose ();
+
+				var resolver = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: ReadSymbols, loadReaderParameters: readerParameters);
+				runState = new RunState (resolver);
+
+				// Add SearchDirectories for the current architecture's ResolvedAssemblies
+				foreach (var kvp in perArchAssemblies [sourceArch]) {
+					ITaskItem assembly = kvp.Value;
+					var path = Path.GetFullPath (Path.GetDirectoryName (assembly.ItemSpec));
+					if (!runState.resolver.SearchDirectories.Contains (path)) {
+						runState.resolver.SearchDirectories.Add (path);
+					}
+				}
+
+				// Set up the FixAbstractMethodsStep and AddKeepAlivesStep
+				var context = new MSBuildLinkContext (runState.resolver, Log);
+
+				CreateRunState (runState, context);
+			}
+
+			Directory.CreateDirectory (Path.GetDirectoryName (destination.ItemSpec));
+
+			RunPipeline (source, destination, runState!, writerParameters);
+		}
+
+		runState?.Dispose ();
+
+		return !Log.HasLoggedErrors;
+	}
+
+	protected virtual void CreateRunState (RunState runState, MSBuildLinkContext context)
+	{
+		var findJavaObjectsStep = new FindJavaObjectsStep (Log) {
+			ApplicationJavaClass = ApplicationJavaClass,
+			ErrorOnCustomJavaObject = ErrorOnCustomJavaObject,
+			UseMarshalMethods = EnableMarshalMethods,
+		};
+
+		findJavaObjectsStep.Initialize (context);
+
+		runState.findJavaObjectsStep = findJavaObjectsStep;
+	}
+
+	protected virtual void RunPipeline (ITaskItem source, ITaskItem destination, RunState runState, WriterParameters writerParameters)
+	{
+		var destinationJLOXml = Path.ChangeExtension (destination.ItemSpec, ".jlo.xml");
+
+		if (!TryScanForJavaObjects (source, destination, runState, writerParameters)) {
+			// Even if we didn't scan for Java objects, we still write an empty .xml file for later steps
+			FindJavaObjectsStep.WriteEmptyXmlFile (destinationJLOXml);
+		}
+	}
+
+	bool TryScanForJavaObjects (ITaskItem source, ITaskItem destination, RunState runState, WriterParameters writerParameters)
+	{
+		if (!ShouldScanAssembly (source))
+			return false;
+
+		var destinationJLOXml = Path.ChangeExtension (destination.ItemSpec, ".jlo.xml");
+		var assemblyDefinition = runState.resolver!.GetAssembly (source.ItemSpec);
+
+		var scanned = runState.findJavaObjectsStep!.ProcessAssembly (assemblyDefinition, destinationJLOXml);
+
+		return scanned;
+	}
+
+	bool ShouldScanAssembly (ITaskItem source)
+	{
+		// Skip this assembly if it is not an Android assembly
+		if (!IsAndroidAssembly (source)) {
+			Log.LogDebugMessage ($"Skipping assembly '{source.ItemSpec}' because it is not an Android assembly");
+			return false;
+		}
+
+		// When marshal methods or non-JavaPeerStyle.XAJavaInterop1 are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during
+		// application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android
+		// build and stored in a jar file.
+		var useMarshalMethods = !Debug && EnableMarshalMethods;
+		var shouldSkipNonUserAssemblies = !useMarshalMethods && codeGenerationTarget == JavaPeerStyle.XAJavaInterop1;
+
+		if (shouldSkipNonUserAssemblies && !ResolvedUserAssemblies.Any (a => a.ItemSpec == source.ItemSpec)) {
+			Log.LogDebugMessage ($"Skipping assembly '{source.ItemSpec}' because it is not a user assembly and we don't need JLOs from non-user assemblies");
+			return false;
+		}
+
+		return true;
+	}
+
+	bool IsAndroidAssembly (ITaskItem source)
+	{
+		string name = Path.GetFileName (source.ItemSpec);
+
+		if (SpecialAssemblies.Contains (name))
+			return true;
+
+		return MonoAndroidHelper.IsMonoAndroidAssembly (source);
+	}
+
+	AndroidTargetArch GetValidArchitecture (ITaskItem item)
+	{
+		AndroidTargetArch ret = MonoAndroidHelper.GetTargetArch (item);
+		if (ret == AndroidTargetArch.None) {
+			throw new InvalidOperationException ($"Internal error: assembly '{item}' doesn't target any architecture.");
+		}
+
+		return ret;
+	}
+
+	protected sealed class RunState : IDisposable
+	{
+		public DirectoryAssemblyResolver resolver;
+		public FixAbstractMethodsStep? fixAbstractMethodsStep = null;
+		public AddKeepAlivesStep? addKeepAliveStep = null;
+		public FixLegacyResourceDesignerStep? fixLegacyResourceDesignerStep = null;
+		public FindJavaObjectsStep? findJavaObjectsStep = null;
+		bool disposed_value;
+
+		public RunState (DirectoryAssemblyResolver resolver)
+		{
+			this.resolver = resolver;
+		}
+
+		private void Dispose (bool disposing)
+		{
+			if (!disposed_value) {
+				if (disposing) {
+					resolver?.Dispose ();
+					fixAbstractMethodsStep = null;
+					fixLegacyResourceDesignerStep = null;
+					addKeepAliveStep = null;
+					findJavaObjectsStep = null;
+				}
+				disposed_value = true;
+			}
+		}
+
+		public void Dispose ()
+		{
+			// Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
+			Dispose (disposing: true);
+			GC.SuppressFinalize (this);
+		}
+	}
+}