diff --git a/NativeLinkingTODO.md b/NativeLinkingTODO.md new file mode 100644 index 00000000000..e284987451f --- /dev/null +++ b/NativeLinkingTODO.md @@ -0,0 +1,30 @@ +# Problems to solve + + * `libSystem.Security.Cryptography.Native.Android.a` contains the `JNI_OnLoad` function + which initializes the whole crypto support library, but we can't use it as it would + conflict with our own. Potential solution is to modify the above library's source code + to add an init function that we will call from our own `JNI_OnLoad` and make the library's + init function do the same. The `JNI_OnLoad` object file would have to be omitted from the + library's `.a` + * `p/invoke usage`. + Currently, all the BCL archives (with exception of the above crypto one) are + linked into the unified runtime using `--whole-archive` - that is, they become part of the + runtime in their entirety. This is wasteful, but necessary, so that `p/invokes` into those + libraries work correctly. Instead, we should scan the application DLLs for p/invokes from + those libraries and generate code to reference the required functions, so that the linker + can do its job and remove code not used by the application. Likely needed is a linker step. + * `p/invoke` handling mechanism. Right now, we `dlopen` the relevant `.so` library and look + up the required symbol in there. With the unified runtime the `.so` disappears, so we either + need to look it up in our own library or, better, call the function directly. The latter is + a bit more complicated to implement but would give us much faster code, thus it's the preferred + solution. + +# Helpers + + * linker can output a list of pinvokes, see https://github.com/dotnet/android/issues/7114 + +# Ideas + + * Use [mold](https://github.com/rui314/mold) which has recently been re-licensed under `MIT/X11` + (and contains components licensed under a mixture of `BSD*` and `Apache 2.0` licenses), so we + can easily redistribute it instead of the LLVM's `lld`. The advantage is `mold`'s [speed](https://github.com/rui314/mold?tab=readme-ov-file#mold-a-modern-linker) diff --git a/build-tools/create-packs/Microsoft.Android.Runtime.proj b/build-tools/create-packs/Microsoft.Android.Runtime.proj index 6ccb3fa4847..a0a99e844c2 100644 --- a/build-tools/create-packs/Microsoft.Android.Runtime.proj +++ b/build-tools/create-packs/Microsoft.Android.Runtime.proj @@ -42,7 +42,20 @@ projects that use the Microsoft.Android framework in .NET 6+. <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libmono-android.release.so" /> <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libxamarin-debug-app-helper.so" /> <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libxamarin-native-tracing.so" /> - <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libunwind_xamarin.a" /> + <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libunwind_xamarin-release.a" /> + <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libruntime-base-release.a" /> + <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libxa-java-interop-release.a" /> + <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libxa-lz4-release.a" /> + <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libxa-shared-bits-release.a" /> + <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libmono-android.release-static-release.a" /> + <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libpinvoke-override-dynamic-release.a" /> + <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\crtbegin_so.o" /> + <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\crtend_so.o" /> + <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libclang_rt.builtins-*-android.a" /> + + + <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libunwind.a" /> + diff --git a/build-tools/installers/create-installers.targets b/build-tools/installers/create-installers.targets index 0a63d3bfcf0..52e40081d6b 100644 --- a/build-tools/installers/create-installers.targets +++ b/build-tools/installers/create-installers.targets @@ -180,17 +180,29 @@ <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)ELFSharp.dll" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)ManifestOverlays\Timing.xml" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm64\libc.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm64\libdl.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm64\liblog.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm64\libm.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm64\libz.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm\libc.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm\libdl.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm\liblog.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm\libm.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm\libz.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x64\libc.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x64\libdl.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x64\liblog.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x64\libm.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x64\libz.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x86\libc.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x86\libdl.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x86\liblog.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x86\libm.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)dsostubs\android-arm64\libarchive-dso-stub.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)dsostubs\android-arm\libarchive-dso-stub.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)dsostubs\android-x64\libarchive-dso-stub.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)dsostubs\android-x86\libarchive-dso-stub.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x86\libz.so" /> <_MSBuildTargetsSrcFiles Include="$(MSBuildTargetsSrcDir)\Xamarin.Android.AvailableItems.targets" /> diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs index 103d07403fc..20a10099129 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs @@ -146,6 +146,37 @@ public static partial class Defaults { "x86_64", "x86_64-linux-android" }, }; + public static readonly Dictionary AbiToRID = new (StringComparer.Ordinal) { + { "armeabi-v7a", "android-arm" }, + { "arm64-v8a", "android-arm64" }, + { "x86", "android-x86" }, + { "x86_64", "android-x64" }, + }; + + public static readonly Dictionary AbiToClangArch = new (StringComparer.Ordinal) { + { "armeabi-v7a", "arm" }, + { "arm64-v8a", "aarch64" }, + { "x86", "i686" }, + { "x86_64", "x86_64" }, + }; + + /// + /// Used in rules.mk generator. Files to include in the XA bundle archives. + /// + public static readonly List BundleZipsInclude = new List { + "$(ZIP_OUTPUT_BASENAME)/THIRD-PARTY-NOTICES.TXT", + "$(ZIP_OUTPUT_BASENAME)/bin/Debug", + "$(ZIP_OUTPUT_BASENAME)/bin/Release", + }; + + /// + /// Used in rules.mk generator. Files to exclude from the XA bundle archives. Must be syntactically + /// correct for GNU Make. + /// + public static readonly List BundleZipsExclude = new List { + "$(ZIP_OUTPUT_BASENAME)/bin/*/bundle-*.zip" + }; + public static readonly List NDKTools = new List { // Tools prefixed with architecture triple new NDKTool (name: "as", prefixed: true), @@ -220,6 +251,7 @@ public static partial class Paths // Other public static string AndroidNdkDirectory => ctx.Properties.GetRequiredValue (KnownProperties.AndroidNdkDirectory); public static string AndroidToolchainRootDirectory => GetCachedPath (ref androidToolchainRootDirectory, () => Path.Combine (AndroidNdkDirectory, "toolchains", "llvm", "prebuilt", NdkToolchainOSTag)); + public static string AndroidClangRootDirectory => GetCachedPath (ref androidClangRootDirectory, () => Path.Combine (AndroidToolchainRootDirectory, "lib", "clang")); public static string AndroidToolchainBinDirectory => GetCachedPath (ref androidToolchainBinDirectory, () => Path.Combine (AndroidToolchainRootDirectory, "bin")); public static string AndroidToolchainSysrootLibDirectory => GetCachedPath (ref androidToolchainSysrootLibDirectory, () => Path.Combine (AndroidToolchainRootDirectory, "sysroot", "usr", "lib")); public static string WindowsBinutilsInstallDir => GetCachedPath (ref windowsBinutilsInstallDir, () => Path.Combine (InstallMSBuildDir, "binutils")); @@ -264,6 +296,7 @@ static string GetCachedPath (ref string? variable, Func creator) static string? buildBinDir; static string? binDir; static string? androidToolchainRootDirectory; + static string? androidClangRootDirectory; static string? androidToolchainBinDirectory; static string? androidToolchainSysrootLibDirectory; static string? installMSBuildDir; diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs b/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs index 5879fcbc703..0ec32b455f9 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Globalization; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -151,6 +152,10 @@ bool AcceptLicenses (Context context, string sdkRoot) bool GatherNDKInfo (Context context) { + if (!CopyRedistributableFiles (context)) { + return false; + } + // Ignore NDK property setting if not installing the NDK if (!DependencyTypeToInstall.HasFlag (AndroidToolchainComponentType.BuildDependency)) return true; @@ -158,6 +163,71 @@ bool GatherNDKInfo (Context context) return context.BuildInfo.GatherNDKInfo (context); } + bool CopyRedistributableFiles (Context context) + { + string androidVersionPath = Path.Combine (Configurables.Paths.AndroidToolchainRootDirectory, "AndroidVersion.txt"); + if (!File.Exists (androidVersionPath)) { + throw new InvalidOperationException ($"Android version file '{androidVersionPath}' not found"); + } + + string[]? lines = File.ReadAllLines (androidVersionPath); + if (lines == null || lines.Length < 1) { + throw new InvalidOperationException ($"Unknown format of Android version file '{androidVersionPath}'"); + } + + // First line is (should be) the LLVM version, we need just the main release number + string[] llvmVersion = lines[0].Split ('.'); + if (llvmVersion.Length < 3) { + throw new InvalidOperationException ($"Unknown LLVM version format for '{lines[0]}'"); + } + + string clangLibPath = Path.Combine ( + Configurables.Paths.AndroidClangRootDirectory, + llvmVersion[0], + "lib", + "linux" + ); + + foreach (var kvp in Configurables.Defaults.AndroidToolchainPrefixes) { + string abi = kvp.Key; + + string crtFilesPath = Path.Combine ( + Configurables.Paths.AndroidToolchainSysrootLibDirectory, + kvp.Value, + BuildAndroidPlatforms.NdkMinimumAPI.ToString (CultureInfo.InvariantCulture) + ); + + string clangArch = Configurables.Defaults.AbiToClangArch[abi]; + CopyFile (abi, crtFilesPath, "crtbegin_so.o"); + CopyFile (abi, crtFilesPath, "crtend_so.o"); + CopyFile (abi, clangLibPath, $"libclang_rt.builtins-{clangArch}-android.a"); + + // Yay, consistency + if (String.Compare (clangArch, "i686", StringComparison.Ordinal) == 0) { + clangArch = "i386"; + } + + // Remove once https://github.com/dotnet/runtime/pull/107615 is merged and released + CopyFile (abi, Path.Combine (clangLibPath, clangArch), "libunwind.a"); + } + + return true; + + void CopyFile (string abi, string sourceDir, string fileName) + { + Log.StatusLine ($" {context.Characters.Bullet} Copying NDK redistributable: ", $"{fileName} ({abi})", tailColor: ConsoleColor.White); + string rid = Configurables.Defaults.AbiToRID [abi]; + string outputDir = Path.Combine ( + context.Properties.GetRequiredValue (KnownProperties.MicrosoftAndroidSdkOutDir), + "lib", + rid + ); + + string sourceFile = Path.Combine (sourceDir, fileName); + Utilities.CopyFileToDir (sourceFile, outputDir); + } + } + void CheckPackageStatus (Context context, string packageCacheDir, AndroidPackage pkg, List toDownload) { Log.StatusLine ($" {context.Characters.Bullet} Installing ", pkg.Component.Name, tailColor: ConsoleColor.White); diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets index 8a64f834b80..b53b4737880 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets @@ -24,6 +24,9 @@ This file is imported *after* the Microsoft.NET.Sdk/Sdk.targets. + + + diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets index 10382ed3cc4..c6496b1f95a 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets @@ -216,7 +216,17 @@ _ResolveAssemblies MSBuild target. - + + + <_MonoComponent Condition=" '$(AndroidEnableProfiler)' == 'true' " Include="diagnostics_tracing" /> + <_MonoComponent Condition=" '$(AndroidUseInterpreter)' == 'true' " Include="hot_reload" /> + <_MonoComponent Condition=" '$(AndroidIncludeDebugSymbols)' == 'true' " Include="debugger" /> + <_MonoComponent Condition=" '$(_AndroidExcludeMarshalIlgenComponent)' != 'true' " Include="marshal-ilgen" /> + + + + <_AndroidIncludeSystemGlobalizationNative Condition=" '$(_AndroidIncludeSystemGlobalizationNative)' == '' ">true <_AndroidEnableNativeStackTracing Condition=" '$(_AndroidEnableNativeStackTracing)' == ''">false @@ -225,15 +235,12 @@ _ResolveAssemblies MSBuild target. <_ResolvedNativeLibraries Include="@(ResolvedFileToPublish)" Condition=" '%(ResolvedFileToPublish.Extension)' == '.so' " /> - <_MonoComponent Condition=" '$(AndroidEnableProfiler)' == 'true' " Include="diagnostics_tracing" /> - <_MonoComponent Condition=" '$(AndroidUseInterpreter)' == 'true' " Include="hot_reload" /> - <_MonoComponent Condition=" '$(AndroidIncludeDebugSymbols)' == 'true' " Include="debugger" /> - <_MonoComponent Condition=" '$(_AndroidExcludeMarshalIlgenComponent)' != 'true' " Include="marshal-ilgen" /> <_ExcludedNativeLibraries Condition=" '$(_AndroidIncludeSystemGlobalizationNative)' != 'true' " Include="libSystem.Globalization.Native" /> <_ExcludedNativeLibraries Condition=" '$(_AndroidEnableNativeStackTracing)' != 'true' " Include="libxamarin-native-tracing" /> + + + + + + + <_UnifiedNativeRuntime Include="$(_AndroidApplicationSharedLibraryPath)%(_BuildTargetAbis.Identity)\libmonodroid-unified.so"> + %(_BuildTargetAbis.Identity) + libmonodroid.so + + + + + <_ResolvedNativeArchive Include="@(ResolvedFileToPublish)" Condition=" '%(ResolvedFileToPublish.Extension)' == '.a' " /> + <_ResolvedNativeObjectFile Include="@(ResolvedFileToPublish)" Condition=" '%(ResolvedFileToPublish.Extension)' == '.o' " /> + <_ApplicationSharedLibrary Include="@(_UnifiedNativeRuntime)" /> + + + + + + + + + + + + + + + + + + + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 024b181fc66..49aa7d3cae8 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -627,7 +627,12 @@ void AddRuntimeLibraries (ZipArchiveEx apk, string [] supportedAbis) foreach (ITaskItem item in ApplicationSharedLibraries) { if (String.Compare (abi, item.GetMetadata ("abi"), StringComparison.Ordinal) != 0) continue; - AddNativeLibraryToArchive (apk, abi, item.ItemSpec, Path.GetFileName (item.ItemSpec), item); + string? inArchiveFileName = item.GetMetadata ("ArchiveFileName"); + if (String.IsNullOrEmpty (inArchiveFileName)) { + inArchiveFileName = Path.GetFileName (item.ItemSpec); + } + + AddNativeLibraryToArchive (apk, abi, item.ItemSpec, inArchiveFileName, item); } } } @@ -684,6 +689,7 @@ private void AddNativeLibraries (ArchiveFileList files, string [] supportedAbis) Item = v, }); + Log.LogDebugMessage ("BuildApk: adding framework native libraries"); AddNativeLibraries (files, supportedAbis, frameworkLibs); var libs = NativeLibraries.Concat (BundleNativeLibraries ?? Enumerable.Empty ()) @@ -697,6 +703,7 @@ private void AddNativeLibraries (ArchiveFileList files, string [] supportedAbis) } ); + Log.LogDebugMessage ("BuildApk: adding other native libraries"); AddNativeLibraries (files, supportedAbis, libs); if (String.IsNullOrWhiteSpace (CheckedBuild)) @@ -797,6 +804,7 @@ private void AddAdditionalNativeLibraries (ArchiveFileList files, string [] supp void AddNativeLibrary (ArchiveFileList files, string path, string abi, string archiveFileName, ITaskItem? taskItem = null) { + Log.LogDebugMessage ($"BuildApk: adding native library '{path}'; abi '{abi}'; archiveFileName '{archiveFileName}'"); string fileName = string.IsNullOrEmpty (archiveFileName) ? Path.GetFileName (path) : archiveFileName; var item = (filePath: path, archivePath: MakeArchiveLibPath (abi, fileName)); if (files.Any (x => x.archivePath == item.archivePath)) { diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 7c13c195b16..65c8be862c7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -90,6 +90,8 @@ public class GenerateJavaStubs : AndroidTask public ITaskItem[] Environments { get; set; } + public bool EnableNativeRuntimeLinking { get; set; } + [Output] public ITaskItem[] GeneratedBinaryTypeMaps { get; set; } @@ -186,6 +188,7 @@ void Run (bool useMarshalMethods) // Now that "never" never happened, we can proceed knowing that at least the assembly sets are the same for each architecture var nativeCodeGenStates = new ConcurrentDictionary (); NativeCodeGenState? templateCodeGenState = null; + PinvokeScanner? pinvokeScanner = EnableNativeRuntimeLinking ? new PinvokeScanner (Log) : null; var firstArch = allAssembliesPerArch.First ().Key; var generateSucceeded = true; @@ -205,6 +208,19 @@ void Run (bool useMarshalMethods) generateSucceeded = false; } + if (!success || state == null) { + return; + } + + if (pinvokeScanner != null) { + (success, List pinfos) = ScanForUsedPinvokes (pinvokeScanner, arch, state.Resolver); + if (!success) { + return; + } + state.PinvokeInfos = pinfos; + Log.LogDebugMessage ($"Number of unique p/invokes for architecture '{arch}': {pinfos.Count}"); + } + // If this is the first architecture, we need to store the state for later use if (generateJavaCode) { templateCodeGenState = state; @@ -373,6 +389,31 @@ IList MergeManifest (NativeCodeGenState codeGenState, Dictionary? pinfos) ScanForUsedPinvokes (PinvokeScanner scanner, AndroidTargetArch arch, XAAssemblyResolver resolver) + { + if (!EnableNativeRuntimeLinking) { + return (true, null); + } + + var frameworkAssemblies = new List (); + + foreach (ITaskItem asm in ResolvedAssemblies) { + string? metadata = asm.GetMetadata ("FrameworkAssembly"); + if (String.IsNullOrEmpty (metadata)) { + continue; + } + + if (!Boolean.TryParse (metadata, out bool isFrameworkAssembly) || !isFrameworkAssembly) { + continue; + } + + frameworkAssemblies.Add (asm); + } + + var pinfos = scanner.Scan (arch, resolver, frameworkAssemblies); + return (true, pinfos); + } + (bool success, NativeCodeGenState? stubsState) GenerateJavaSourcesAndMaybeClassifyMarshalMethods (AndroidTargetArch arch, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods, bool generateJavaCode) { XAAssemblyResolver resolver = MakeResolver (useMarshalMethods, arch, assemblies); diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index c0f7f0432ce..eef8f7e8c19 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -76,6 +76,7 @@ public class GeneratePackageManagerJava : AndroidTask public string AndroidSequencePointsMode { get; set; } public bool EnableSGenConcurrent { get; set; } public string? CustomBundleConfigFile { get; set; } + public bool EnableNativeRuntimeLinking { get; set; } bool _Debug { get { @@ -312,7 +313,7 @@ void AddEnvironment () } ConcurrentDictionary? nativeCodeGenStates = null; - if (enableMarshalMethods) { + if (enableMarshalMethods || EnableNativeRuntimeLinking) { nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal> ( ProjectSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateRegisterTaskKey), RegisteredTaskObjectLifetime.Build @@ -352,8 +353,10 @@ void AddEnvironment () string targetAbi = abi.ToLowerInvariant (); string environmentBaseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"environment.{targetAbi}"); string marshalMethodsBaseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"marshal_methods.{targetAbi}"); + string? pinvokePreserveBaseAsmFilePath = EnableNativeRuntimeLinking ? Path.Combine (EnvironmentOutputDirectory, $"pinvoke_preserve.{targetAbi}") : null; string environmentLlFilePath = $"{environmentBaseAsmFilePath}.ll"; string marshalMethodsLlFilePath = $"{marshalMethodsBaseAsmFilePath}.ll"; + string? pinvokePreserveLlFilePath = pinvokePreserveBaseAsmFilePath != null ? $"{pinvokePreserveBaseAsmFilePath}.ll" : null; AndroidTargetArch targetArch = GetAndroidTargetArchForAbi (abi); using var appConfigWriter = MemoryStreamPool.Shared.CreateStreamWriter (); @@ -383,6 +386,20 @@ void AddEnvironment () ); } + if (EnableNativeRuntimeLinking) { + var pinvokePreserveGen = new PreservePinvokesNativeAssemblyGenerator (Log, EnsureCodeGenState (targetArch), MonoComponents); + LLVMIR.LlvmIrModule pinvokePreserveModule = pinvokePreserveGen.Construct (); + using var pinvokePreserveWriter = MemoryStreamPool.Shared.CreateStreamWriter (); + try { + pinvokePreserveGen.Generate (pinvokePreserveModule, targetArch, pinvokePreserveWriter, pinvokePreserveLlFilePath); + } catch { + throw; + } finally { + pinvokePreserveWriter.Flush (); + Files.CopyIfStreamChanged (pinvokePreserveWriter.BaseStream, pinvokePreserveLlFilePath); + } + } + LLVMIR.LlvmIrModule marshalMethodsModule = marshalMethodsAsmGen.Construct (); using var marshalMethodsWriter = MemoryStreamPool.Shared.CreateStreamWriter (); try { diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs new file mode 100644 index 00000000000..0f6587035f0 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GetNativeRuntimeComponents.cs @@ -0,0 +1,131 @@ +using System; +using System.IO; +using System.Collections.Generic; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Microsoft.Android.Build.Tasks; + +namespace Xamarin.Android.Tasks; + +public class GetNativeRuntimeComponents : AndroidTask +{ + public override string TaskPrefix => "GNRC"; + + public ITaskItem[] MonoComponents { get; set; } + + [Required] + public ITaskItem[] ResolvedNativeArchives { get; set; } + + [Required] + public ITaskItem[] ResolvedNativeObjectFiles { get; set; } + + [Output] + public ITaskItem[] NativeArchives { get; set; } + + [Output] + public ITaskItem[] RequiredLibraries { get; set; } + + [Output] + public ITaskItem[] LinkStartFiles { get; set; } + + [Output] + public ITaskItem[] LinkEndFiles { get; set; } + + // TODO: more research, for now it seems `--export-dynamic-symbol=name` options generated from + // this array don't work as expected. + [Output] + public ITaskItem[] NativeSymbolsToExport { get; set; } + + public override bool RunTask () + { + var components = new NativeRuntimeComponents (MonoComponents); + var uniqueAbis = new HashSet (StringComparer.OrdinalIgnoreCase); + var archives = new List (); + var symbolsToExport = new List (); + + foreach (NativeRuntimeComponents.Archive archiveItem in components.KnownArchives) { + if (!archiveItem.Include) { + continue; + } + MakeArchiveItem (archiveItem, archives, uniqueAbis); + if (archiveItem.SymbolsToPreserve == null || archiveItem.SymbolsToPreserve.Count == 0) { + continue; + } + + foreach (string symbolName in archiveItem.SymbolsToPreserve) { + MakeLibItem (symbolName, symbolsToExport, uniqueAbis); + } + } + NativeArchives = archives.ToArray (); + NativeSymbolsToExport = symbolsToExport.ToArray (); + + var items = new List (); + foreach (string lib in components.NativeLibraries) { + MakeLibItem (lib, items, uniqueAbis); + } + RequiredLibraries = items.ToArray (); + + items = new List (); + foreach (string startFile in components.LinkStartFiles) { + MakeFileItem ("_NativeLinkStartFiles", startFile, ResolvedNativeObjectFiles, items, uniqueAbis); + } + LinkStartFiles = items.ToArray (); + + items = new List (); + foreach (string endFile in components.LinkEndFiles) { + MakeFileItem ("_NativeLinkEndFiles", endFile, ResolvedNativeObjectFiles, items, uniqueAbis); + } + LinkEndFiles = items.ToArray (); + + return !Log.HasLoggedErrors; + } + + void MakeLibItem (string libName, List libraries, HashSet uniqueAbis) + { + foreach (string abi in uniqueAbis) { + var item = new TaskItem (libName); + item.SetMetadata (KnownMetadata.Abi, abi); + libraries.Add (item); + } + } + + void MakeFileItem (string msbuildItemName, string fileName, ITaskItem[] inputItems, List outputItems, HashSet uniqueAbis) + { + foreach (ITaskItem item in inputItems) { + string name = Path.GetFileName (item.ItemSpec); + if (String.Compare (name, fileName, StringComparison.OrdinalIgnoreCase) == 0) { + outputItems.Add (DoMakeItem (msbuildItemName, item, uniqueAbis)); + } + } + } + + void MakeArchiveItem (NativeRuntimeComponents.Archive archive, List archives, HashSet uniqueAbis) + { + foreach (ITaskItem resolvedArchive in ResolvedNativeArchives) { + string fileName = Path.GetFileName (resolvedArchive.ItemSpec); + if (String.Compare (fileName, archive.Name, StringComparison.OrdinalIgnoreCase) != 0) { + continue; + } + + ITaskItem newItem = DoMakeItem ("_ResolvedNativeArchive", resolvedArchive, uniqueAbis); + newItem.SetMetadata (KnownMetadata.NativeLinkWholeArchive, archive.WholeArchive.ToString ()); + if (archive.DontExportSymbols) { + newItem.SetMetadata (KnownMetadata.NativeDontExportSymbols, "true"); + } + archives.Add (newItem); + } + } + + ITaskItem DoMakeItem (string msbuildItemName, ITaskItem sourceItem, HashSet uniqueAbis) + { + var ret = new TaskItem (sourceItem.ItemSpec); + string rid = sourceItem.GetRequiredMetadata (msbuildItemName, KnownMetadata.RuntimeIdentifier, Log); + string abi = MonoAndroidHelper.RidToAbi (rid); + uniqueAbis.Add (abi); + ret.SetMetadata (KnownMetadata.Abi, abi); + ret.SetMetadata (KnownMetadata.RuntimeIdentifier, rid); + + return ret; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs b/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs index 6fb2dac8967..4f0735996ea 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs @@ -179,11 +179,11 @@ IEnumerable GetLinkerConfigs () targetLinkerArgs.Add (elf_arch); foreach (string file in inputs.ObjectFiles) { - targetLinkerArgs.Add (QuoteFileName (file)); + targetLinkerArgs.Add (MonoAndroidHelper.QuoteFileNameArgument (file)); } targetLinkerArgs.Add ("-o"); - targetLinkerArgs.Add (QuoteFileName (inputs.OutputSharedLibrary)); + targetLinkerArgs.Add (MonoAndroidHelper.QuoteFileNameArgument (inputs.OutputSharedLibrary)); if (inputs.ExtraLibraries != null) { foreach (string lib in inputs.ExtraLibraries) { @@ -247,12 +247,5 @@ void OnErrorData (string linkerName, object sender, DataReceivedEventArgs e) if (e.Data != null) LogMessage ($"[{linkerName} stderr] {e.Data}"); } - - static string QuoteFileName (string fileName) - { - var builder = new CommandLineBuilder (); - builder.AppendFileNameIfNotNull (fileName); - return builder.ToString (); - } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs b/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs new file mode 100644 index 00000000000..034066c77b6 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/LinkNativeRuntime.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +using Microsoft.Android.Build.Tasks; + +namespace Xamarin.Android.Tasks; + +public class LinkNativeRuntime : AsyncTask +{ + public override string TaskPrefix => "LNR"; + + public ITaskItem[] MonoComponents { get; set; } + + [Required] + public string AndroidBinUtilsDirectory { get; set; } + + [Required] + public string IntermediateOutputPath { get; set; } + + [Required] + public ITaskItem[] LinkLibraries { get; set; } + + [Required] + public ITaskItem[] NativeArchives { get; set; } + + [Required] + public ITaskItem[] NativeObjectFiles { get; set; } + + [Required] + public ITaskItem[] NativeLinkStartFiles { get; set; } + + [Required] + public ITaskItem[] NativeLinkEndFiles { get; set; } + + [Required] + public ITaskItem[] NativeSymbolsToExport { get; set; } + + [Required] + public ITaskItem[] OutputRuntimes { get; set; } + + [Required] + public ITaskItem[] SupportedAbis { get; set; } + + public bool SaveDebugSymbols { get; set; } + + public override System.Threading.Tasks.Task RunTaskAsync () + { + return this.WhenAll (SupportedAbis, LinkRuntime); + } + + void LinkRuntime (ITaskItem abiItem) + { + string abi = abiItem.ItemSpec; + Log.LogDebugMessage ($"LinkRuntime ({abi})"); + ITaskItem outputRuntime = GetFirstAbiItem (OutputRuntimes, "_UnifiedNativeRuntime", abi); + string soname = Path.GetFileNameWithoutExtension (outputRuntime.ItemSpec); + if (soname.StartsWith ("lib", StringComparison.OrdinalIgnoreCase)) { + soname = soname.Substring (3); + } + + var linker = new NativeLinker (Log, abi, soname, AndroidBinUtilsDirectory, IntermediateOutputPath, CancellationToken, Cancel) { + SaveDebugSymbols = SaveDebugSymbols, + }; + linker.Link ( + outputRuntime, + GetAbiItems (NativeObjectFiles, "_NativeAssemblyTarget", abi), + GetAbiItems (NativeArchives, "_SelectedNativeArchive", abi), + GetAbiItems (LinkLibraries, "_RequiredLinkLibraries", abi), + GetAbiItems (NativeLinkStartFiles, "_NativeLinkStartFiles", abi), + GetAbiItems (NativeLinkEndFiles, "_NativeLinkEndFiles", abi), + GetAbiItems (NativeSymbolsToExport, "_NativeSymbolsToExport", abi) + ); + } + + List GetAbiItems (ITaskItem[] source, string itemName, string abi) + { + var ret = new List (); + + foreach (ITaskItem item in source) { + if (AbiMatches (abi, item, itemName)) { + ret.Add (item); + } + } + + return ret; + } + + ITaskItem GetFirstAbiItem (ITaskItem[] source, string itemName, string abi) + { + foreach (ITaskItem item in source) { + if (AbiMatches (abi, item, itemName)) { + return item; + } + } + + throw new InvalidOperationException ($"Internal error: item '{itemName}' for ABI '{abi}' not found"); + } + + bool AbiMatches (string abi, ITaskItem item, string itemName) + { + return String.Compare (abi, item.GetRequiredMetadata (itemName, "Abi", Log), StringComparison.OrdinalIgnoreCase) == 0; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs index fe494103982..f15d8f3459b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs @@ -16,6 +16,7 @@ public class PrepareAbiItems : AndroidTask const string CompressedAssembliesBase = "compressed_assemblies"; const string JniRemappingBase = "jni_remap"; const string MarshalMethodsBase = "marshal_methods"; + const string PinvokePreserveBase = "pinvoke_preserve"; public override string TaskPrefix => "PAI"; @@ -53,6 +54,8 @@ public override bool RunTask () baseName = JniRemappingBase; } else if (String.Compare ("marshal_methods", Mode, StringComparison.OrdinalIgnoreCase) == 0) { baseName = MarshalMethodsBase; + } else if (String.Compare ("runtime_linking", Mode, StringComparison.OrdinalIgnoreCase) == 0) { + baseName = PinvokePreserveBase; } else { Log.LogError ($"Unknown mode: {Mode}"); return false; diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessNativeLibraries.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessNativeLibraries.cs index 4970875d659..386909216c6 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessNativeLibraries.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessNativeLibraries.cs @@ -22,6 +22,19 @@ public class ProcessNativeLibraries : AndroidTask "libxamarin-debug-app-helper", }; + // Please keep the list sorted. Any new runtime libraries that are added upstream need to be mentioned here. + static readonly HashSet KnownRuntimeNativeLibraries = new (StringComparer.OrdinalIgnoreCase) { + "libSystem.Globalization.Native.so", + "libSystem.IO.Compression.Native.so", + "libSystem.Native.so", + "libSystem.Security.Cryptography.Native.Android.so", + "libmono-component-debugger.so", + "libmono-component-diagnostics_tracing.so", + "libmono-component-hot_reload.so", + "libmono-component-marshal-ilgen.so", + "libmonosgen-2.0.so", + }; + /// /// Assumed to be .so files only /// @@ -30,6 +43,7 @@ public class ProcessNativeLibraries : AndroidTask public string [] ExcludedLibraries { get; set; } public bool IncludeDebugSymbols { get; set; } + public bool NativeRuntimeLinking { get; set; } [Output] public ITaskItem [] OutputLibraries { get; set; } @@ -64,6 +78,11 @@ public override bool RunTask () // We may eventually have files such as `libmono-android-checked+asan.release.so` as well. var fileName = Path.GetFileNameWithoutExtension (library.ItemSpec); if (fileName.StartsWith ("libmono-android", StringComparison.Ordinal)) { + if (NativeRuntimeLinking) { + // We don't need the precompiled runtime, it will be linked during application build + continue; + } + if (fileName.EndsWith (".debug", StringComparison.Ordinal)) { if (!IncludeDebugSymbols) continue; @@ -87,12 +106,31 @@ public override bool RunTask () continue; } - output.Add (library); + if (!IgnoreLibraryWhenLinkingRuntime (library)) { + output.Add (library); + } } OutputLibraries = output.ToArray (); return !Log.HasLoggedErrors; } + + bool IgnoreLibraryWhenLinkingRuntime (ITaskItem libItem) + { + if (!NativeRuntimeLinking) { + return false; + } + + // We ignore all the shared libraries coming from the runtime packages, as they are all linked into our runtime and + // need not be packaged. + string packageId = libItem.GetMetadata ("NuGetPackageId"); + if (packageId.StartsWith ("Microsoft.NETCore.App.Runtime.Mono.android-", StringComparison.OrdinalIgnoreCase)) { + return true; + } + + // Should `NuGetPackageId` be empty, we check the libs by name, as the last resort. + return KnownRuntimeNativeLibraries.Contains (Path.GetFileName (libItem.ItemSpec)); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.TestCaseSource.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.TestCaseSource.cs index f30f2f2388d..959eb989f47 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.TestCaseSource.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.TestCaseSource.cs @@ -17,96 +17,119 @@ public partial class BuildTest : BaseTest /* isRelease */ false, /* aot */ false, /* usesAssemblyStore */ false, + /* useRuntimeLinking */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm", /* isRelease */ false, /* aot */ false, /* usesAssemblyStore */ true, + /* useRuntimeLinking */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm64", /* isRelease */ false, /* aot */ false, /* usesAssemblyStore */ false, + /* useRuntimeLinking */ false, }, new object [] { /* runtimeIdentifiers */ "android-x86", /* isRelease */ false, /* aot */ false, /* usesAssemblyStore */ false, + /* useRuntimeLinking */ false, }, new object [] { /* runtimeIdentifiers */ "android-x64", /* isRelease */ false, /* aot */ false, /* usesAssemblyStore */ false, + /* useRuntimeLinking */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm", /* isRelease */ true, /* aot */ false, /* usesAssemblyStore */ false, + /* useRuntimeLinking */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm", /* isRelease */ true, /* aot */ false, /* usesAssemblyStore */ true, + /* useRuntimeLinking */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm", /* isRelease */ true, /* aot */ true, /* usesAssemblyStore */ false, + /* useRuntimeLinking */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm", /* isRelease */ true, /* aot */ true, /* usesAssemblyStore */ true, + /* useRuntimeLinking */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm64", /* isRelease */ true, /* aot */ false, /* usesAssemblyStore */ false, + /* useRuntimeLinking */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ false, /* aot */ false, /* usesAssemblyStore */ false, + /* useRuntimeLinking */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ false, /* aot */ false, /* usesAssemblyStore */ true, + /* useRuntimeLinking */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86", /* isRelease */ true, /* aot */ false, /* usesAssemblyStore */ false, + /* useRuntimeLinking */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ true, /* aot */ false, /* usesAssemblyStore */ false, + /* useRuntimeLinking */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ true, /* aot */ false, /* usesAssemblyStore */ true, + /* useRuntimeLinking */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ true, /* aot */ true, /* usesAssemblyStore */ false, + /* useRuntimeLinking */ false, + }, + new object [] { + /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", + /* isRelease */ true, + /* aot */ false, + /* usesAssemblyStore */ true, + /* useRuntimeLinking */ true, }, }; diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index 8cc76792220..00e310ba055 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -26,8 +26,13 @@ public partial class BuildTest : BaseTest [Category ("SmokeTests")] [TestCaseSource (nameof (DotNetBuildSource))] [NonParallelizable] // On MacOS, parallel /restore causes issues - public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot, bool usesAssemblyStore) + public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot, bool usesAssemblyStore, bool useRuntimeLinking) { + if (!isRelease && useRuntimeLinking) { + Assert.Warn ("Dynamic runtime linking is supported only in Release builds, disabling it for this Debug build."); + useRuntimeLinking = false; + } + var proj = new XamarinAndroidApplicationProject { IsRelease = isRelease, EnableDefaultItems = true, @@ -68,6 +73,7 @@ public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot, bo .Replace ("//${AFTER_ONCREATE}", @"button.Text = Resource.CancelButton;"); proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyStore.ToString ()); proj.SetProperty ("RunAOTCompilation", aot.ToString ()); + proj.SetProperty ("_AndroidEnableNativeRuntimeLinking", useRuntimeLinking.ToString ()); proj.OtherBuildItems.Add (new AndroidItem.InputJar ("javaclasses.jar") { BinaryContent = () => ResourceData.JavaSourceJarTestJar, }); @@ -170,7 +176,11 @@ public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot, bo helper.AssertContainsEntry ($"assemblies/de-DE/{proj.ProjectName}.resources.dll", shouldContainEntry: expectEmbeddedAssembies); foreach (var abi in rids.Select (AndroidRidAbiHelper.RuntimeIdentifierToAbi)) { helper.AssertContainsEntry ($"lib/{abi}/libmonodroid.so"); - helper.AssertContainsEntry ($"lib/{abi}/libmonosgen-2.0.so"); + + if (!useRuntimeLinking) { + helper.AssertContainsEntry ($"lib/{abi}/libmonosgen-2.0.so"); + } + if (rids.Length > 1) { helper.AssertContainsEntry ($"assemblies/{abi}/System.Private.CoreLib.dll", shouldContainEntry: expectEmbeddedAssembies); } else { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs index 058269ceaa4..dbc61a84604 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs @@ -156,6 +156,9 @@ public void BuildReleaseArm64 ([Values (false, true)] bool forms) proj.SetProperty ("LinkerDumpDependencies", "True"); proj.SetProperty ("AndroidUseAssemblyStore", "False"); + // Make sure that we never use native runtime linking mode in this test. It would generate false negatives in the apkdesc checks. + proj.SetProperty ("_AndroidEnableNativeRuntimeLinking", "False"); + var flavor = (forms ? "XForms" : "Simple") + "DotNet"; var apkDescFilename = $"BuildReleaseArm64{flavor}.apkdesc"; var apkDescReference = "reference.apkdesc"; @@ -319,7 +322,7 @@ public void XA1037PropertyDeprecatedWarning (string property, string value, bool XamarinAndroidProject proj = isBindingProject ? new XamarinAndroidBindingProject () : new XamarinAndroidApplicationProject (); proj.IsRelease = isRelease; proj.SetProperty (property, value); - + using (ProjectBuilder b = isBindingProject ? CreateDllBuilder (Path.Combine ("temp", TestName)) : CreateApkBuilder (Path.Combine ("temp", TestName))) { Assert.IsTrue (b.Build (proj), "Build should have succeeded."); Assert.IsTrue (StringAssertEx.ContainsText (b.LastBuildOutput, $"The '{property}' MSBuild property is deprecated and will be removed"), diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildWithLibraryTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildWithLibraryTests.cs index a4ffdb5ca31..61f38a8edd6 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildWithLibraryTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildWithLibraryTests.cs @@ -413,6 +413,7 @@ public void BuildWithNativeLibraries ([Values (true, false)] bool isRelease) } }; proj.SetRuntimeIdentifiers (["armeabi-v7a", "x86"]); + proj.SetProperty ("_AndroidEnableNativeRuntimeLinking", "False"); var path = Path.Combine (Root, "temp", string.Format ("BuildWithNativeLibraries_{0}", isRelease)); using (var b1 = CreateDllBuilder (Path.Combine (path, dll2.ProjectName))) { Assert.IsTrue (b1.Build (dll2), "Build should have succeeded."); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/EnvironmentContentTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/EnvironmentContentTests.cs index 1d8ff21e199..82cc5fb78a4 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/EnvironmentContentTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/EnvironmentContentTests.cs @@ -15,7 +15,7 @@ public class EnvironmentContentTests : BaseTest { [Test] [NonParallelizable] - public void BuildApplicationWithMonoEnvironment ([Values ("", "Normal", "Offline")] string sequencePointsMode) + public void BuildApplicationWithMonoEnvironment ([Values ("", "Normal", "Offline")] string sequencePointsMode, [Values(false, true)] bool useNativeRuntimeLinkingMode) { const string supportedAbis = "armeabi-v7a;x86"; @@ -38,6 +38,7 @@ public void BuildApplicationWithMonoEnvironment ([Values ("", "Normal", "Offline string linkSkip = "FormsViewGroup"; app.SetProperty ("AndroidLinkSkip", linkSkip); app.SetProperty ("_AndroidSequencePointsMode", sequencePointsMode); + app.SetProperty ("_AndroidEnableNativeRuntimeLinking", useNativeRuntimeLinkingMode.ToString ()); app.SetAndroidSupportedAbis (supportedAbis); using (var libb = CreateDllBuilder (Path.Combine ("temp", TestName, lib.ProjectName))) using (var appb = CreateApkBuilder (Path.Combine ("temp", TestName, app.ProjectName))) { @@ -57,7 +58,7 @@ public void BuildApplicationWithMonoEnvironment ([Values ("", "Normal", "Offline if (!String.IsNullOrEmpty (sequencePointsMode)) Assert.IsTrue (monoDebugVar.IndexOf ("gen-compact-seq-points", StringComparison.Ordinal) >= 0, "The values from Mono.env should have been merged into environment"); - EnvironmentHelper.AssertValidEnvironmentSharedLibrary (intermediateOutputDir, AndroidSdkPath, AndroidNdkPath, supportedAbis); + EnvironmentHelper.AssertValidEnvironmentSharedLibrary (intermediateOutputDir, AndroidSdkPath, AndroidNdkPath, supportedAbis, useNativeRuntimeLinkingMode); var assemblyDir = Path.Combine (Root, appb.ProjectDirectory, app.IntermediateOutputPath, "android", "assets"); var rp = new ReaderParameters { ReadSymbols = false }; @@ -75,7 +76,7 @@ public void BuildApplicationWithMonoEnvironment ([Values ("", "Normal", "Offline } [Test] - public void CheckMonoDebugIsAddedToEnvironment ([Values ("", "Normal", "Offline")] string sequencePointsMode) + public void CheckMonoDebugIsAddedToEnvironment ([Values ("", "Normal", "Offline")] string sequencePointsMode, [Values(false, true)] bool useNativeRuntimeLinkingMode) { const string supportedAbis = "armeabi-v7a;x86"; @@ -83,6 +84,7 @@ public void CheckMonoDebugIsAddedToEnvironment ([Values ("", "Normal", "Offline" IsRelease = true, }; proj.SetProperty ("_AndroidSequencePointsMode", sequencePointsMode); + proj.SetProperty ("_AndroidEnableNativeRuntimeLinking", useNativeRuntimeLinkingMode.ToString ()); proj.SetAndroidSupportedAbis (supportedAbis); using (var b = CreateApkBuilder ()) { Assert.IsTrue (b.Build (proj), "Build should have succeeded."); @@ -101,7 +103,7 @@ public void CheckMonoDebugIsAddedToEnvironment ([Values ("", "Normal", "Offline" Assert.AreEqual ("gen-compact-seq-points", monoDebugVar, "environment should contain MONO_DEBUG=gen-compact-seq-points"); } - EnvironmentHelper.AssertValidEnvironmentSharedLibrary (intermediateOutputDir, AndroidSdkPath, AndroidNdkPath, supportedAbis); + EnvironmentHelper.AssertValidEnvironmentSharedLibrary (intermediateOutputDir, AndroidSdkPath, AndroidNdkPath, supportedAbis, useNativeRuntimeLinkingMode); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs index a06312f2fb3..94876152e62 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs @@ -958,8 +958,29 @@ public void CasingOnJavaLangObject () } } + static object[] GenerateJavaStubsAndAssemblyData () => new object[] { + new object[] { + true, // isRelease + true, // useNativeRuntimeLinkingMode + }, + + new object[] { + true, // isRelease + false, // useNativeRuntimeLinkingMode + }, + + new object[] { + false, // isRelease + false, // useNativeRuntimeLinkingMode + }, + + // Configuration for debug build and native runtime linking is not used, since + // runtime linking happens only in release builds + }; + [Test] - public void GenerateJavaStubsAndAssembly ([Values (true, false)] bool isRelease) + [TestCaseSource (nameof (GenerateJavaStubsAndAssemblyData))] + public void GenerateJavaStubsAndAssembly (bool isRelease, bool useNativeRuntimeLinkingMode) { var targets = new [] { "_GenerateJavaStubs", @@ -968,6 +989,7 @@ public void GenerateJavaStubsAndAssembly ([Values (true, false)] bool isRelease) var proj = new XamarinAndroidApplicationProject { IsRelease = isRelease, }; + proj.SetProperty ("_AndroidEnableNativeRuntimeLinking", useNativeRuntimeLinkingMode.ToString ()); proj.SetAndroidSupportedAbis ("armeabi-v7a"); proj.OtherBuildItems.Add (new AndroidItem.AndroidEnvironment ("Foo.txt") { TextContent = () => "Foo=Bar", @@ -978,7 +1000,7 @@ public void GenerateJavaStubsAndAssembly ([Values (true, false)] bool isRelease) foreach (var target in targets) { Assert.IsFalse (b.Output.IsTargetSkipped (target), $"`{target}` should *not* be skipped!"); } - AssertAssemblyFilesInFileWrites (proj, b); + AssertAssemblyFilesInFileWrites (proj, b, useNativeRuntimeLinkingMode); // Change C# file and AndroidEvironment file proj.MainActivity += Environment.NewLine + "// comment"; @@ -988,30 +1010,40 @@ public void GenerateJavaStubsAndAssembly ([Values (true, false)] bool isRelease) foreach (var target in targets) { Assert.IsFalse (b.Output.IsTargetSkipped (target), $"`{target}` should *not* be skipped!"); } - AssertAssemblyFilesInFileWrites (proj, b); + AssertAssemblyFilesInFileWrites (proj, b, useNativeRuntimeLinkingMode); // No changes Assert.IsTrue (b.Build (proj), "third build should have succeeded."); foreach (var target in targets) { Assert.IsTrue (b.Output.IsTargetSkipped (target), $"`{target}` should be skipped!"); } - AssertAssemblyFilesInFileWrites (proj, b); + AssertAssemblyFilesInFileWrites (proj, b, useNativeRuntimeLinkingMode); } } - readonly string [] ExpectedAssemblyFiles = new [] { - Path.Combine ("android", "environment.armeabi-v7a.o"), - Path.Combine ("android", "environment.armeabi-v7a.ll"), - Path.Combine ("android", "typemaps.armeabi-v7a.o"), - Path.Combine ("android", "typemaps.armeabi-v7a.ll"), - Path.Combine ("app_shared_libraries", "armeabi-v7a", "libxamarin-app.so") - }; + List GetExpectedAssemblyFiles (bool useNativeRuntimeLinkingMode) + { + var ret = new List { + Path.Combine ("android", "environment.armeabi-v7a.o"), + Path.Combine ("android", "environment.armeabi-v7a.ll"), + Path.Combine ("android", "typemaps.armeabi-v7a.o"), + Path.Combine ("android", "typemaps.armeabi-v7a.ll") + }; + + if (useNativeRuntimeLinkingMode) { + ret.Add (Path.Combine ("app_shared_libraries", "armeabi-v7a", "libmonodroid-unified.so")); + } else { + ret.Add (Path.Combine ("app_shared_libraries", "armeabi-v7a", "libxamarin-app.so")); + } + + return ret; + } - void AssertAssemblyFilesInFileWrites (XamarinAndroidApplicationProject proj, ProjectBuilder b) + void AssertAssemblyFilesInFileWrites (XamarinAndroidApplicationProject proj, ProjectBuilder b, bool useNativeRuntimeLinkingMode) { var intermediate = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath); var lines = File.ReadAllLines (Path.Combine (intermediate, $"{proj.ProjectName}.csproj.FileListAbsolute.txt")); - foreach (var file in ExpectedAssemblyFiles) { + foreach (var file in GetExpectedAssemblyFiles (useNativeRuntimeLinkingMode)) { var path = Path.Combine (intermediate, file); CollectionAssert.Contains (lines, path, $"{file} is not in FileWrites!"); FileAssert.Exists (path); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs index 075f0154f98..600da664c9c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs @@ -456,7 +456,7 @@ public static List GatherEnvironmentFiles (string outputDirecto return environmentFiles; } - public static void AssertValidEnvironmentSharedLibrary (string outputDirectoryRoot, string sdkDirectory, string ndkDirectory, string supportedAbis) + public static void AssertValidEnvironmentSharedLibrary (string outputDirectoryRoot, string sdkDirectory, string ndkDirectory, string supportedAbis, bool nativeRuntimeLinkingModeEnabled) { NdkTools ndk = NdkTools.Create (ndkDirectory); MonoAndroidHelper.AndroidSdk = new AndroidSdkInfo ((arg1, arg2) => {}, sdkDirectory, ndkDirectory, AndroidSdkResolver.GetJavaSdkPath ()); @@ -487,15 +487,25 @@ public static void AssertValidEnvironmentSharedLibrary (string outputDirectoryRo throw new Exception ("Unsupported Android target architecture ABI: " + abi); } - string envSharedLibrary = Path.Combine (outputDirectoryRoot, "app_shared_libraries", abi, "libxamarin-app.so"); - Assert.IsTrue (File.Exists (envSharedLibrary), $"Application environment SharedLibrary '{envSharedLibrary}' must exist"); + if (!nativeRuntimeLinkingModeEnabled) { + string envSharedLibrary = Path.Combine (outputDirectoryRoot, "app_shared_libraries", abi, "libxamarin-app.so"); + Assert.IsTrue (File.Exists (envSharedLibrary), $"Application environment SharedLibrary '{envSharedLibrary}' must exist"); - // API level doesn't matter in this case - var readelf = ndk.GetToolPath ("readelf", arch, 0); - if (!File.Exists (readelf)) { - readelf = ndk.GetToolPath ("llvm-readelf", arch, 0); + // API level doesn't matter in this case + var readelf = ndk.GetToolPath ("readelf", arch, 0); + if (!File.Exists (readelf)) { + readelf = ndk.GetToolPath ("llvm-readelf", arch, 0); + } + AssertSharedLibraryHasRequiredSymbols (envSharedLibrary, readelf); + } else { + string runtimeLibrary = Path.Combine (outputDirectoryRoot, "app_shared_libraries", abi, "libmonodroid-unified.so"); + Assert.IsTrue (File.Exists (runtimeLibrary), $"Application dynamically linked (unified) runtime library '{runtimeLibrary}' must exist"); + + runtimeLibrary = Path.Combine (outputDirectoryRoot, "app_shared_libraries", abi, "libmonodroid-unified.dbg.so"); + Assert.IsTrue (File.Exists (runtimeLibrary), $"Application dynamically linked (unified) runtime library '{runtimeLibrary}' debug symbols must exist"); + + // We can't verify fields in this mode, the symbols aren't exported. } - AssertSharedLibraryHasRequiredSymbols (envSharedLibrary, readelf); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs index 896c17e43c3..8305a5e5c2b 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs @@ -100,8 +100,11 @@ string Generate (string baseOutputDirectory, AndroidTargetArch arch, List (); var descriptors = new List (); ulong namesSize = 0; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/KnownMetadata.cs b/src/Xamarin.Android.Build.Tasks/Utilities/KnownMetadata.cs new file mode 100644 index 00000000000..0918aba990d --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/KnownMetadata.cs @@ -0,0 +1,9 @@ +namespace Xamarin.Android.Tasks; + +static class KnownMetadata +{ + public const string Abi = "Abi"; + public const string NativeLinkWholeArchive = "LinkWholeArchive"; + public const string RuntimeIdentifier = "RuntimeIdentifier"; + public const string NativeDontExportSymbols = "DontExportSymbols"; +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrCallingConvention.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrCallingConvention.cs new file mode 100644 index 00000000000..c9abcc34a0b --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrCallingConvention.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; + +namespace Xamarin.Android.Tasks.LLVMIR; + +/// +/// Function calling convention, see https://llvm.org/docs/LangRef.html#callingconv for more detailed docs. +/// Not all conventions are included in this enumeration, only those we may potentially need. +/// +enum LlvmIrCallingConvention +{ + /// + /// Outputs no keyword, making function use whatever is the compiler's default calling convention. + /// + Default, + + /// + /// The C calling convention (`ccc`) + /// + Ccc, + + /// + /// The fast calling convention (`fastcc`). This calling convention attempts to make calls as fast + /// as possible (e.g. by passing things in registers). + /// + Fastcc, + + /// + /// Tail callable calling convention (`tailcc`). This calling convention ensures that calls in tail + /// position will always be tail call optimized. This calling convention is equivalent to fastcc, + /// except for an additional guarantee that tail calls will be produced whenever possible. + /// + Tailcc, +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs index aa156ecaef9..10238567864 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs @@ -21,6 +21,7 @@ sealed class SavedParameterState : ILlvmIrSavedFunctionParameterState public bool? ReadNone; public bool? SignExt; public bool? ZeroExt; + public bool? WriteOnly; public bool? IsCplusPlusReference; public bool IsVarArgs; @@ -40,6 +41,7 @@ public SavedParameterState (LlvmIrFunctionParameter owner, SavedParameterState? ReadNone = previousState.ReadNone; SignExt = previousState.SignExt; ZeroExt = previousState.ZeroExt; + WriteOnly = previousState.WriteOnly; IsCplusPlusReference = previousState.IsCplusPlusReference; IsVarArgs = previousState.IsVarArgs; } @@ -131,6 +133,14 @@ public bool? ZeroExt { set => state.ZeroExt = value; } + /// + /// writeonly attribute, see + /// + public bool? WriteOnly { + get => state.WriteOnly; + set => state.WriteOnly = value; + } + /// /// This serves a purely documentational purpose, when generating comments about types. It describes a parameter that is a C++ reference, something we can't /// reflect on the managed side. @@ -411,6 +421,7 @@ public SavedFunctionState (LlvmIrFunction owner, ILlvmIrSavedFunctionSignatureSt public LlvmIrLinkage Linkage { get; set; } = LlvmIrLinkage.Default; public LlvmIrRuntimePreemption RuntimePreemption { get; set; } = LlvmIrRuntimePreemption.Default; public LlvmIrVisibility Visibility { get; set; } = LlvmIrVisibility.Default; + public LlvmIrCallingConvention CallingConvention { get; set; } = LlvmIrCallingConvention.Default; public LlvmIrFunctionBody Body { get; } public string? Comment { get; set; } public bool ReturnsValue => Signature.ReturnType != typeof(void); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs index 5c1b03990ae..f8cd42eab0a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs @@ -171,7 +171,7 @@ public LlvmIrFunctionBody (LlvmIrFunction func, LlvmIrFunction.FunctionState fun previousLabel = implicitStartBlock = new LlvmIrFunctionImplicitStartLabel (functionState.StartingBlockNumber); } - public void Add (LlvmIrFunctionLabelItem label) + public void Add (LlvmIrFunctionLabelItem label, string? comment = null) { label.WillAddToBody (this, functionState); if (definedLabels.Contains (label.Name)) { @@ -189,13 +189,16 @@ public void Add (LlvmIrFunctionLabelItem label) precedingBlock1 = previousLabel; previousLabel = label; - var comment = new StringBuilder (" preds = %"); - comment.Append (precedingBlock1.Name); - if (precedingBlock2 != null) { - comment.Append (", %"); - comment.Append (precedingBlock2.Name); + if (comment == null) { + var sb = new StringBuilder (" preds = %"); + sb.Append (precedingBlock1.Name); + if (precedingBlock2 != null) { + sb.Append (", %"); + sb.Append (precedingBlock2.Name); + } + comment = sb.ToString (); } - label.Comment = comment.ToString (); + label.Comment = comment; } public void Add (LlvmIrFunctionBodyItem item) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.Constants.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.Constants.cs index 9600dbc81e1..c71b7753b59 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.Constants.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.Constants.cs @@ -46,5 +46,12 @@ partial class LlvmIrGenerator { LlvmIrWritability.Constant, "constant" }, { LlvmIrWritability.Writable, "global" }, }; + + // https://llvm.org/docs/LangRef.html#callingconv + static readonly Dictionary llvmCallingConvention = new () { + { LlvmIrCallingConvention.Ccc, "ccc" }, + { LlvmIrCallingConvention.Fastcc, "fastcc" }, + { LlvmIrCallingConvention.Tailcc, "tailcc" }, + }; } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs index 7e29ecad9f2..a45fce3fcd9 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs @@ -3,6 +3,7 @@ using System.Collections; using System.Collections.Generic; using System.Globalization; +using System.Reflection; using System.IO; using System.Text; @@ -28,12 +29,14 @@ sealed class GeneratorWriteContext public readonly LlvmIrModuleTarget Target; public readonly LlvmIrMetadataManager MetadataManager; public readonly LlvmIrTypeCache TypeCache; + public readonly LlvmIrGenerator Generator; public string CurrentIndent { get; private set; } = String.Empty; public bool InVariableGroup { get; set; } public LlvmIrVariableNumberFormat NumberFormat { get; set; } = LlvmIrVariableNumberFormat.Default; - public GeneratorWriteContext (TextWriter writer, LlvmIrModule module, LlvmIrModuleTarget target, LlvmIrMetadataManager metadataManager, LlvmIrTypeCache cache) + public GeneratorWriteContext (LlvmIrGenerator generator, TextWriter writer, LlvmIrModule module, LlvmIrModuleTarget target, LlvmIrMetadataManager metadataManager, LlvmIrTypeCache cache) { + Generator = generator; Output = writer; Module = module; Target = target; @@ -163,7 +166,7 @@ public void Generate (TextWriter writer, LlvmIrModule module) LlvmIrMetadataManager metadataManager = module.GetMetadataManagerCopy (); target.AddTargetSpecificMetadata (metadataManager); - var context = new GeneratorWriteContext (writer, module, target, metadataManager, module.TypeCache); + var context = new GeneratorWriteContext (this, writer, module, target, metadataManager, module.TypeCache); if (!String.IsNullOrEmpty (FilePath)) { WriteCommentLine (context, $" ModuleID = '{FileName}'"); context.Output.WriteLine ($"source_filename = \"{FileName}\""); @@ -246,6 +249,12 @@ void WriteGlobalVariables (GeneratorWriteContext context) } } + void WriteVariableReference (GeneratorWriteContext context, LlvmIrVariableReference variable) + { + context.Output.Write (variable.Global ? '@' : '%'); + context.Output.Write (variable.Name); + } + void WriteGlobalVariableStart (GeneratorWriteContext context, LlvmIrGlobalVariable variable) { if (!String.IsNullOrEmpty (variable.Comment)) { @@ -353,6 +362,10 @@ ulong GetAggregateValueElementCount (GeneratorWriteContext context, Type type, o return (uint)((ICollection)value).Count; } + if (type.ImplementsInterface (typeof(ICollection<>))) { + return (ulong)GetCollectionOfTCount (type, value); + } + throw new InvalidOperationException ($"Internal error: should never get here"); } @@ -670,7 +683,7 @@ string ToHex (BasicType basicTypeDesc, Type type, object? value) return $"{(basicTypeDesc.IsUnsigned ? prefixUnsigned : prefixSigned)}0x{hex}"; } - void WriteValue (GeneratorWriteContext context, Type type, object? value) + public void WriteValue (GeneratorWriteContext context, Type type, object? value) { if (value is LlvmIrVariable variableRef) { context.Output.Write (variableRef.Reference); @@ -730,6 +743,11 @@ void WriteValue (GeneratorWriteContext context, Type type, object? value) throw new NotSupportedException ($"Internal error: array of type {type} is unsupported"); } + if (type == typeof (LlvmIrVariableReference) || type.IsSubclassOf (typeof (LlvmIrVariableReference))) { + WriteVariableReference (context, (LlvmIrVariableReference)value); + return; + } + throw new NotSupportedException ($"Internal error: value type '{type}' is unsupported"); } @@ -938,8 +956,19 @@ void WriteArrayValue (GeneratorWriteContext context, LlvmIrVariable variable) list.Add (kvp.Value); } entries = list; - } else { + } else if (variable.Type.ImplementsInterface (typeof (ICollection))) { entries = (ICollection)variable.Value; + } else if (variable.Type.ImplementsInterface (typeof (ICollection<>))) { + // This is slow and messy, but should work for a wide range of types without us having to add + // any explicit support + Type elementType = variable.Type.GetArrayElementType (); + int elementCount = GetCollectionOfTCount (variable.Type, variable.Value); + Array array = Array.CreateInstance (elementType, elementCount); + MethodInfo copyTo = variable.Type.GetMethod ("CopyTo", new Type[] { array.GetType (), typeof (int) }); + copyTo.Invoke (variable.Value, new object[] { array, (int)0 }); + entries = array; + } else { + throw new NotSupportedException ($"Unsupported array value type '{variable.Type}'"); } if (entries.Count == 0) { @@ -1226,6 +1255,11 @@ void WriteFunctionLeadingDecorations (GeneratorWriteContext context, LlvmIrFunct context.Output.Write (llvmVisibility[func.Visibility]); context.Output.Write (' '); } + + if (func.CallingConvention != LlvmIrCallingConvention.Default) { + context.Output.Write (llvmCallingConvention[func.CallingConvention]); + context.Output.Write (' '); + } } void WriteFunctionDeclarationTrailingDecorations (GeneratorWriteContext context, LlvmIrFunction func) @@ -1251,6 +1285,10 @@ void WriteFunctionTrailingDecorations (GeneratorWriteContext context, LlvmIrFunc public static void WriteReturnAttributes (GeneratorWriteContext context, LlvmIrFunctionSignature.ReturnTypeAttributes returnAttrs) { + if (AttributeIsSet (returnAttrs.InReg)) { + context.Output.Write ("inreg"); + } + if (AttributeIsSet (returnAttrs.NoUndef)) { context.Output.Write ("noundef "); } @@ -1349,6 +1387,10 @@ public static void WriteParameterAttributes (GeneratorWriteContext context, Llvm attributes.Add ("zeroext"); } + if (AttributeIsSet (parameter.WriteOnly)) { + attributes.Add ("writeonly"); + } + if (parameter.Align.HasValue) { attributes.Add ($"align({ValueOrPointerSize (parameter.Align.Value)})"); } @@ -1602,5 +1644,21 @@ public static string QuoteString (byte[] bytes, int byteCount, out ulong stringS return QuoteStringNoEscape (sb.ToString ()); } + + static int GetCollectionOfTCount (Type type, object instance) + { + if (!type.ImplementsInterface (typeof (ICollection<>))) { + throw new ArgumentException ("Must implement the ICollection interface", nameof (type)); + } + + PropertyInfo countProperty = type.GetProperty ("Count"); + var ret = (int)countProperty.GetValue (instance); + + if (ret < 0) { + throw new InvalidOperationException ($"Collection count is negative: {ret}"); + } + + return ret; + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs index 1d93cbec9e5..8c6848ef5ff 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs @@ -73,6 +73,8 @@ protected void WriteValue (GeneratorWriteContext context, Type type, object? val context.Output.Write ("null"); } else if (value is LlvmIrVariable variable) { context.Output.Write (variable.Reference); + } else if (value is bool) { + context.Output.Write ((bool)value ? "true" : "false"); } else { context.Output.Write (MonoAndroidHelper.CultureInvariantToString (value)); } @@ -510,11 +512,27 @@ protected override void WriteBody (GeneratorWriteContext context) public class Phi : LlvmIrInstruction { + sealed class Node + { + public readonly LlvmIrVariableReference? Variable; + public readonly LlvmIrFunctionLabelItem? Label; + + public Node (LlvmIrVariableReference? variable, LlvmIrFunctionLabelItem? label) + { + Variable = variable; + Label = label; + } + } + LlvmIrVariable result; - LlvmIrVariable val1; - LlvmIrFunctionLabelItem label1; - LlvmIrVariable val2; - LlvmIrFunctionLabelItem label2; + readonly List nodes; + + public Phi (LlvmIrVariable result) + : base ("phi") + { + nodes = new (); + this.result = result; + } /// /// Represents the `phi` instruction form we use the most throughout marshal methods generator - one which refers to an if/else block and where @@ -522,14 +540,20 @@ public class Phi : LlvmIrInstruction /// it is possible that hasn't had the required blocks defined prior to adding the `phi` instruction and, thus, /// we must check for the possibility here. /// - public Phi (LlvmIrVariable result, LlvmIrVariable val1, LlvmIrFunctionLabelItem? label1, LlvmIrVariable val2, LlvmIrFunctionLabelItem? label2) + public Phi (LlvmIrVariable result, LlvmIrVariableReference val1, LlvmIrFunctionLabelItem? label1, LlvmIrVariableReference val2, LlvmIrFunctionLabelItem? label2) : base ("phi") { this.result = result; - this.val1 = val1; - this.label1 = label1 ?? throw new ArgumentNullException (nameof (label1)); - this.val2 = val2; - this.label2 = label2 ?? throw new ArgumentNullException (nameof (label2)); + + nodes = new () { + new Node (val1, label1 ?? throw new ArgumentNullException (nameof (label1))), + new Node (val2, label2 ?? throw new ArgumentNullException (nameof (label2))), + }; + } + + public void AddNode (LlvmIrFunctionLabelItem label, LlvmIrVariableReference? variable) + { + nodes.Add (new Node (variable, label)); } protected override void WriteValueAssignment (GeneratorWriteContext context) @@ -541,14 +565,42 @@ protected override void WriteValueAssignment (GeneratorWriteContext context) protected override void WriteBody (GeneratorWriteContext context) { context.Output.Write (LlvmIrGenerator.MapToIRType (result.Type, context.TypeCache)); + context.IncreaseIndent (); + + bool first = true; + foreach (Node node in nodes) { + if (!first) { + context.Output.WriteLine (','); + } else { + first = false; + context.Output.WriteLine (); + } + context.Output.Write (context.CurrentIndent); + WriteNode (context, node); + } + context.DecreaseIndent (); + + // context.Output.Write (" ["); + // context.Output.Write (val1.Reference); + // context.Output.Write (", %"); + // context.Output.Write (label1.Name); + // context.Output.Write ("], ["); + // context.Output.Write (val2.Reference); + // context.Output.Write (", %"); + // context.Output.Write (label2.Name); + // context.Output.Write (']'); + } + + void WriteNode (GeneratorWriteContext context, Node node) + { + if (node.Label == null) { + throw new NotImplementedException ("Internal error: null labels not implemented"); + } + context.Output.Write (" ["); - context.Output.Write (val1.Reference); - context.Output.Write (", %"); - context.Output.Write (label1.Name); - context.Output.Write ("], ["); - context.Output.Write (val2.Reference); + context.Output.Write (node.Variable == null ? "null" : node.Variable.Reference); context.Output.Write (", %"); - context.Output.Write (label2.Name); + context.Output.Write (node.Label.Name); context.Output.Write (']'); } } @@ -594,6 +646,13 @@ public Store (LlvmIrVariable from, LlvmIrVariable to) this.to = to; } + public Store (object from, LlvmIrVariable to) + : base (Opcode) + { + this.from = from; + this.to = to; + } + /// /// Stores `null` in the indicated variable /// @@ -605,7 +664,8 @@ public Store (LlvmIrVariable to) protected override void WriteBody (GeneratorWriteContext context) { - string irType = LlvmIrGenerator.MapToIRType (to.Type, context.TypeCache, out ulong size, out bool isPointer); + Type type = from?.GetType () ?? to.Type; + string irType = LlvmIrGenerator.MapToIRType (type, context.TypeCache, out ulong size, out bool isPointer); context.Output.Write (irType); context.Output.Write (' '); @@ -624,4 +684,110 @@ public Unreachable () : base ("unreachable") {} } + + public class Switch : LlvmIrInstruction where T: struct + { + // Since we can't use System.Numerics.IBinaryInteger, this is the poor man's verification that T is acceptable for us + static readonly HashSet acceptedTypes = new () { + typeof (byte), + typeof (sbyte), + typeof (short), + typeof (ushort), + typeof (int), + typeof (uint), + typeof (long), + typeof (ulong), + }; + + readonly LlvmIrVariable value; + readonly LlvmIrFunctionLabelItem defaultDest; + readonly string? automaticLabelPrefix; + ulong automaticLabelCounter = 0; + List<(T constant, LlvmIrFunctionLabelItem label, string? comment)>? items; + + public Switch (LlvmIrVariable value, LlvmIrFunctionLabelItem defaultDest, string? automaticLabelPrefix = null) + : base ("switch") + { + if (!acceptedTypes.Contains (typeof(T))) { + throw new NotSupportedException ($"Type '{typeof(T)}' is unsupported, only integer types are accepted"); + } + + if (value.Type != typeof (T)) { + throw new ArgumentException ($"Must refer to value of type '{typeof(T)}'", nameof (value)); + } + + this.value = value; + this.defaultDest = defaultDest; + this.automaticLabelPrefix = automaticLabelPrefix; + + if (!String.IsNullOrEmpty (automaticLabelPrefix)) { + items = new (); + } + } + + protected override void WriteBody (GeneratorWriteContext context) + { + string irType = LlvmIrGenerator.MapToIRType (value.Type, context.TypeCache, out _, out bool isPointer); + + context.Output.Write (irType); + context.Output.Write (' '); + + WriteValue (context, value.Type, value, isPointer); + + context.Output.Write (", label %"); + context.Output.Write (defaultDest.Name); + context.Output.WriteLine (" ["); + context.IncreaseIndent (); + + foreach ((T constant, LlvmIrFunctionLabelItem label, string? comment) in items) { + context.Output.Write (context.CurrentIndent); + context.Output.Write (irType); + context.Output.Write (' '); + context.Generator.WriteValue (context, value.Type, constant); + context.Output.Write (", label %"); + context.Output.Write (label.Name); + if (!String.IsNullOrEmpty (comment)) { + context.Output.Write (' '); + context.Generator.WriteCommentLine (context, comment); + } else { + context.Output.WriteLine (); + } + } + + context.DecreaseIndent (); + context.Output.Write (context.CurrentIndent); + context.Output.Write (']'); + } + + public LlvmIrFunctionLabelItem Add (T val, LlvmIrFunctionLabelItem? dest = null, string? comment = null) + { + var label = MakeLabel (dest); + items.Add ((val, label, comment)); + return label; + } + + void EnsureValidity (LlvmIrFunctionLabelItem? dest) + { + if (dest != null) { + return; + } + + if (String.IsNullOrEmpty (automaticLabelPrefix)) { + throw new InvalidOperationException ($"Internal error: automatic label management requested, but prefix not defined"); + } + } + + LlvmIrFunctionLabelItem MakeLabel (LlvmIrFunctionLabelItem? maybeDest) + { + EnsureValidity (maybeDest); + if (maybeDest != null) { + return maybeDest; + } + + var ret = new LlvmIrFunctionLabelItem (automaticLabelCounter == 0 ? automaticLabelPrefix : $"{automaticLabelPrefix}{automaticLabelCounter}"); + automaticLabelCounter++; + + return ret; + } + } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs index d66bda3d7c9..324868328a2 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs @@ -21,12 +21,45 @@ enum LlvmIrVariableNumberFormat Decimal, } -abstract class LlvmIrVariable : IEquatable +abstract class LlvmIrVariableReference { public abstract bool Global { get; } public abstract string NamePrefix { get; } - public string? Name { get; protected set; } + + /// + /// Returns a string which constitutes a reference to a local (using the % prefix character) or a global + /// (using the @ prefix character) variable, ready for use in the generated code wherever variables are + /// referenced. + /// + public virtual string Reference { + get { + if (String.IsNullOrEmpty (Name)) { + throw new InvalidOperationException ("Variable doesn't have a name, it cannot be referenced"); + } + + return $"{NamePrefix}{Name}"; + } + } + + protected LlvmIrVariableReference (string name) + { + Name = name; + } +} + +class LlvmIrGlobalVariableReference : LlvmIrVariableReference +{ + public override bool Global => true; + public override string NamePrefix => "@"; + + public LlvmIrGlobalVariableReference (string name) + : base (name) + {} +} + +abstract class LlvmIrVariable : LlvmIrVariableReference, IEquatable +{ public Type Type { get; protected set; } public LlvmIrVariableWriteOptions WriteOptions { get; set; } = LlvmIrVariableWriteOptions.ArrayWriteIndexComments; @@ -47,21 +80,6 @@ abstract class LlvmIrVariable : IEquatable /// will ignore name when checking for equality. protected bool NameMatters { get; set; } = true; - /// - /// Returns a string which constitutes a reference to a local (using the % prefix character) or a global - /// (using the @ prefix character) variable, ready for use in the generated code wherever variables are - /// referenced. - /// - public virtual string Reference { - get { - if (String.IsNullOrEmpty (Name)) { - throw new InvalidOperationException ("Variable doesn't have a name, it cannot be referenced"); - } - - return $"{NamePrefix}{Name}"; - } - } - /// /// /// Certain data must be calculated when the target architecture is known, because it may depend on certain aspects of @@ -103,6 +121,7 @@ public virtual string Reference { /// is treated as an opaque pointer type. /// protected LlvmIrVariable (Type type, string? name = null) + : base (name) { Type = type; Name = name; @@ -253,7 +272,7 @@ class LlvmIrGlobalVariable : LlvmIrVariable public LlvmIrStreamedArrayDataProvider? ArrayDataProvider { get; set; } /// - /// Constructs a local variable. is translated to one of the LLVM IR first class types (see + /// Constructs a global variable. is translated to one of the LLVM IR first class types (see /// https://llvm.org/docs/LangRef.html#t-firstclass) only if it's an integral or floating point type. In all other cases it /// is treated as an opaque pointer type. is required because global variables must be named. /// diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/TypeUtilities.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/TypeUtilities.cs index 8865238bff0..eec6dd7a771 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/TypeUtilities.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/TypeUtilities.cs @@ -178,7 +178,7 @@ public static bool IsArray (this Type t) // TODO: cache results here // IDictionary is a special case for name:value string arrays which we use for some constructs. - return (t.ImplementsInterface (typeof(ICollection<>)) && t.ImplementsInterface (typeof(ICollection))) || + return (t.ImplementsInterface (typeof(ICollection<>)) || t.ImplementsInterface (typeof(ICollection))) || t.ImplementsInterface (typeof(IDictionary)); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs index 5a541f2d09c..4e106555df6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs @@ -30,6 +30,13 @@ class NativeCodeGenState /// public List AllJavaTypes { get; } + /// + /// Contains information about p/invokes used by the managed assemblies included in the + /// application. Will be **null** unless native runtime linking at application build time + /// is enabled. + /// + public List? PinvokeInfos { get; set; } + public List JavaTypesForJCW { get; } public XAAssemblyResolver Resolver { get; } public TypeDefinitionCache TypeCache { get; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs new file mode 100644 index 00000000000..8e8b5b2ec29 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeLinker.cs @@ -0,0 +1,370 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Collections.Generic; +using System.Text; +using System.Threading; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Microsoft.Android.Build.Tasks; + +namespace Xamarin.Android.Tasks; + +class NativeLinker +{ + static readonly List standardArgs = new () { + "--shared", + "--allow-shlib-undefined", + // TODO: need to enable zstd in binutils build + // "--compress-debug-sections=zstd", + // TODO: test the commented-out flags + "--gc-sections", + // "--icf=safe", + // "--lto=full|thin", + "--export-dynamic", + "-z relro", + "-z noexecstack", + "-z max-page-size=16384", + "--enable-new-dtags", + "--build-id=sha1", + "--warn-shared-textrel", + "--fatal-warnings", + "--no-rosegment" + }; + + readonly List extraArgs = new (); + readonly TaskLoggingHelper log; + readonly string abi; + readonly string ld; + readonly string objcopy; + readonly string intermediateDir; + readonly CancellationToken? cancellationToken; + readonly Action? cancelTask; + + public bool StripDebugSymbols { get; set; } + public bool SaveDebugSymbols { get; set; } + + public NativeLinker (TaskLoggingHelper log, string abi, string soname, string binutilsDir, string intermediateDir, + CancellationToken? cancellationToken = null, Action? cancelTask = null) + { + this.log = log; + this.abi = abi; + this.intermediateDir = intermediateDir; + this.cancellationToken = cancellationToken; + this.cancelTask = cancelTask; + + ld = Path.Combine (binutilsDir, MonoAndroidHelper.GetExecutablePath (binutilsDir, "ld")); + objcopy = Path.Combine (binutilsDir, MonoAndroidHelper.GetExecutablePath (binutilsDir, "llvm-objcopy")); + + extraArgs.Add ($"-soname {soname}"); + + string? elfArch = null; + switch (abi) { + case "armeabi-v7a": + extraArgs.Add ("-X"); + elfArch = "armelf_linux_eabi"; + break; + + case "arm64": + case "arm64-v8a": + case "aarch64": + extraArgs.Add ("--fix-cortex-a53-843419"); + elfArch = "aarch64linux"; + break; + + case "x86": + elfArch = "elf_i386"; + break; + + case "x86_64": + elfArch = "elf_x86_64"; + break; + + default: + throw new NotSupportedException ($"Unsupported Android target architecture ABI: {abi}"); + } + + if (!String.IsNullOrEmpty (elfArch)) { + extraArgs.Add ($"-m {elfArch}"); + } + + string runtimeNativeLibsDir = MonoAndroidHelper.GetNativeLibsRootDirectoryPath (binutilsDir); + string runtimeNativeLibStubsDir = MonoAndroidHelper.GetLibstubsRootDirectoryPath (binutilsDir); + string RID = MonoAndroidHelper.AbiToRid (abi); + string libStubsPath = Path.Combine (runtimeNativeLibStubsDir, RID); + string runtimeLibsDir = Path.Combine (runtimeNativeLibsDir, RID); + + extraArgs.Add ($"-L {MonoAndroidHelper.QuoteFileNameArgument (libStubsPath)}"); + extraArgs.Add ($"-L {MonoAndroidHelper.QuoteFileNameArgument (runtimeLibsDir)}"); + } + + public bool Link (ITaskItem outputLibraryPath, List objectFiles, List archives, List libraries, + List linkStartFiles, List linkEndFiles, ICollection? exportDynamicSymbols = null) + { + log.LogDebugMessage ($"Linking: {outputLibraryPath}"); + EnsureCorrectAbi (outputLibraryPath); + EnsureCorrectAbi (objectFiles); + EnsureCorrectAbi (archives); + EnsureCorrectAbi (libraries); + EnsureCorrectAbi (linkStartFiles); + EnsureCorrectAbi (linkEndFiles); + + Directory.CreateDirectory (Path.GetDirectoryName (outputLibraryPath.ItemSpec)); + + string libBaseName = Path.GetFileNameWithoutExtension (outputLibraryPath.ItemSpec); + string respFilePath = Path.Combine (intermediateDir, $"ld.{libBaseName}.{abi}.rsp"); + using var sw = new StreamWriter (File.Open (respFilePath, FileMode.Create, FileAccess.Write, FileShare.Read), new UTF8Encoding (false)); + foreach (string arg in standardArgs) { + sw.WriteLine (arg); + } + + foreach (string arg in extraArgs) { + sw.WriteLine (arg); + } + + if (StripDebugSymbols && !SaveDebugSymbols) { + sw.WriteLine ("-s"); + } + + var excludeExportsLibs = new List (); + WriteFilesToResponseFile (sw, linkStartFiles); + WriteFilesToResponseFile (sw, objectFiles); + WriteFilesToResponseFile (sw, archives); + + if (exportDynamicSymbols != null && exportDynamicSymbols.Count > 0) { + foreach (ITaskItem symbolItem in exportDynamicSymbols) { + sw.WriteLine ($"--export-dynamic-symbol={symbolItem.ItemSpec}"); + } + } + + if (excludeExportsLibs.Count > 0) { + string libs = String.Join (",", excludeExportsLibs); + sw.WriteLine ($"--exclude-libs={libs}"); + } + + foreach (ITaskItem libItem in libraries) { + sw.WriteLine ($"-l{libItem.ItemSpec}"); + } + + WriteFilesToResponseFile (sw, linkEndFiles); + sw.Flush (); + + var ldArgs = new List { + $"@{respFilePath}", + "-o", + MonoAndroidHelper.QuoteFileNameArgument (outputLibraryPath.ItemSpec) + }; + + var watch = new Stopwatch (); + watch.Start (); + bool ret = RunLinker (ldArgs, outputLibraryPath); + watch.Stop (); + log.LogDebugMessage ($"[{Path.GetFileName (outputLibraryPath.ItemSpec)} link time] {watch.Elapsed}"); + + if (!ret || !SaveDebugSymbols) { + return ret; + } + + ret = ExtractDebugSymbols (outputLibraryPath); + return ret; + + void WriteFilesToResponseFile (StreamWriter sw, List files) + { + foreach (ITaskItem file in files) { + bool wholeArchive = IncludeWholeArchive (file); + + if (ExcludeFromExports (file)) { + excludeExportsLibs.Add (Path.GetFileName (file.ItemSpec)); + } + + if (wholeArchive) { + sw.Write ("--whole-archive "); + } + sw.Write (MonoAndroidHelper.QuoteFileNameArgument (file.ItemSpec)); + // string abi = file.GetMetadata ("Abi") ?? String.Empty; + // string destDir = Path.Combine ("/tmp/t", abi); + // Directory.CreateDirectory (destDir); + // File.Copy (file.ItemSpec, Path.Combine (destDir, Path.GetFileName (file.ItemSpec))); + if (wholeArchive) { + sw.Write (" --no-whole-archive"); + } + sw.WriteLine (); + } + } + + bool IncludeWholeArchive (ITaskItem item) => ParseBooleanMetadata (item, KnownMetadata.NativeLinkWholeArchive); + bool ExcludeFromExports (ITaskItem item) => ParseBooleanMetadata (item, KnownMetadata.NativeDontExportSymbols); + + bool ParseBooleanMetadata (ITaskItem item, string metadata) + { + string? value = item.GetMetadata (metadata); + if (String.IsNullOrEmpty (value)) { + return false; + } + + // Purposefully not calling TryParse, let it throw and let us know if the value isn't a boolean. + return Boolean.Parse (value); + } + } + + void EnsureCorrectAbi (ITaskItem item) + { + // The exception is just a precaution, since the items passed to us should have already been checked + string itemAbi = item.GetMetadata (KnownMetadata.Abi) ?? throw new InvalidOperationException ($"Internal error: 'Abi' metadata not found in item '{item}'"); + if (String.Compare (abi, itemAbi, StringComparison.OrdinalIgnoreCase) == 0) { + return; + } + + throw new InvalidOperationException ($"Internal error: '{item}' ABI ('{itemAbi}') doesn't have the expected value '{abi}'"); + } + + void EnsureCorrectAbi (List items) + { + foreach (ITaskItem item in items) { + EnsureCorrectAbi (item); + } + } + + bool ExtractDebugSymbols (ITaskItem outputSharedLibrary) + { + var stdoutLines = new List (); + var stderrLines = new List (); + + string sourceLib = outputSharedLibrary.ItemSpec; + string sourceLibQuoted = MonoAndroidHelper.QuoteFileNameArgument (sourceLib); + string destLib = Path.Combine (Path.GetDirectoryName (sourceLib), $"{Path.GetFileNameWithoutExtension (sourceLib)}.dbg.so"); + string destLibQuoted = MonoAndroidHelper.QuoteFileNameArgument (destLib); + + var args = new List { + "--only-keep-debug", + sourceLibQuoted, + destLibQuoted, + }; + + if (!RunCommand ("Extract Debug Info", objcopy, args, stdoutLines, stderrLines)) { + LogFailure (); + return false; + } + + stdoutLines.Clear (); + stderrLines.Clear (); + args.Clear (); + args.Add ("--strip-debug"); + args.Add ("--strip-unneeded"); + args.Add (sourceLibQuoted); + + if (!RunCommand ("Strip Debug Info", objcopy, args, stdoutLines, stderrLines)) { + LogFailure (); + return false; + } + + stdoutLines.Clear (); + stderrLines.Clear (); + args.Clear (); + args.Add ($"--add-gnu-debuglink={destLibQuoted}"); + args.Add (sourceLibQuoted); + + if (!RunCommand ("Add Debug Info Link", objcopy, args, stdoutLines, stderrLines)) { + LogFailure (); + return false; + } + + return true; + + void LogFailure () + { + var sb = MonoAndroidHelper.MergeStdoutAndStderrMessages (stdoutLines, stderrLines); + // TODO: consider making it a warning + // TODO: make it a coded message + log.LogError ("Failed to extract debug info", Path.GetFileName (sourceLib), sb.ToString ()); + } + } + + bool RunLinker (List args, ITaskItem outputSharedLibrary) + { + var stdoutLines = new List (); + var stderrLines = new List (); + + if (!RunCommand ("Native Linker", ld, args, stdoutLines, stderrLines)) { + var sb = MonoAndroidHelper.MergeStdoutAndStderrMessages (stdoutLines, stderrLines); + log.LogCodedError ("XA3007", Properties.Resources.XA3007, Path.GetFileName (outputSharedLibrary.ItemSpec), sb.ToString ()); + return false; + } + + return true; + } + + bool RunCommand (string label, string binaryPath, List args, List stdoutLines, List stderrLines) + { + using var stdout_completed = new ManualResetEvent (false); + using var stderr_completed = new ManualResetEvent (false); + var psi = new ProcessStartInfo () { + FileName = binaryPath, + Arguments = String.Join (" ", args), + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + WindowStyle = ProcessWindowStyle.Hidden, + }; + + string binaryName = Path.GetFileName (ld); + log.LogDebugMessage ($"[{label}] {psi.FileName} {psi.Arguments}"); + + using var proc = new Process (); + proc.OutputDataReceived += (s, e) => { + if (e.Data != null) { + OnOutputData (binaryName, s, e); + stdoutLines.Add (e.Data); + } else { + stdout_completed.Set (); + } + }; + + proc.ErrorDataReceived += (s, e) => { + if (e.Data != null) { + OnErrorData (binaryName, s, e); + stderrLines.Add (e.Data); + } else { + stderr_completed.Set (); + } + }; + + proc.StartInfo = psi; + proc.Start (); + proc.BeginOutputReadLine (); + proc.BeginErrorReadLine (); + cancellationToken?.Register (() => { try { proc.Kill (); } catch (Exception) { } }); + proc.WaitForExit (); + + if (psi.RedirectStandardError) { + stderr_completed.WaitOne (TimeSpan.FromSeconds (30)); + } + + if (psi.RedirectStandardOutput) { + stdout_completed.WaitOne (TimeSpan.FromSeconds (30)); + } + + if (proc.ExitCode != 0) { + cancelTask?.Invoke (); + return false; + } + + return true; + } + + void OnOutputData (string linkerName, object sender, DataReceivedEventArgs e) + { + if (e.Data != null) { + log.LogMessage ($"[{linkerName} stdout] {e.Data}"); + } + } + + void OnErrorData (string linkerName, object sender, DataReceivedEventArgs e) + { + if (e.Data != null) { + log.LogMessage ($"[{linkerName} stderr] {e.Data}"); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs new file mode 100644 index 00000000000..b75e80d3977 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeRuntimeComponents.cs @@ -0,0 +1,184 @@ +using System; +using System.Collections.Generic; + +using Microsoft.Build.Framework; + +namespace Xamarin.Android.Tasks; + +class NativeRuntimeComponents +{ + internal class Archive + { + public readonly string Name; + public readonly string JniOnLoadName; + public bool Include => shouldInclude (this); + public readonly bool WholeArchive; + public bool DontExportSymbols { get; set; } + public HashSet? SymbolsToPreserve { get; set; } + + Func shouldInclude; + + public Archive (string name, Func? include = null, bool wholeArchive = false, string? jniOnLoadName = null) + { + Name = name; + shouldInclude = include == null ? ((Archive arch) => true) : include; + WholeArchive = wholeArchive; + JniOnLoadName = jniOnLoadName; + } + } + + internal class MonoComponentArchive : Archive + { + public readonly string ComponentName; + + public MonoComponentArchive (string name, string componentName, Func include) + : base (name, include) + { + ComponentName = componentName; + DontExportSymbols = true; + } + } + + sealed class ClangBuiltinsArchive : Archive + { + public ClangBuiltinsArchive (string clangAbi) + : base ($"libclang_rt.builtins-{clangAbi}-android.a") + {} + } + + class AndroidArchive : Archive + { + public AndroidArchive (string name) + : base (name, wholeArchive: false) + {} + } + + sealed class BclArchive : Archive + { + public BclArchive (string name, bool wholeArchive = false, string? jniOnLoadName = null) + : base (name, wholeArchive: wholeArchive, jniOnLoadName: jniOnLoadName) + { + DontExportSymbols = true; + } + } + + readonly ITaskItem[] monoComponents; + + public readonly List KnownArchives; + public readonly List NativeLibraries; + public readonly List LinkStartFiles; + public readonly List LinkEndFiles; + + public NativeRuntimeComponents (ITaskItem[] monoComponents) + { + this.monoComponents = monoComponents; + KnownArchives = new () { + // Mono components + new MonoComponentArchive ("libmono-component-debugger-static.a", "debugger", IncludeIfMonoComponentPresent), + new MonoComponentArchive ("libmono-component-debugger-stub-static.a", "debugger", IncludeIfMonoComponentAbsent), + new MonoComponentArchive ("libmono-component-diagnostics_tracing-static.a", "diagnostics_tracing", IncludeIfMonoComponentPresent), + new MonoComponentArchive ("libmono-component-diagnostics_tracing-stub-static.a", "diagnostics_tracing", IncludeIfMonoComponentAbsent), + new MonoComponentArchive ("libmono-component-hot_reload-static.a", "hot_reload", IncludeIfMonoComponentPresent), + new MonoComponentArchive ("libmono-component-hot_reload-stub-static.a", "hot_reload", IncludeIfMonoComponentAbsent), + new MonoComponentArchive ("libmono-component-marshal-ilgen-static.a", "marshal-ilgen", IncludeIfMonoComponentPresent), + new MonoComponentArchive ("libmono-component-marshal-ilgen-stub-static.a", "marshal-ilgen", IncludeIfMonoComponentAbsent), + + // MonoVM runtime + BCL + new Archive ("libmonosgen-2.0.a") { + DontExportSymbols = true, + }, + new BclArchive ("libSystem.Globalization.Native.a"), + new BclArchive ("libSystem.IO.Compression.Native.a"), + new BclArchive ("libSystem.Native.a"), + new BclArchive ("libSystem.Security.Cryptography.Native.Android.a", jniOnLoadName: "AndroidCryptoNative_InitLibraryOnLoad") { + SymbolsToPreserve = new (StringComparer.Ordinal) { + // This isn't referenced directly by any code in libSystem.Security.Cryptography.Native.Android. It is instead + // referenced by the Java code shipped with the component (`DotnetProxyTrustManager`), as a native Java method: + // + // static native boolean verifyRemoteCertificate(long sslStreamProxyHandle); + // + // Therefore we must reference it explicitly + "Java_net_dot_android_crypto_DotnetProxyTrustManager_verifyRemoteCertificate" + }, + + // For now, we have to export all the symbols from this archive because we need the above `Java_net*` symbol to be + // externally visible, and the linker's `--exclude-libs` flag works on the archive (.a) level. + // + // TODO: use `llvm-ar` to extract the relevant object file and link it separately? + DontExportSymbols = false, + }, + + // .NET for Android + new AndroidArchive ("libpinvoke-override-dynamic-release.a"), + new AndroidArchive ("libruntime-base-release.a"), + new AndroidArchive ("libxa-java-interop-release.a"), + new AndroidArchive ("libxa-lz4-release.a"), + new AndroidArchive ("libxa-shared-bits-release.a"), + new AndroidArchive ("libmono-android.release-static-release.a"), + + // LLVM clang built-ins archives + new ClangBuiltinsArchive ("aarch64"), + new ClangBuiltinsArchive ("arm"), + new ClangBuiltinsArchive ("i686"), + new ClangBuiltinsArchive ("x86_64"), + + // Remove once https://github.com/dotnet/runtime/pull/107615 is merged and released + new Archive ("libunwind.a") { + DontExportSymbols = true, + }, + }; + + // Just the base names of libraries to link into the unified runtime. Must have all the dependencies of all the static archives we + // link into the final library. + NativeLibraries = new () { + "c", + "dl", + "m", + "z", + "log", + + // Atomic is a static library in clang, need to investigate if it's really needed +// "atomic", + }; + + // Files that will be linked before any other object/archive/library files + LinkStartFiles = new () { + "crtbegin_so.o", + }; + + // Files that will be linked after any other object/archive/library files + LinkEndFiles = new () { + "crtend_so.o", + }; + } + + bool MonoComponentExists (Archive archive) + { + if (monoComponents.Length == 0) { + return false; + } + + var mcArchive = archive as MonoComponentArchive; + if (mcArchive == null) { + throw new ArgumentException (nameof (archive), "Must be an instance of MonoComponentArchive"); + } + + foreach (ITaskItem item in monoComponents) { + if (String.Compare (item.ItemSpec, mcArchive.ComponentName, StringComparison.OrdinalIgnoreCase) == 0) { + return true; + } + } + + return false; + } + + bool IncludeIfMonoComponentAbsent (Archive archive) + { + return !MonoComponentExists (archive); + } + + bool IncludeIfMonoComponentPresent (Archive archive) + { + return MonoComponentExists (archive); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/PinvokeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/PinvokeScanner.cs new file mode 100644 index 00000000000..bbce0e6acdb --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/PinvokeScanner.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Microsoft.Android.Build.Tasks; + +using Mono.Cecil; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class PinvokeScanner +{ + public sealed class PinvokeEntryInfo + { + public readonly string LibraryName; + public readonly string EntryName; + + public PinvokeEntryInfo (MethodDefinition method) + { + LibraryName = method.PInvokeInfo.Module.Name; + EntryName = method.PInvokeInfo.EntryPoint; + } + } + + readonly TaskLoggingHelper log; + + public PinvokeScanner (TaskLoggingHelper log) + { + this.log = log; + } + + public List Scan (AndroidTargetArch targetArch, XAAssemblyResolver resolver, ICollection frameworkAssemblies) + { + var pinvokes = new List (); + var pinvokeCache = new HashSet (StringComparer.Ordinal); + + foreach (ITaskItem fasm in frameworkAssemblies) { + string asmName = Path.GetFileNameWithoutExtension (fasm.ItemSpec); + AssemblyDefinition? asmdef = resolver.Resolve (asmName); + if (asmdef == null) { + log.LogWarning ($"Failed to resolve assembly '{fasm.ItemSpec}' for target architecture {targetArch}"); + continue; + } + Scan (targetArch, asmdef, pinvokeCache, pinvokes); + } + + return pinvokes; + } + + void Scan (AndroidTargetArch targetArch, AssemblyDefinition assembly, HashSet pinvokeCache, List pinvokes) + { + log.LogDebugMessage ($"[p/invoke][{targetArch}] Scanning assembly {assembly}"); + foreach (ModuleDefinition module in assembly.Modules) { + if (!module.HasTypes) { + continue; + } + + foreach (TypeDefinition type in module.Types) { + Scan (targetArch, type, pinvokeCache, pinvokes); + } + } + } + + void Scan (AndroidTargetArch targetArch, TypeDefinition type, HashSet pinvokeCache, List pinvokes) + { + if (type.HasNestedTypes) { + foreach (TypeDefinition nestedType in type.NestedTypes) { + Scan (targetArch, nestedType, pinvokeCache, pinvokes); + } + } + + if (!type.HasMethods) { + return; + } + + log.LogDebugMessage ($"[p/invoke][{targetArch}] Scanning type '{type}'"); + foreach (MethodDefinition method in type.Methods) { + if (!method.HasPInvokeInfo) { + continue; + } + + var pinfo = new PinvokeEntryInfo (method); + string key = $"{pinfo.LibraryName}/{pinfo.EntryName}"; + if (pinvokeCache.Contains (key)) { + continue; + } + + log.LogDebugMessage ($" [{targetArch}] p/invoke method: {pinfo.LibraryName}/{pinfo.EntryName}"); + pinvokeCache.Add (key); + pinvokes.Add (pinfo); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/PreservePinvokesNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/PreservePinvokesNativeAssemblyGenerator.cs new file mode 100644 index 00000000000..4517518ce72 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/PreservePinvokesNativeAssemblyGenerator.cs @@ -0,0 +1,378 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +using Xamarin.Android.Tasks.LLVMIR; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class PreservePinvokesNativeAssemblyGenerator : LlvmIrComposer +{ + sealed class PInvoke + { + public readonly LlvmIrFunction NativeFunction; + public readonly PinvokeScanner.PinvokeEntryInfo Info; + public readonly ulong Hash; + + public PInvoke (LlvmIrModule module, PinvokeScanner.PinvokeEntryInfo pinfo, bool is64Bit) + { + Info = pinfo; + Hash = MonoAndroidHelper.GetXxHash (pinfo.EntryName, is64Bit); + + // All the p/invoke functions use the same dummy signature. The only thing we care about is + // a way to reference to the symbol at build time so that we can return pointer to it. For + // that all we need is a known name, signature doesn't matter to us. + var funcSig = new LlvmIrFunctionSignature (name: pinfo.EntryName, returnType: typeof(void)); + NativeFunction = module.DeclareExternalFunction (funcSig); + } + } + + sealed class Component + { + public readonly string Name; + public readonly ulong NameHash; + public readonly List PInvokes; + public bool Is64Bit; + + public Component (string name, bool is64Bit) + { + Name = name; + NameHash = MonoAndroidHelper.GetXxHash (name, is64Bit); + PInvokes = new (); + Is64Bit = is64Bit; + } + + public void Add (LlvmIrModule module, PinvokeScanner.PinvokeEntryInfo pinfo) + { + PInvokes.Add (new PInvoke (module, pinfo, Is64Bit)); + } + + public void Sort () + { + PInvokes.Sort ((PInvoke a, PInvoke b) => a.Hash.CompareTo (b.Hash)); + } + } + + sealed class ConstructionState + { + public LlvmIrFunction Func; + public LlvmIrFunctionLabelItem ReturnLabel; + public LlvmIrFunctionParameter EntryPointHashParam; + public LlvmIrInstructions.Phi Phi; + public bool Is64Bit; + } + + // Maps a component name after ridding it of the `lib` prefix and the extension to a "canonical" + // name of a library, as used in `[DllImport]` attributes. + readonly Dictionary libraryNameMap = new (StringComparer.Ordinal) { + { "xa-java-interop", "java-interop" }, + { "mono-android.release-static", String.Empty }, + { "mono-android.release", String.Empty }, + }; + + readonly NativeCodeGenState state; + readonly ITaskItem[] monoComponents; + + public PreservePinvokesNativeAssemblyGenerator (TaskLoggingHelper log, NativeCodeGenState codeGenState, ITaskItem[] monoComponents) + : base (log) + { + if (codeGenState.PinvokeInfos == null) { + throw new InvalidOperationException ($"Internal error: {nameof (codeGenState)} `{nameof (codeGenState.PinvokeInfos)}` property is `null`"); + } + + this.state = codeGenState; + this.monoComponents = monoComponents; + } + + protected override void Construct (LlvmIrModule module) + { + Log.LogDebugMessage ($"[{state.TargetArch}] Constructing p/invoke preserve code"); + List pinvokeInfos = state.PinvokeInfos!; + if (pinvokeInfos.Count == 0) { + // This is a very unlikely scenario, but we will work just fine. The module that this generator produces will merely result + // in an empty (but valid) .ll file and an "empty" object file to link into the shared library. + return; + } + + Log.LogDebugMessage (" Looking for enabled native components"); + var componentNames = new HashSet (StringComparer.Ordinal); + var jniOnLoadNames = new HashSet (StringComparer.Ordinal); + var symbolsToExplicitlyPreserve = new HashSet (); + var nativeComponents = new NativeRuntimeComponents (monoComponents); + foreach (NativeRuntimeComponents.Archive archiveItem in nativeComponents.KnownArchives) { + if (!archiveItem.Include) { + continue; + } + + Log.LogDebugMessage ($" {archiveItem.Name}"); + componentNames.Add (archiveItem.Name); + if (!String.IsNullOrEmpty (archiveItem.JniOnLoadName)) { + jniOnLoadNames.Add (archiveItem.JniOnLoadName); + } + + if (archiveItem.SymbolsToPreserve == null || archiveItem.SymbolsToPreserve.Count == 0) { + continue; + } + + foreach (string symbolName in archiveItem.SymbolsToPreserve) { + symbolsToExplicitlyPreserve.Add (new LlvmIrGlobalVariableReference (symbolName)); + DeclareDummyFunction (module, symbolName); + } + } + + if (componentNames.Count == 0) { + Log.LogDebugMessage ("No native framework components are included in the build, not scanning for p/invoke usage"); + return; + } + + module.AddGlobalVariable ("__jni_on_load_handler_count", (uint)jniOnLoadNames.Count, LlvmIrVariableOptions.GlobalConstant); + var jniOnLoadPointers = new List (); + foreach (string name in jniOnLoadNames) { + jniOnLoadPointers.Add (new LlvmIrGlobalVariableReference (name)); + DeclareDummyFunction (module, name); + } + module.AddGlobalVariable ("__jni_on_load_handlers", jniOnLoadPointers, LlvmIrVariableOptions.GlobalConstant); + module.AddGlobalVariable ("__jni_on_load_handler_names", jniOnLoadNames, LlvmIrVariableOptions.GlobalConstant); + module.AddGlobalVariable ("__explicitly_preserved_symbols", symbolsToExplicitlyPreserve, LlvmIrVariableOptions.GlobalConstant); + + bool is64Bit = state.TargetArch switch { + AndroidTargetArch.Arm64 => true, + AndroidTargetArch.X86_64 => true, + AndroidTargetArch.Arm => false, + AndroidTargetArch.X86 => false, + _ => throw new NotSupportedException ($"Architecture {state.TargetArch} is not supported here") + }; + + Log.LogDebugMessage (" Checking discovered p/invokes against the list of components"); + var preservedPerComponent = new Dictionary (StringComparer.OrdinalIgnoreCase); + var processedCache = new HashSet (StringComparer.OrdinalIgnoreCase); + + foreach (PinvokeScanner.PinvokeEntryInfo pinfo in pinvokeInfos) { + Log.LogDebugMessage ($" p/invoke: {pinfo.EntryName} in {pinfo.LibraryName}"); + string key = $"{pinfo.LibraryName}/${pinfo.EntryName}"; + if (processedCache.Contains (key)) { + Log.LogDebugMessage ($" already processed"); + continue; + } + + processedCache.Add (key); + if (!MustPreserve (pinfo, componentNames)) { + Log.LogDebugMessage (" no need to preserve"); + continue; + } + Log.LogDebugMessage (" must be preserved"); + + if (!preservedPerComponent.TryGetValue (pinfo.LibraryName, out Component? component)) { + component = new Component (pinfo.LibraryName, is64Bit); + preservedPerComponent.Add (component.Name, component); + } + component.Add (module, pinfo); + } + + var components = new List (preservedPerComponent.Values); + if (is64Bit) { + AddFindPinvoke (module, components, is64Bit); + } else { + AddFindPinvoke (module, components, is64Bit); + } + } + + void AddFindPinvoke (LlvmIrModule module, List components, bool is64Bit) where T: struct + { + var hashType = is64Bit ? typeof (ulong) : typeof (uint); + var parameters = new List { + new LlvmIrFunctionParameter (hashType, "library_name_hash") { + NoUndef = true, + }, + + new LlvmIrFunctionParameter (hashType, "entrypoint_hash") { + NoUndef = true, + }, + + new LlvmIrFunctionParameter (typeof(IntPtr), "known_library") { + Align = 1, // it's a reference to C++ `bool` + Dereferenceable = 1, + IsCplusPlusReference = true, + NoCapture = true, + NonNull = true, + NoUndef = true, + WriteOnly = true, + }, + }; + + var sig = new LlvmIrFunctionSignature ( + name: "find_pinvoke", + returnType: typeof(IntPtr), + parameters: parameters, + new LlvmIrFunctionSignature.ReturnTypeAttributes { + NoUndef = true, + } + ); + + var func = new LlvmIrFunction (sig, MakeFindPinvokeAttributeSet (module)) { + CallingConvention = LlvmIrCallingConvention.Fastcc, + Visibility = LlvmIrVisibility.Hidden, + }; + LlvmIrLocalVariable retval = func.CreateLocalVariable (typeof(IntPtr), "retval"); + var state = new ConstructionState { + Func = func, + ReturnLabel = new LlvmIrFunctionLabelItem ("return"), + EntryPointHashParam = parameters[1], + Phi = new LlvmIrInstructions.Phi (retval), + Is64Bit = is64Bit, + }; + module.Add (state.Func); + state.Func.Body.Add (new LlvmIrFunctionLabelItem ("entry")); + + var libraryNameSwitchEpilog = new LlvmIrFunctionLabelItem ("libNameSW.epilog"); + var componentSwitch = new LlvmIrInstructions.Switch (parameters[0], libraryNameSwitchEpilog, "sw.libname"); + + state.Func.Body.Add (componentSwitch); + state.Phi.AddNode (libraryNameSwitchEpilog, null); + + components.Sort ((Component a, Component b) => a.NameHash.CompareTo (b.NameHash)); + Log.LogDebugMessage (" Components to be preserved:"); + uint componentID = 1; + + foreach (Component component in components) { + Log.LogDebugMessage ($" {component.Name} (hash: 0x{component.NameHash:x}; {component.PInvokes.Count} p/invoke(s))"); + + string comment = $" {component.Name} (p/invoke count: {component.PInvokes.Count})"; + LlvmIrFunctionLabelItem componentLabel = AddSwitchItem (componentSwitch, component.NameHash, is64Bit, comment, null); + + func.Body.Add (componentLabel, comment); + AddPInvokeSwitch (state, componentLabel, component, componentID++); + } + + func.Body.Add (libraryNameSwitchEpilog); + + var setKnownLib = new LlvmIrInstructions.Store (false, parameters[2]); + func.Body.Add (setKnownLib); + AddReturnBranch (func, state.ReturnLabel); + + func.Body.Add (state.ReturnLabel); + func.Body.Add (state.Phi); + func.Body.Add (new LlvmIrInstructions.Ret (typeof (IntPtr), retval)); + } + + void AddPInvokeSwitch (ConstructionState state, LlvmIrFunctionLabelItem componentLabel, Component component, uint id) where T: struct + { + var pinvokeSwitchEpilog = new LlvmIrFunctionLabelItem ($"pinvokeSW.epilog.{id}"); + state.Phi.AddNode (pinvokeSwitchEpilog, null); + + var pinvokeSwitch = new LlvmIrInstructions.Switch (state.EntryPointHashParam, pinvokeSwitchEpilog, $"sw.pinvoke.{id}"); + state.Func.Body.Add (pinvokeSwitch); + + component.Sort (); + bool first = true; + foreach (PInvoke pi in component.PInvokes) { + string pinvokeName = pi.NativeFunction.Signature.Name; + string comment = $" {pinvokeName}"; + LlvmIrFunctionLabelItem pinvokeLabel = AddSwitchItem (pinvokeSwitch, pi.Hash, state.Is64Bit, comment, first ? state.ReturnLabel : null); + + // First item of every component switch block "reuses" the block's label + if (first) { + first = false; + } else { + state.Func.Body.Add (pinvokeLabel, comment); + AddReturnBranch (state.Func, state.ReturnLabel); + } + + state.Phi.AddNode (pinvokeLabel == state.ReturnLabel ? componentLabel : pinvokeLabel, new LlvmIrGlobalVariableReference (pinvokeName)); + } + + state.Func.Body.Add (pinvokeSwitchEpilog); + AddReturnBranch (state.Func, state.ReturnLabel); + } + + void AddReturnBranch (LlvmIrFunction func, LlvmIrFunctionLabelItem returnLabel) + { + var branch = new LlvmIrInstructions.Br (returnLabel); + func.Body.Add (branch); + } + + LlvmIrFunctionLabelItem AddSwitchItem (LlvmIrInstructions.Switch sw, ulong hash, bool is64Bit, string? comment, LlvmIrFunctionLabelItem? label) where T: struct + { + if (is64Bit) { + return sw.Add ((T)(object)hash, dest: label, comment: comment); + } + return sw.Add ((T)(object)(uint)hash, dest: label, comment: comment); + } + + LlvmIrFunctionAttributeSet MakeFindPinvokeAttributeSet (LlvmIrModule module) + { + var attrSet = new LlvmIrFunctionAttributeSet { + new MustprogressFunctionAttribute (), + new NofreeFunctionAttribute (), + new NorecurseFunctionAttribute (), + new NosyncFunctionAttribute (), + new NounwindFunctionAttribute (), + new WillreturnFunctionAttribute (), + new MemoryFunctionAttribute { + Default = MemoryAttributeAccessKind.Write, + Argmem = MemoryAttributeAccessKind.None, + InaccessibleMem = MemoryAttributeAccessKind.None, + }, + new UwtableFunctionAttribute (), + new NoTrappingMathFunctionAttribute (true), + }; + + return module.AddAttributeSet (attrSet); + } + + // Returns `true` for all p/invokes that we know are part of our set of components, otherwise returns `false`. + // Returning `false` merely means that the p/invoke isn't in any of BCL or our code and therefore we shouldn't + // care. It doesn't mean the p/invoke will be removed in any way. + bool MustPreserve (PinvokeScanner.PinvokeEntryInfo pinfo, ICollection components) + { + if (String.Compare ("xa-internal-api", pinfo.LibraryName, StringComparison.Ordinal) == 0) { + return true; + } + + foreach (string component in components) { + // The most common pattern for the BCL - file name without extension + string componentName = Path.GetFileNameWithoutExtension (component); + if (Matches (pinfo.LibraryName, componentName)) { + return true; + } + + // If it starts with `lib`, drop the prefix + if (componentName.StartsWith ("lib", StringComparison.Ordinal)) { + if (Matches (pinfo.LibraryName, componentName.Substring (3))) { + return true; + } + } + + // Might require mapping of component name to a canonical one + if (libraryNameMap.TryGetValue (componentName, out string? mappedComponentName) && !String.IsNullOrEmpty (mappedComponentName)) { + if (Matches (pinfo.LibraryName, mappedComponentName)) { + return true; + } + } + + // Try full file name, as the last resort + if (Matches (pinfo.LibraryName, Path.GetFileName (component))) { + return true; + } + } + + return false; + + bool Matches (string libraryName, string componentName) + { + return String.Compare (libraryName, componentName, StringComparison.Ordinal) == 0; + } + } + + static void DeclareDummyFunction (LlvmIrModule module, string name) + { + // Just a dummy declaration, we don't care about the arguments + var funcSig = new LlvmIrFunctionSignature (name, returnType: typeof(void)); + var _ = module.DeclareExternalFunction (funcSig); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index e4deccf9ab0..f4534c164be 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -327,6 +327,9 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. False <_AndroidUseMarshalMethods Condition=" '$(AndroidIncludeDebugSymbols)' == 'True' ">False <_AndroidUseMarshalMethods Condition=" '$(AndroidIncludeDebugSymbols)' != 'True' ">$(AndroidEnableMarshalMethods) + + + <_AndroidEnableNativeRuntimeLinking Condition=" '$(_AndroidEnableNativeRuntimeLinking)' == '' and '$(AndroidIncludeDebugSymbols)' != 'true' ">true @@ -1519,7 +1522,8 @@ because xbuild doesn't support framework reference assemblies. LinkingEnabled="$(_LinkingEnabled)" HaveMultipleRIDs="$(_HaveMultipleRIDs)" IntermediateOutputDirectory="$(IntermediateOutputPath)" - Environments="@(_EnvironmentFiles)"> + Environments="@(_EnvironmentFiles)" + EnableNativeRuntimeLinking="$(_AndroidEnableNativeRuntimeLinking)"> @@ -1614,6 +1618,14 @@ because xbuild doesn't support framework reference assemblies. Mode="marshal_methods"> + + + @@ -1746,11 +1758,14 @@ because xbuild doesn't support framework reference assemblies. UseAssemblyStore="$(AndroidUseAssemblyStore)" EnableMarshalMethods="$(_AndroidUseMarshalMethods)" CustomBundleConfigFile="$(AndroidBundleConfigurationFile)" + EnableNativeRuntimeLinking="$(_AndroidEnableNativeRuntimeLinking)" > + + @@ -1865,6 +1880,7 @@ because xbuild doesn't support framework reference assemblies. <_CompileToDalvikDependsOnTargets> _CompileJava; _CreateApplicationSharedLibraries; + _LinkNativeRuntime; _GetMonoPlatformJarPath; _GetLibraryImports; _SetProguardMappingFileProperty; @@ -1976,6 +1992,9 @@ because xbuild doesn't support framework reference assemblies. <_NativeAssemblyTarget Include="@(_AndroidRemapAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> %(_AndroidRemapAssemblySource.abi) + <_NativeAssemblyTarget Include="@(_RuntimeLinkingAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + %(_RuntimeLinkingAssemblySource.abi) + @@ -1995,10 +2014,10 @@ because xbuild doesn't support framework reference assemblies. + Outputs="@(_ApplicationSharedLibrary)" + Condition=" '$(_AndroidEnableNativeRuntimeLinking)' != 'true' "> GetEnv ((void**)&env, JNI_VERSION_1_6); osBridge.initialize_on_onload (vm, env); +#if defined(DYNAMIC_RUNTIME_LINKING) + PinvokeOverride::handle_jni_on_load (vm, reserved); +#endif return JNI_VERSION_1_6; } diff --git a/src/native/monodroid/monodroid-networkinfo.cc b/src/native/monodroid/monodroid-networkinfo.cc index 50267299def..6a328db218f 100644 --- a/src/native/monodroid/monodroid-networkinfo.cc +++ b/src/native/monodroid/monodroid-networkinfo.cc @@ -127,19 +127,19 @@ _monodroid_get_network_interface_state (const char *ifname, mono_bool *is_up, mo return ret; } -mono_bool +extern "C" mono_bool _monodroid_get_network_interface_up_state (const char *ifname, mono_bool *is_up) { return _monodroid_get_network_interface_state (ifname, is_up, nullptr); } -mono_bool +extern "C" mono_bool _monodroid_get_network_interface_supports_multicast (const char *ifname, mono_bool *supports_multicast) { return _monodroid_get_network_interface_state (ifname, nullptr, supports_multicast); } -int +extern "C" int _monodroid_get_dns_servers (void **dns_servers_array) { if (!dns_servers_array) { diff --git a/src/native/pinvoke-override/dynamic.cc b/src/native/pinvoke-override/dynamic.cc index 447606650f5..0c2219b70da 100644 --- a/src/native/pinvoke-override/dynamic.cc +++ b/src/native/pinvoke-override/dynamic.cc @@ -1,2 +1,97 @@ +#include +#include + #define PINVOKE_OVERRIDE_INLINE [[gnu::noinline]] -#include "pinvoke-override-api.hh" +#include "pinvoke-override-api-impl.hh" + +using namespace xamarin::android; +using namespace xamarin::android::internal; + +// +// This is generated during application build (see obj/${CONFIGURATION}/${RID}/android/pinvoke_preserve.*.ll) +// +using JniOnLoadHandler = jint (*) (JavaVM *vm, void *reserved); + +extern "C" { + void* find_pinvoke (hash_t library_name_hash, hash_t entrypoint_hash, bool &known_library); + + extern const uint32_t __jni_on_load_handler_count; + extern const JniOnLoadHandler __jni_on_load_handlers[]; + extern const char* __jni_on_load_handler_names[]; + extern const void* __explicitly_preserved_symbols[]; +} + +[[gnu::flatten]] +void* +PinvokeOverride::monodroid_pinvoke_override (const char *library_name, const char *entrypoint_name) noexcept +{ + log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); + log_debug (LOG_ASSEMBLY, "library_name == '%s'; entrypoint_name == '%s'", library_name, entrypoint_name); + + if (library_name == nullptr || entrypoint_name == nullptr) [[unlikely]] { + Helpers::abort_application ( + LOG_ASSEMBLY, + Util::monodroid_strdup_printf ( + "Both library name ('%s') and entry point name ('%s') must be specified", + library_name, + entrypoint_name + ) + ); + } + + hash_t library_name_hash = xxhash::hash (library_name, strlen (library_name)); + hash_t entrypoint_hash = xxhash::hash (entrypoint_name, strlen (entrypoint_name)); + log_debug (LOG_ASSEMBLY, "library_name_hash == 0x%zx; entrypoint_hash == 0x%zx", library_name_hash, entrypoint_hash); + + bool known_library = true; + void *pinvoke_ptr = find_pinvoke (library_name_hash, entrypoint_hash, known_library); + if (pinvoke_ptr != nullptr) [[likely]] { + log_debug (LOG_ASSEMBLY, "pinvoke_ptr == %p", pinvoke_ptr); + return pinvoke_ptr; + } + + if (known_library) [[unlikely]] { + log_debug (LOG_ASSEMBLY, "Lookup in a known library == internal"); + // Should "never" happen. It seems we have a known library hash (of one that's linked into the dynamically + // built DSO) but an unknown symbol hash. The symbol **probably** doesn't exist (was most likely linked out if + // the find* functions didn't know its hash), but we cannot be sure of that so we'll try to load it. + pinvoke_ptr = dlsym (RTLD_DEFAULT, entrypoint_name); + if (pinvoke_ptr == nullptr) { + Helpers::abort_application ( + LOG_ASSEMBLY, + Util::monodroid_strdup_printf ( + "Unable to load p/invoke entry '%s/%s' from the unified runtime DSO", + library_name, + entrypoint_name + ) + ); + } + + return pinvoke_ptr; + } + + log_debug (LOG_ASSEMBLY, "p/invoke not from a known library, slow path taken."); + pinvoke_ptr = handle_other_pinvoke_request (library_name, library_name_hash, entrypoint_name, entrypoint_hash);; + log_debug (LOG_ASSEMBLY, "foreign library pinvoke_ptr == %p", pinvoke_ptr); + return pinvoke_ptr; +} + +void PinvokeOverride::handle_jni_on_load (JavaVM *vm, void *reserved) noexcept +{ + if (__jni_on_load_handler_count == 0) { + return; + } + + for (uint32_t i = 0; i < __jni_on_load_handler_count; i++) { + __jni_on_load_handlers[i] (vm, reserved); + } + + // This is just to reference the generated array, all we need from it is to be there + // TODO: see if there's an attribute we can use to make the linker keep the symbol instead. + // void *first_ptr = __explicitly_preserved_symbols; + // if (first_ptr == nullptr) { + // // This will never actually be logged, since by the time this function is called we haven't initialized + // // logging categories yet. It's here just to have some code in the if statement body. + // log_debug (LOG_ASSEMBLY, "No explicitly preserved symbols"); + // } +} diff --git a/src/native/pinvoke-override/pinvoke-override-api-impl.hh b/src/native/pinvoke-override/pinvoke-override-api-impl.hh index 36523f092a9..c48f2647c56 100644 --- a/src/native/pinvoke-override/pinvoke-override-api-impl.hh +++ b/src/native/pinvoke-override/pinvoke-override-api-impl.hh @@ -73,11 +73,11 @@ namespace xamarin::android { bool already_loaded = !__atomic_compare_exchange ( /* ptr */ &entry.func, - /* expected */ &expected_null, - /* desired */ &entry_handle, - /* weak */ false, - /* success_memorder */ __ATOMIC_ACQUIRE, - /* failure_memorder */ __ATOMIC_RELAXED + /* expected */ &expected_null, + /* desired */ &entry_handle, + /* weak */ false, + /* success_memorder */ __ATOMIC_ACQUIRE, + /* failure_memorder */ __ATOMIC_RELAXED ); if (already_loaded) { diff --git a/src/native/pinvoke-override/pinvoke-override-api.hh b/src/native/pinvoke-override/pinvoke-override-api.hh index 445f8cccf0b..5ac5b449a8e 100644 --- a/src/native/pinvoke-override/pinvoke-override-api.hh +++ b/src/native/pinvoke-override/pinvoke-override-api.hh @@ -2,6 +2,8 @@ #include +#include + #include "cppcompat.hh" #include "xxhash.hh" @@ -75,7 +77,8 @@ namespace xamarin::android { static void* fetch_or_create_pinvoke_map_entry (std::string const& library_name, std::string const& entrypoint_name, hash_t entrypoint_name_hash, pinvoke_api_map_ptr api_map, bool need_lock) noexcept; static PinvokeEntry* find_pinvoke_address (hash_t hash, const PinvokeEntry *entries, size_t entry_count) noexcept; static void* handle_other_pinvoke_request (const char *library_name, hash_t library_name_hash, const char *entrypoint_name, hash_t entrypoint_name_hash) noexcept; - static void* monodroid_pinvoke_override (const char *library_name, const char *entrypoint_name); + static void* monodroid_pinvoke_override (const char *library_name, const char *entrypoint_name) noexcept; + static void handle_jni_on_load (JavaVM *vm, void *reserved) noexcept; private: static xamarin::android::mutex pinvoke_map_write_lock; diff --git a/src/native/pinvoke-override/precompiled.cc b/src/native/pinvoke-override/precompiled.cc index 1c84095080a..0b57f105169 100644 --- a/src/native/pinvoke-override/precompiled.cc +++ b/src/native/pinvoke-override/precompiled.cc @@ -9,7 +9,7 @@ using namespace xamarin::android; [[gnu::flatten]] void* -PinvokeOverride::monodroid_pinvoke_override (const char *library_name, const char *entrypoint_name) +PinvokeOverride::monodroid_pinvoke_override (const char *library_name, const char *entrypoint_name) noexcept { if (library_name == nullptr || entrypoint_name == nullptr) { return nullptr; diff --git a/src/native/runtime-base/cpu-arch.hh b/src/native/runtime-base/cpu-arch.hh index de0e9cf93d5..db9cd6bb43f 100644 --- a/src/native/runtime-base/cpu-arch.hh +++ b/src/native/runtime-base/cpu-arch.hh @@ -8,5 +8,5 @@ #define CPU_KIND_X86 ((unsigned short)4) #define CPU_KIND_X86_64 ((unsigned short)5) -void _monodroid_detect_cpu_and_architecture (unsigned short *built_for_cpu, unsigned short *running_on_cpu, unsigned char *is64bit); +extern "C" void _monodroid_detect_cpu_and_architecture (unsigned short *built_for_cpu, unsigned short *running_on_cpu, unsigned char *is64bit); #endif // ndef NET diff --git a/src/native/runtime-base/internal-pinvokes.hh b/src/native/runtime-base/internal-pinvokes.hh index b73e390cccf..b9175ee4ee3 100644 --- a/src/native/runtime-base/internal-pinvokes.hh +++ b/src/native/runtime-base/internal-pinvokes.hh @@ -12,51 +12,52 @@ #include "xamarin-app.hh" #include "xamarin_getifaddrs.h" -int _monodroid_getifaddrs (struct _monodroid_ifaddrs **ifap); -void _monodroid_freeifaddrs (struct _monodroid_ifaddrs *ifa); - -mono_bool _monodroid_get_network_interface_up_state (const char *ifname, mono_bool *is_up); -mono_bool _monodroid_get_network_interface_supports_multicast (const char *ifname, mono_bool *supports_multicast); -int _monodroid_get_dns_servers (void **dns_servers_array); - -unsigned int monodroid_get_log_categories (); -int monodroid_get_system_property (const char *name, char **value); -int monodroid_embedded_assemblies_set_assemblies_prefix (const char *prefix); -void monodroid_log (xamarin::android::LogLevel level, LogCategories category, const char *message); -void monodroid_free (void *ptr); -int _monodroid_max_gref_get (); -int _monodroid_gref_get (); -void _monodroid_gref_log (const char *message); -int _monodroid_gref_log_new (jobject curHandle, char curType, jobject newHandle, char newType, const char *threadName, int threadId, const char *from, int from_writable); -void _monodroid_gref_log_delete (jobject handle, char type, const char *threadName, int threadId, const char *from, int from_writable); -int _monodroid_weak_gref_get (); -void _monodroid_weak_gref_new (jobject curHandle, char curType, jobject newHandle, char newType, const char *threadName, int threadId, const char *from, int from_writable); -void _monodroid_weak_gref_delete (jobject handle, char type, const char *threadName, int threadId, const char *from, int from_writable); -void _monodroid_lref_log_new (int lrefc, jobject handle, char type, const char *threadName, int threadId, const char *from, int from_writable); -void _monodroid_lref_log_delete (int lrefc, jobject handle, char type, const char *threadName, int threadId, const char *from, int from_writable); -void _monodroid_gc_wait_for_bridge_processing (); -void monodroid_clear_gdb_wait (); -void* _monodroid_get_identity_hash_code (JNIEnv *env, void *v); -void* _monodroid_timezone_get_default_id (); -void _monodroid_counters_dump ([[maybe_unused]] const char *format, [[maybe_unused]] va_list args); -xamarin::android::managed_timing_sequence* monodroid_timing_start (const char *message); -void monodroid_timing_stop (xamarin::android::managed_timing_sequence *sequence, const char *message); -char** monodroid_strsplit (const char *str, const char *delimiter, size_t max_tokens); -void monodroid_strfreev (char **str_array); -char* monodroid_strdup_printf (const char *format, ...); -char* monodroid_TypeManager_get_java_class_name (jclass klass); -int monodroid_get_namespaced_system_property (const char *name, char **value); -FILE* monodroid_fopen (const char* filename, const char* mode); -int send_uninterrupted (int fd, void *buf, int len); -int recv_uninterrupted (int fd, void *buf, int len); -void set_world_accessable (const char *path); -void create_public_directory (const char *dir); -char* path_combine (const char *path1, const char *path2); -void* monodroid_dylib_mono_new ([[maybe_unused]] const char *libmono_path); -void monodroid_dylib_mono_free ([[maybe_unused]] void *mono_imports); -int monodroid_dylib_mono_init (void *mono_imports, [[maybe_unused]] const char *libmono_path); -void* monodroid_get_dylib (); -const char* _monodroid_lookup_replacement_type (const char *jniSimpleReference); -const JniRemappingReplacementMethod* _monodroid_lookup_replacement_method_info (const char *jniSourceType, const char *jniMethodName, const char *jniMethodSignature); -void monodroid_log_traces (uint32_t kind, const char *first_line); -void _monodroid_detect_cpu_and_architecture (unsigned short *built_for_cpu, unsigned short *running_on_cpu, unsigned char *is64bit); +extern "C" { + int _monodroid_getifaddrs (struct _monodroid_ifaddrs **ifap); + void _monodroid_freeifaddrs (struct _monodroid_ifaddrs *ifa); + mono_bool _monodroid_get_network_interface_up_state (const char *ifname, mono_bool *is_up); + mono_bool _monodroid_get_network_interface_supports_multicast (const char *ifname, mono_bool *supports_multicast); + int _monodroid_get_dns_servers (void **dns_servers_array); + unsigned int monodroid_get_log_categories (); + int monodroid_get_system_property (const char *name, char **value); + int monodroid_embedded_assemblies_set_assemblies_prefix (const char *prefix); + void monodroid_log (xamarin::android::LogLevel level, LogCategories category, const char *message); + void monodroid_free (void *ptr); + int _monodroid_max_gref_get (); + int _monodroid_gref_get (); + void _monodroid_gref_log (const char *message); + int _monodroid_gref_log_new (jobject curHandle, char curType, jobject newHandle, char newType, const char *threadName, int threadId, const char *from, int from_writable); + void _monodroid_gref_log_delete (jobject handle, char type, const char *threadName, int threadId, const char *from, int from_writable); + int _monodroid_weak_gref_get (); + void _monodroid_weak_gref_new (jobject curHandle, char curType, jobject newHandle, char newType, const char *threadName, int threadId, const char *from, int from_writable); + void _monodroid_weak_gref_delete (jobject handle, char type, const char *threadName, int threadId, const char *from, int from_writable); + void _monodroid_lref_log_new (int lrefc, jobject handle, char type, const char *threadName, int threadId, const char *from, int from_writable); + void _monodroid_lref_log_delete (int lrefc, jobject handle, char type, const char *threadName, int threadId, const char *from, int from_writable); + void _monodroid_gc_wait_for_bridge_processing (); + int _monodroid_get_android_api_level (); + void monodroid_clear_gdb_wait (); + void* _monodroid_get_identity_hash_code (JNIEnv *env, void *v); + void* _monodroid_timezone_get_default_id (); + void _monodroid_counters_dump ([[maybe_unused]] const char *format, [[maybe_unused]] va_list args); + xamarin::android::managed_timing_sequence* monodroid_timing_start (const char *message); + void monodroid_timing_stop (xamarin::android::managed_timing_sequence *sequence, const char *message); + char** monodroid_strsplit (const char *str, const char *delimiter, size_t max_tokens); + void monodroid_strfreev (char **str_array); + char* monodroid_strdup_printf (const char *format, ...); + char* monodroid_TypeManager_get_java_class_name (jclass klass); + int monodroid_get_namespaced_system_property (const char *name, char **value); + FILE* monodroid_fopen (const char* filename, const char* mode); + int send_uninterrupted (int fd, void *buf, int len); + int recv_uninterrupted (int fd, void *buf, int len); + void set_world_accessable (const char *path); + void create_public_directory (const char *dir); + char* path_combine (const char *path1, const char *path2); + void* monodroid_dylib_mono_new ([[maybe_unused]] const char *libmono_path); + void monodroid_dylib_mono_free ([[maybe_unused]] void *mono_imports); + int monodroid_dylib_mono_init (void *mono_imports, [[maybe_unused]] const char *libmono_path); + void* monodroid_get_dylib (); + const char* _monodroid_lookup_replacement_type (const char *jniSimpleReference); + const JniRemappingReplacementMethod* _monodroid_lookup_replacement_method_info (const char *jniSourceType, const char *jniMethodName, const char *jniMethodSignature); + void monodroid_log_traces (uint32_t kind, const char *first_line); + void _monodroid_detect_cpu_and_architecture (unsigned short *built_for_cpu, unsigned short *running_on_cpu, unsigned char *is64bit); +} diff --git a/tests/MSBuildDeviceIntegration/Tests/AotProfileTests.cs b/tests/MSBuildDeviceIntegration/Tests/AotProfileTests.cs index bbc15d40d64..a0451b4bd71 100644 --- a/tests/MSBuildDeviceIntegration/Tests/AotProfileTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/AotProfileTests.cs @@ -33,6 +33,12 @@ public void BuildBasicApplicationAndAotProfileIt () }; proj.SetAndroidSupportedAbis (DeviceAbi); + // Currently, AOT profiling won't work with dynamic runtime linking because the `libmono-profiler-aot.so` library from + // the `mono.aotprofiler.android` package depends on `libmonosgen-2.0.so` which, when dynamic linking is enabled, simply + // doesn't exist (it's linked statically into `libmonodroid.so`). AOT profiler would have to have a static library in + // the nuget in order for us to support profiling in this mode. + proj.SetProperty ("_AndroidEnableNativeRuntimeLinking", "False"); + // TODO: only needed in .NET 6+ // See https://github.com/dotnet/runtime/issues/56989 proj.PackageReferences.Add (KnownPackages.Mono_AotProfiler_Android); diff --git a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs index 6c61f9fbe70..246f1a0ccd9 100644 --- a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs @@ -15,8 +15,23 @@ namespace Xamarin.Android.Build.Tests public class BundleToolTests : DeviceTest { static readonly object[] FixtureArgs = { - new object[] { false }, - new object[] { true }, + new object[] { + false, // useAssemblyBlobs + false, // useNativeRuntimeLinkingMode + }, + + new object[] { + true, // useAssemblyBlobs + false, // useNativeRuntimeLinkingMode + }, + + new object[] { + true, // useAssemblyBlobs + true, // useNativeRuntimeLinkingMode + }, + + // There's no point in testing further combinations of the two parameters, the tests + // wouldn't actually differ. }; static readonly string [] Abis = new [] { "armeabi-v7a", "arm64-v8a", "x86", "x86_64" }; @@ -26,6 +41,7 @@ public class BundleToolTests : DeviceTest string intermediate; string bin; bool usesAssemblyBlobs; + bool useNativeRuntimeLinkingMode; // Disable split by language const string BuildConfig = @"{ @@ -46,9 +62,10 @@ public class BundleToolTests : DeviceTest } }"; - public BundleToolTests (bool usesAssemblyBlobs) + public BundleToolTests (bool usesAssemblyBlobs, bool useNativeRuntimeLinkingMode) { this.usesAssemblyBlobs = usesAssemblyBlobs; + this.useNativeRuntimeLinkingMode = useNativeRuntimeLinkingMode; } [OneTimeSetUp] @@ -97,6 +114,7 @@ public void OneTimeSetUp () app.SetAndroidSupportedAbis (Abis); app.SetProperty ("AndroidBundleConfigurationFile", "buildConfig.json"); app.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); + app.SetProperty ("_AndroidEnableNativeRuntimeLinking", useNativeRuntimeLinkingMode.ToString ()); libBuilder = CreateDllBuilder (Path.Combine (path, lib.ProjectName), cleanupOnDispose: true); Assert.IsTrue (libBuilder.Build (lib), "Library build should have succeeded."); @@ -177,15 +195,19 @@ public void BaseZip () } expectedFiles.Add ($"lib/{abi}/libmonodroid.so"); - expectedFiles.Add ($"lib/{abi}/libmonosgen-2.0.so"); - expectedFiles.Add ($"lib/{abi}/libxamarin-app.so"); + if (!useNativeRuntimeLinkingMode) { + // None of these exist if dynamic native runtime linking is enabled + expectedFiles.Add ($"lib/{abi}/libmonosgen-2.0.so"); + expectedFiles.Add ($"lib/{abi}/libxamarin-app.so"); + expectedFiles.Add ($"lib/{abi}/libSystem.IO.Compression.Native.so"); + expectedFiles.Add ($"lib/{abi}/libSystem.Native.so"); + } + if (usesAssemblyBlobs) { expectedFiles.Add ($"{blobEntryPrefix}{abi}/lib_System.Private.CoreLib.dll.so"); } else { expectedFiles.Add ($"lib/{abi}/lib_System.Private.CoreLib.dll.so"); } - expectedFiles.Add ($"lib/{abi}/libSystem.IO.Compression.Native.so"); - expectedFiles.Add ($"lib/{abi}/libSystem.Native.so"); } foreach (var expected in expectedFiles) { CollectionAssert.Contains (contents, expected, $"`{baseZip}` did not contain `{expected}`"); @@ -237,15 +259,20 @@ public void AppBundle () } expectedFiles.Add ($"base/lib/{abi}/libmonodroid.so"); - expectedFiles.Add ($"base/lib/{abi}/libmonosgen-2.0.so"); - expectedFiles.Add ($"base/lib/{abi}/libxamarin-app.so"); + + if (!useNativeRuntimeLinkingMode) { + // None of these exist if dynamic native runtime linking is enabled + expectedFiles.Add ($"base/lib/{abi}/libmonosgen-2.0.so"); + expectedFiles.Add ($"base/lib/{abi}/libxamarin-app.so"); + expectedFiles.Add ($"base/lib/{abi}/libSystem.IO.Compression.Native.so"); + expectedFiles.Add ($"base/lib/{abi}/libSystem.Native.so"); + } + if (usesAssemblyBlobs) { expectedFiles.Add ($"{blobEntryPrefix}{abi}/lib_System.Private.CoreLib.dll.so"); } else { expectedFiles.Add ($"base/lib/{abi}/lib_System.Private.CoreLib.dll.so"); } - expectedFiles.Add ($"base/lib/{abi}/libSystem.IO.Compression.Native.so"); - expectedFiles.Add ($"base/lib/{abi}/libSystem.Native.so"); } foreach (var expected in expectedFiles) { CollectionAssert.Contains (contents, expected, $"`{aab}` did not contain `{expected}`");