Skip to content

Use Android shared library loader for JNI libraries #10376

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 16 commits into
base: main
Choose a base branch
from
Draft

Use Android shared library loader for JNI libraries #10376

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
97f7f46
Managed side
grendello Aug 5, 2025
e8337e3
Native side WIP
grendello Aug 6, 2025
b8cfc69
System.loadLibrary works now
grendello Aug 7, 2025
cbdb7c4
Odds and ends
grendello Aug 7, 2025
eb3197b
[WIP] MonoVM side
grendello Aug 11, 2025
e1c4164
Fix JNIEnv* usage in multiple threads
grendello Aug 11, 2025
b38405c
This shouldn't be needed anymore
grendello Aug 11, 2025
c978c3a
Handle Java exceptions
grendello Aug 11, 2025
fea366b
Stuff
grendello Aug 11, 2025
f689583
Let's try SSL tests on CoreCLR
grendello Aug 12, 2025
d818251
[WIP] workaround for System.loadLibrary not working on some threads
grendello Aug 12, 2025
8cba400
Lots of code, Android doesn't cooperate...
grendello Aug 13, 2025
2a7fc2e
Preparation for JNI DSO preloading in the runtime. TBC tomorrow.
grendello Aug 13, 2025
b5b6291
Managed part of JNI DSO preloading done
grendello Aug 14, 2025
8355471
Now it's done :P
grendello Aug 14, 2025
f072092
Works locally, let's see how it fares on CI
grendello Aug 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 73 additions & 12 deletions src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ sealed class DSOCacheEntry
[NativeAssembler (NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)]
public ulong real_name_hash;
public bool ignore;
public bool is_jni_library;

[NativeAssembler (UsesDataProvider = true)]
public string? name;
Expand Down Expand Up @@ -156,14 +157,19 @@ sealed class XamarinAndroidBundledAssembly
}
#pragma warning restore CS0649

sealed class DsoCacheState
{
public List<StructureInstance<DSOCacheEntry>> DsoCache = [];
public List<DSOCacheEntry> JniPreloadDSOs = [];
public List<StructureInstance<DSOCacheEntry>> AotDsoCache = [];
}

// Keep in sync with FORMAT_TAG in src/monodroid/jni/xamarin-app.hh
const ulong FORMAT_TAG = 0x00025E6972616D58; // 'Xmari^XY' where XY is the format version

SortedDictionary <string, string>? environmentVariables;
SortedDictionary <string, string>? systemProperties;
StructureInstance? application_config;
List<StructureInstance<DSOCacheEntry>>? dsoCache;
List<StructureInstance<DSOCacheEntry>>? aotDsoCache;
List<StructureInstance<XamarinAndroidBundledAssembly>>? xamarinAndroidBundledAssemblies;

StructureInfo? applicationConfigStructureInfo;
Expand Down Expand Up @@ -229,7 +235,7 @@ protected override void Construct (LlvmIrModule module)
};
module.Add (sysProps, stringGroupName: "sysprop", stringGroupComment: " System properties name:value pairs");

(dsoCache, aotDsoCache) = InitDSOCache ();
DsoCacheState dsoState = InitDSOCache ();
var app_cfg = new ApplicationConfig {
uses_mono_llvm = UsesMonoLLVM,
uses_mono_aot = UsesMonoAOT,
Expand All @@ -249,8 +255,8 @@ protected override void Construct (LlvmIrModule module)
number_of_assemblies_in_apk = (uint)NumberOfAssembliesInApk,
number_of_shared_libraries = (uint)NativeLibraries.Count,
bundled_assembly_name_width = (uint)BundledAssemblyNameWidth,
number_of_dso_cache_entries = (uint)dsoCache.Count,
number_of_aot_cache_entries = (uint)aotDsoCache.Count,
number_of_dso_cache_entries = (uint)dsoState.DsoCache.Count,
number_of_aot_cache_entries = (uint)dsoState.AotDsoCache.Count,
android_runtime_jnienv_class_token = (uint)AndroidRuntimeJNIEnvToken,
jnienv_initialize_method_token = (uint)JNIEnvInitializeToken,
jnienv_registerjninatives_method_token = (uint)JNIEnvRegisterJniNativesToken,
Expand All @@ -262,13 +268,23 @@ protected override void Construct (LlvmIrModule module)
application_config = new StructureInstance<ApplicationConfig> (applicationConfigStructureInfo, app_cfg);
module.AddGlobalVariable ("application_config", application_config);

var dso_cache = new LlvmIrGlobalVariable (dsoCache, "dso_cache", LlvmIrVariableOptions.GlobalWritable) {
var dso_cache = new LlvmIrGlobalVariable (dsoState.DsoCache, "dso_cache", LlvmIrVariableOptions.GlobalWritable) {
Comment = " DSO cache entries",
BeforeWriteCallback = HashAndSortDSOCache,
};
module.Add (dso_cache);

var aot_dso_cache = new LlvmIrGlobalVariable (aotDsoCache, "aot_dso_cache", LlvmIrVariableOptions.GlobalWritable) {
// This variable MUST be written after `dso_cache` since it relies on sorting performed by HashAndSortDSOCache
var dso_jni_preloads_idx = new LlvmIrGlobalVariable (new List<uint> (), "dso_jni_preloads_idx", LlvmIrVariableOptions.GlobalConstant) {
Comment = " Indices into dso_cache[] of DSO libraries to preload because of JNI use",
ArrayItemCount = (uint)dsoState.JniPreloadDSOs.Count,
BeforeWriteCallback = PopulatePreloadIndices,
BeforeWriteCallbackCallerState = dsoState,
};
module.AddGlobalVariable ("dso_jni_preloads_idx_count", dso_jni_preloads_idx.ArrayItemCount);
module.Add (dso_jni_preloads_idx);

var aot_dso_cache = new LlvmIrGlobalVariable (dsoState.AotDsoCache, "aot_dso_cache", LlvmIrVariableOptions.GlobalWritable) {
Comment = " AOT DSO cache entries",
BeforeWriteCallback = HashAndSortDSOCache,
};
Expand Down Expand Up @@ -330,6 +346,36 @@ void AddAssemblyStores (LlvmIrModule module)
module.Add (assembly_store);
}

void PopulatePreloadIndices (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state)
{
var indices = variable.Value as List<uint>;
if (indices == null) {
throw new InvalidOperationException ($"Internal error: DSO preload indices list instance not present.");
}

var dsoState = state as DsoCacheState;
if (dsoState == null) {
throw new InvalidOperationException ($"Internal error: DSO state not present.");
}

foreach (DSOCacheEntry preload in dsoState.JniPreloadDSOs) {
int dsoIdx = dsoState.DsoCache.FindIndex (entry => {
if (entry.Instance == null) {
return false;
}

return entry.Instance.hash == preload.hash && entry.Instance.real_name_hash == preload.real_name_hash;
});

if (dsoIdx == -1) {
throw new InvalidOperationException ($"Internal error: DSO entry in JNI preload list not found in the DSO cache list.");
}

indices.Add ((uint)dsoIdx);
}
indices.Sort ();
}

void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state)
{
var cache = variable.Value as List<StructureInstance<DSOCacheEntry>>;
Expand Down Expand Up @@ -358,9 +404,9 @@ void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, ob
});
}

(List<StructureInstance<DSOCacheEntry>> dsoCache, List<StructureInstance<DSOCacheEntry>> aotDsoCache) InitDSOCache ()
DsoCacheState InitDSOCache ()
{
var dsos = new List<(string name, string nameLabel, bool ignore)> ();
var dsos = new List<(string name, string nameLabel, bool ignore, ITaskItem item)> ();
var nameCache = new HashSet<string> (StringComparer.OrdinalIgnoreCase);

foreach (ITaskItem item in NativeLibraries) {
Expand All @@ -374,26 +420,37 @@ void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, ob
continue;
}

dsos.Add ((name, $"dsoName{dsos.Count.ToString (CultureInfo.InvariantCulture)}", ELFHelper.IsEmptyAOTLibrary (Log, item.ItemSpec)));
dsos.Add ((name, $"dsoName{dsos.Count.ToString (CultureInfo.InvariantCulture)}", ELFHelper.IsEmptyAOTLibrary (Log, item.ItemSpec), item));
}

var dsoCache = new List<StructureInstance<DSOCacheEntry>> ();
var jniPreloads = new List<DSOCacheEntry> ();
var aotDsoCache = new List<StructureInstance<DSOCacheEntry>> ();
var nameMutations = new List<string> ();

for (int i = 0; i < dsos.Count; i++) {
string name = dsos[i].name;

bool isJniLibrary = ELFHelper.IsJniLibrary (Log, dsos[i].item.ItemSpec);
bool ignore = dsos[i].ignore;

nameMutations.Clear();
AddNameMutations (name);

// All mutations point to the actual library name, but have hash of the mutated one
foreach (string entryName in nameMutations) {
var entry = new DSOCacheEntry {
HashedName = entryName,
hash = 0, // Hash is arch-specific, we compute it before writing
ignore = dsos[i].ignore,
ignore = ignore,
is_jni_library = isJniLibrary,
name = name,
};

if (entry.is_jni_library && entry.HashedName == name && !ApplicationConfigNativeAssemblyGeneratorCLR.DsoCacheJniPreloadIgnore.Contains (name)) {
jniPreloads.Add (entry);
}

var item = new StructureInstance<DSOCacheEntry> (dsoCacheEntryStructureInfo, entry);
if (name.StartsWith ("libaot-", StringComparison.OrdinalIgnoreCase)) {
aotDsoCache.Add (item);
Expand All @@ -403,7 +460,11 @@ void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, ob
}
}

return (dsoCache, aotDsoCache);
return new DsoCacheState {
DsoCache = dsoCache,
AotDsoCache = aotDsoCache,
JniPreloadDSOs = jniPreloads,
};

void AddNameMutations (string name)
{
Expand Down
98 changes: 82 additions & 16 deletions src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGeneratorCLR.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ sealed class DSOCacheEntry
[NativeAssembler (NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)]
public ulong real_name_hash;
public bool ignore;
public bool is_jni_library;

[NativeAssembler (UsesDataProvider = true)]
public uint name_index;
Expand Down Expand Up @@ -238,15 +239,27 @@ sealed class XamarinAndroidBundledAssembly
}
#pragma warning restore CS0649

sealed class DsoCacheState
{
public List<StructureInstance<DSOCacheEntry>> DsoCache = [];
public List<DSOCacheEntry> JniPreloadDSOs = [];
public List<StructureInstance<DSOCacheEntry>> AotDsoCache = [];
public LlvmIrStringBlob NamesBlob = null!;
}

// Keep in sync with FORMAT_TAG in src/monodroid/jni/xamarin-app.hh
const ulong FORMAT_TAG = 0x00025E6972616D58; // 'Xmari^XY' where XY is the format version

// List of library names to ignore when generating the list of JNI-using libraries to preload
internal static readonly HashSet<string> DsoCacheJniPreloadIgnore = new (StringComparer.OrdinalIgnoreCase) {
"libmonodroid.so",
};

SortedDictionary <string, string>? environmentVariables;
SortedDictionary <string, string>? systemProperties;
SortedDictionary <string, string>? runtimeProperties;
StructureInstance? application_config;
List<StructureInstance<DSOCacheEntry>>? dsoCache;
List<StructureInstance<DSOCacheEntry>>? aotDsoCache;

#pragma warning disable CS0649 // Field is never assigned to, and will always have its default value - assigned conditionally by build process
List<StructureInstance<XamarinAndroidBundledAssembly>>? xamarinAndroidBundledAssemblies;
#pragma warning restore CS0649
Expand All @@ -261,10 +274,10 @@ sealed class XamarinAndroidBundledAssembly
StructureInfo? assemblyStoreRuntimeDataStructureInfo;
StructureInfo? runtimePropertyStructureInfo;
StructureInfo? runtimePropertyIndexEntryStructureInfo;
#pragma warning disable CS0169 // Field is never used - might be used in future versions
#pragma warning disable CS0169 // Field is never used - might be used in future versions
StructureInfo? hostConfigurationPropertyStructureInfo;
#pragma warning restore CS0169
#pragma warning disable CS0169 // Field is never used - might be used in future versions
#pragma warning disable CS0169 // Field is never used - might be used in future versions
StructureInfo? hostConfigurationPropertiesStructureInfo;
#pragma warning restore CS0169
StructureInfo? appEnvironmentVariableStructureInfo;
Expand Down Expand Up @@ -349,7 +362,7 @@ protected override void Construct (LlvmIrModule module)
};
module.Add (sysProps, stringGroupName: "sysprop", stringGroupComment: " System properties name:value pairs");

(dsoCache, aotDsoCache, LlvmIrStringBlob dsoNamesBlob) = InitDSOCache ();
DsoCacheState dsoState = InitDSOCache ();
var app_cfg = new ApplicationConfigCLR {
uses_assembly_preload = UsesAssemblyPreload,
jni_add_native_method_registration_attribute_present = JniAddNativeMethodRegistrationAttributePresent,
Expand All @@ -363,8 +376,8 @@ protected override void Construct (LlvmIrModule module)
number_of_assemblies_in_apk = (uint)NumberOfAssembliesInApk,
number_of_shared_libraries = (uint)NativeLibraries.Count,
bundled_assembly_name_width = (uint)BundledAssemblyNameWidth,
number_of_dso_cache_entries = (uint)dsoCache.Count,
number_of_aot_cache_entries = (uint)aotDsoCache.Count,
number_of_dso_cache_entries = (uint)dsoState.DsoCache.Count,
number_of_aot_cache_entries = (uint)dsoState.AotDsoCache.Count,
android_runtime_jnienv_class_token = (uint)AndroidRuntimeJNIEnvToken,
jnienv_initialize_method_token = (uint)JNIEnvInitializeToken,
jnienv_registerjninatives_method_token = (uint)JNIEnvRegisterJniNativesToken,
Expand All @@ -375,18 +388,28 @@ protected override void Construct (LlvmIrModule module)
application_config = new StructureInstance<ApplicationConfigCLR> (applicationConfigStructureInfo, app_cfg);
module.AddGlobalVariable ("application_config", application_config);

var dso_cache = new LlvmIrGlobalVariable (dsoCache, "dso_cache", LlvmIrVariableOptions.GlobalWritable) {
var dso_cache = new LlvmIrGlobalVariable (dsoState.DsoCache, "dso_cache", LlvmIrVariableOptions.GlobalWritable) {
Comment = " DSO cache entries",
BeforeWriteCallback = HashAndSortDSOCache,
};
module.Add (dso_cache);

var aot_dso_cache = new LlvmIrGlobalVariable (aotDsoCache, "aot_dso_cache", LlvmIrVariableOptions.GlobalWritable) {
// This variable MUST be written after `dso_cache` since it relies on sorting performed by HashAndSortDSOCache
var dso_jni_preloads_idx = new LlvmIrGlobalVariable (new List<uint> (), "dso_jni_preloads_idx", LlvmIrVariableOptions.GlobalConstant) {
Comment = " Indices into dso_cache[] of DSO libraries to preload because of JNI use",
ArrayItemCount = (uint)dsoState.JniPreloadDSOs.Count,
BeforeWriteCallback = PopulatePreloadIndices,
BeforeWriteCallbackCallerState = dsoState,
};
module.AddGlobalVariable ("dso_jni_preloads_idx_count", dso_jni_preloads_idx.ArrayItemCount);
module.Add (dso_jni_preloads_idx);

var aot_dso_cache = new LlvmIrGlobalVariable (dsoState.AotDsoCache, "aot_dso_cache", LlvmIrVariableOptions.GlobalWritable) {
Comment = " AOT DSO cache entries",
BeforeWriteCallback = HashAndSortDSOCache,
};
module.Add (aot_dso_cache);
module.AddGlobalVariable ("dso_names_data", dsoNamesBlob, LlvmIrVariableOptions.GlobalConstant);
module.AddGlobalVariable ("dso_names_data", dsoState.NamesBlob, LlvmIrVariableOptions.GlobalConstant);

var dso_apk_entries = new LlvmIrGlobalVariable (typeof(List<StructureInstance<DSOApkEntry>>), "dso_apk_entries") {
ArrayItemCount = (ulong)NativeLibraries.Count,
Expand Down Expand Up @@ -543,6 +566,36 @@ void AddAssemblyStores (LlvmIrModule module)
module.Add (assembly_store);
}

void PopulatePreloadIndices (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state)
{
var indices = variable.Value as List<uint>;
if (indices == null) {
throw new InvalidOperationException ($"Internal error: DSO preload indices list instance not present.");
}

var dsoState = state as DsoCacheState;
if (dsoState == null) {
throw new InvalidOperationException ($"Internal error: DSO state not present.");
}

foreach (DSOCacheEntry preload in dsoState.JniPreloadDSOs) {
int dsoIdx = dsoState.DsoCache.FindIndex (entry => {
if (entry.Instance == null) {
return false;
}

return entry.Instance.hash == preload.hash && entry.Instance.real_name_hash == preload.real_name_hash;
});

if (dsoIdx == -1) {
throw new InvalidOperationException ($"Internal error: DSO entry in JNI preload list not found in the DSO cache list.");
}

indices.Add ((uint)dsoIdx);
}
indices.Sort ();
}

void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state)
{
var cache = variable.Value as List<StructureInstance<DSOCacheEntry>>;
Expand Down Expand Up @@ -571,10 +624,9 @@ void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, ob
});
}

(List<StructureInstance<DSOCacheEntry>> dsoCache, List<StructureInstance<DSOCacheEntry>> aotDsoCache, LlvmIrStringBlob namesBlob)
InitDSOCache ()
DsoCacheState InitDSOCache ()
{
var dsos = new List<(string name, string nameLabel, bool ignore)> ();
var dsos = new List<(string name, string nameLabel, bool ignore, ITaskItem item)> ();
var nameCache = new HashSet<string> (StringComparer.OrdinalIgnoreCase);

foreach (ITaskItem item in NativeLibraries) {
Expand All @@ -588,10 +640,11 @@ void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, ob
continue;
}

dsos.Add ((name, $"dsoName{dsos.Count.ToString (CultureInfo.InvariantCulture)}", ELFHelper.IsEmptyAOTLibrary (Log, item.ItemSpec)));
dsos.Add ((name, $"dsoName{dsos.Count.ToString (CultureInfo.InvariantCulture)}", ELFHelper.IsEmptyAOTLibrary (Log, item.ItemSpec), item));
}

var dsoCache = new List<StructureInstance<DSOCacheEntry>> ();
var jniPreloads = new List<DSOCacheEntry> ();
var aotDsoCache = new List<StructureInstance<DSOCacheEntry>> ();
var nameMutations = new List<string> ();
var dsoNamesBlob = new LlvmIrStringBlob ();
Expand All @@ -600,6 +653,9 @@ void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, ob
string name = dsos[i].name;
(int nameOffset, _) = dsoNamesBlob.Add (name);

bool isJniLibrary = ELFHelper.IsJniLibrary (Log, dsos[i].item.ItemSpec);
bool ignore = dsos[i].ignore;

nameMutations.Clear();
AddNameMutations (name);
// All mutations point to the actual library name, but have hash of the mutated one
Expand All @@ -609,10 +665,15 @@ void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, ob
RealName = name,

hash = 0, // Hash is arch-specific, we compute it before writing
ignore = dsos[i].ignore,
ignore = ignore,
is_jni_library = isJniLibrary,
name_index = (uint)nameOffset,
};

if (entry.is_jni_library && entry.HashedName == name && !DsoCacheJniPreloadIgnore.Contains (name)) {
jniPreloads.Add (entry);
}

var item = new StructureInstance<DSOCacheEntry> (dsoCacheEntryStructureInfo, entry);
if (name.StartsWith ("libaot-", StringComparison.OrdinalIgnoreCase)) {
aotDsoCache.Add (item);
Expand All @@ -622,7 +683,12 @@ void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, ob
}
}

return (dsoCache, aotDsoCache, dsoNamesBlob);
return new DsoCacheState {
DsoCache = dsoCache,
JniPreloadDSOs = jniPreloads,
AotDsoCache = aotDsoCache,
NamesBlob = dsoNamesBlob,
};

void AddNameMutations (string name)
{
Expand Down
Loading
Loading