Skip to content

Commit 20cd062

Browse files
jpobstjpobst
authored
[XABT] GeneratePackageManagerJava - Convert NativeCodeGenState to Cecil-less DTOs. (#10058)
Context: #10062 _One_ issue with our desire to move around LLVM Marshal Method build code so that it can work with Ready2Run is that our current implementation relies on holding Cecil `AssemblyDefinition` objects open in global state across multiple targets. While these objects are open, no other process can access our assemblies, making it harder to reorder targets and tasks. As a first step towards refactoring this process, let's move the data into a DTO that does not require Cecil. Additionally, this makes it clear _what_ data is actually consumed by `<GeneratePackageManagerJava>` rather than "entire assemblies" so we can focus on how to properly obtain and persist the data. This still relies on `BuildEngine4.RegisterTaskObject` to store the data, but a likely future step would be to persist it to a file instead.
1 parent 979306c commit 20cd062

File tree

5 files changed

+312
-49
lines changed

5 files changed

+312
-49
lines changed

‎src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ namespace Xamarin.Android.Tasks
2424
public class GenerateJavaStubs : AndroidTask
2525
{
2626
public const string NativeCodeGenStateRegisterTaskKey = ".:!MarshalMethods!:.";
27+
public const string NativeCodeGenStateObjectRegisterTaskKey = ".:!MarshalMethodsObject!:.";
2728

2829
public override string TaskPrefix => "GJS";
2930

‎src/Xamarin.Android.Build.Tasks/Tasks/GenerateMainAndroidManifest.cs

+16-9
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,8 @@ public class GenerateMainAndroidManifest : AndroidTask
5656

5757
public override bool RunTask ()
5858
{
59-
// Retrieve the stored NativeCodeGenState
60-
var nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<ConcurrentDictionary<AndroidTargetArch, NativeCodeGenState>> (
59+
// Retrieve the stored NativeCodeGenState (and remove it from the cache)
60+
var nativeCodeGenStates = BuildEngine4.UnregisterTaskObjectAssemblyLocal<ConcurrentDictionary<AndroidTargetArch, NativeCodeGenState>> (
6161
MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory),
6262
RegisteredTaskObjectLifetime.Build
6363
);
@@ -74,14 +74,21 @@ public override bool RunTask ()
7474
var additionalProviders = MergeManifest (templateCodeGenState, GenerateJavaStubs.MaybeGetArchAssemblies (userAssembliesPerArch, templateCodeGenState.TargetArch));
7575
GenerateAdditionalProviderSources (templateCodeGenState, additionalProviders);
7676

77-
// Marshal methods needs this data in the <GeneratePackageManagerJava/> later,
78-
// but if we're not using marshal methods we need to dispose of the resolver.
79-
if (!UseMarshalMethods) {
80-
Log.LogDebugMessage ($"Disposing all {nameof (NativeCodeGenState)}.{nameof (NativeCodeGenState.Resolver)}");
8177

82-
foreach (var state in nativeCodeGenStates.Values) {
83-
state.Resolver.Dispose ();
84-
}
78+
// If we still need the NativeCodeGenState in <GeneratePackageManagerJava/> because we're using marshal methods,
79+
// we're going to transfer it to a new object that doesn't require holding open Cecil AssemblyDefinitions.
80+
if (UseMarshalMethods) {
81+
var nativeCodeGenStateObject = MarshalMethodCecilAdapter.GetNativeCodeGenStateCollection (Log, nativeCodeGenStates);
82+
83+
Log.LogDebugMessage ($"Saving {nameof (NativeCodeGenStateObject)} to {nameof (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey)}");
84+
BuildEngine4.RegisterTaskObjectAssemblyLocal (MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory), nativeCodeGenStateObject, RegisteredTaskObjectLifetime.Build);
85+
}
86+
87+
// Dispose the Cecil resolvers so the assemblies are closed.
88+
Log.LogDebugMessage ($"Disposing all {nameof (NativeCodeGenState)}.{nameof (NativeCodeGenState.Resolver)}");
89+
90+
foreach (var state in nativeCodeGenStates.Values) {
91+
state.Resolver.Dispose ();
8592
}
8693

8794
if (Log.HasLoggedErrors) {

‎src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs

+6-13
Original file line numberDiff line numberDiff line change
@@ -314,10 +314,11 @@ void AddEnvironment ()
314314
}
315315
}
316316

317-
ConcurrentDictionary<AndroidTargetArch, NativeCodeGenState>? nativeCodeGenStates = null;
317+
NativeCodeGenStateCollection? nativeCodeGenStates = null;
318+
318319
if (enableMarshalMethods) {
319-
nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<ConcurrentDictionary<AndroidTargetArch, NativeCodeGenState>> (
320-
MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory),
320+
nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<NativeCodeGenStateCollection> (
321+
MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory),
321322
RegisteredTaskObjectLifetime.Build
322323
);
323324
}
@@ -423,17 +424,9 @@ void AddEnvironment ()
423424
}
424425
}
425426

426-
if (nativeCodeGenStates is not null) {
427-
// Dispose all XAAssemblyResolvers
428-
Log.LogDebugMessage ($"Disposing all {nameof (NativeCodeGenState)}.{nameof (NativeCodeGenState.Resolver)}");
429-
foreach (var state in nativeCodeGenStates.Values) {
430-
state.Resolver.Dispose ();
431-
}
432-
}
433-
434-
NativeCodeGenState EnsureCodeGenState (AndroidTargetArch targetArch)
427+
NativeCodeGenStateObject EnsureCodeGenState (AndroidTargetArch targetArch)
435428
{
436-
if (nativeCodeGenStates == null || !nativeCodeGenStates.TryGetValue (targetArch, out NativeCodeGenState? state)) {
429+
if (nativeCodeGenStates == null || !nativeCodeGenStates.States.TryGetValue (targetArch, out NativeCodeGenStateObject? state)) {
437430
throw new InvalidOperationException ($"Internal error: missing native code generation state for architecture '{targetArch}'");
438431
}
439432

‎src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodCecilAdapter.cs

+253
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,253 @@
1+
#nullable enable
2+
using System;
3+
using System.Collections.Concurrent;
4+
using System.Collections.Generic;
5+
using System.Diagnostics.CodeAnalysis;
6+
using Microsoft.Android.Build.Tasks;
7+
using Microsoft.Build.Utilities;
8+
using Mono.Cecil;
9+
using Xamarin.Android.Tools;
10+
11+
namespace Xamarin.Android.Tasks;
12+
13+
class MarshalMethodCecilAdapter
14+
{
15+
public static NativeCodeGenStateCollection? GetNativeCodeGenStateCollection (TaskLoggingHelper log, ConcurrentDictionary<AndroidTargetArch, NativeCodeGenState>? nativeCodeGenStates)
16+
{
17+
if (nativeCodeGenStates is null) {
18+
log.LogDebugMessage ($"No {nameof (NativeCodeGenState)} found");
19+
return null;
20+
}
21+
22+
var collection = new NativeCodeGenStateCollection ();
23+
24+
// Convert each architecture
25+
foreach (var kvp in nativeCodeGenStates) {
26+
var arch = kvp.Key;
27+
var state = kvp.Value;
28+
var obj = CreateNativeCodeGenState (arch, state);
29+
collection.States.Add (arch, obj);
30+
}
31+
32+
return collection;
33+
}
34+
35+
static NativeCodeGenStateObject CreateNativeCodeGenState (AndroidTargetArch arch, NativeCodeGenState state)
36+
{
37+
var obj = new NativeCodeGenStateObject ();
38+
39+
if (state.Classifier is null)
40+
return obj;
41+
42+
foreach (var group in state.Classifier.MarshalMethods) {
43+
var methods = new List<MarshalMethodEntryObject> (group.Value.Count);
44+
45+
foreach (var method in group.Value) {
46+
var entry = CreateEntry (method, state.ManagedMarshalMethodsLookupInfo);
47+
methods.Add (entry);
48+
}
49+
50+
obj.MarshalMethods.Add (group.Key, methods);
51+
}
52+
53+
return obj;
54+
}
55+
56+
static MarshalMethodEntryObject CreateEntry (MarshalMethodEntry entry, ManagedMarshalMethodsLookupInfo? info)
57+
{
58+
var obj = new MarshalMethodEntryObject (
59+
declaringType: CreateDeclaringType (entry.DeclaringType),
60+
implementedMethod: CreateMethod (entry.ImplementedMethod),
61+
isSpecial: entry.IsSpecial,
62+
jniTypeName: entry.JniTypeName,
63+
jniMethodName: entry.JniMethodName,
64+
jniMethodSignature: entry.JniMethodSignature,
65+
nativeCallback: CreateMethod (entry.NativeCallback),
66+
registeredMethod: CreateMethodBase (entry.RegisteredMethod)
67+
);
68+
69+
if (info is not null) {
70+
(uint assemblyIndex, uint classIndex, uint methodIndex) = info.GetIndex (entry.NativeCallback);
71+
72+
obj.NativeCallback.AssemblyIndex = assemblyIndex;
73+
obj.NativeCallback.ClassIndex = classIndex;
74+
obj.NativeCallback.MethodIndex = methodIndex;
75+
}
76+
77+
return obj;
78+
}
79+
80+
static MarshalMethodEntryTypeObject CreateDeclaringType (TypeDefinition type)
81+
{
82+
var cecilModule = type.Module;
83+
var cecilAssembly = cecilModule.Assembly;
84+
85+
var assembly = new MarshalMethodEntryAssemblyObject (
86+
fullName: cecilAssembly.FullName,
87+
nameFullName: cecilAssembly.Name.FullName,
88+
mainModuleFileName: cecilAssembly.MainModule.FileName,
89+
nameName: cecilAssembly.Name.Name
90+
);
91+
92+
var module = new MarshalMethodEntryModuleObject (
93+
assembly: assembly
94+
);
95+
96+
return new MarshalMethodEntryTypeObject (
97+
fullName: type.FullName,
98+
metadataToken: type.MetadataToken.ToUInt32 (),
99+
module: module);
100+
}
101+
102+
[return:NotNullIfNotNull (nameof (method))]
103+
static MarshalMethodEntryMethodObject? CreateMethod (MethodDefinition? method)
104+
{
105+
if (method is null)
106+
return null;
107+
108+
var parameters = new List<MarshalMethodEntryMethodParameterObject> (method.Parameters.Count);
109+
110+
foreach (var parameter in method.Parameters) {
111+
parameters.Add (new MarshalMethodEntryMethodParameterObject (
112+
name: parameter.Name,
113+
parameterTypeName: parameter.ParameterType.Name
114+
));
115+
}
116+
117+
return new MarshalMethodEntryMethodObject (
118+
name: method.Name,
119+
fullName: method.FullName,
120+
declaringType: CreateDeclaringType (method.DeclaringType),
121+
metadataToken: method.MetadataToken.ToUInt32 (),
122+
parameters: parameters
123+
);
124+
}
125+
126+
static MarshalMethodEntryMethodBaseObject? CreateMethodBase (MethodDefinition? method)
127+
{
128+
if (method is null)
129+
return null;
130+
return new MarshalMethodEntryMethodBaseObject (
131+
fullName: method.FullName
132+
);
133+
}
134+
}
135+
136+
class NativeCodeGenStateCollection
137+
{
138+
public Dictionary<AndroidTargetArch, NativeCodeGenStateObject> States { get; } = [];
139+
}
140+
141+
class NativeCodeGenStateObject
142+
{
143+
public Dictionary<string, IList<MarshalMethodEntryObject>> MarshalMethods { get; } = [];
144+
}
145+
146+
class MarshalMethodEntryObject
147+
{
148+
public MarshalMethodEntryTypeObject DeclaringType { get; }
149+
public MarshalMethodEntryMethodObject? ImplementedMethod { get; }
150+
public bool IsSpecial { get; }
151+
public string JniTypeName { get; }
152+
public string JniMethodName { get; }
153+
public string JniMethodSignature { get; }
154+
public MarshalMethodEntryMethodObject NativeCallback { get; }
155+
public MarshalMethodEntryMethodBaseObject? RegisteredMethod { get; }
156+
157+
public MarshalMethodEntryObject (MarshalMethodEntryTypeObject declaringType, MarshalMethodEntryMethodObject? implementedMethod, bool isSpecial, string jniTypeName, string jniMethodName, string jniMethodSignature, MarshalMethodEntryMethodObject nativeCallback, MarshalMethodEntryMethodBaseObject? registeredMethod)
158+
{
159+
DeclaringType = declaringType;
160+
ImplementedMethod = implementedMethod;
161+
IsSpecial = isSpecial;
162+
JniTypeName = jniTypeName;
163+
JniMethodName = jniMethodName;
164+
JniMethodSignature = jniMethodSignature;
165+
NativeCallback = nativeCallback;
166+
RegisteredMethod = registeredMethod;
167+
}
168+
}
169+
170+
class MarshalMethodEntryAssemblyObject
171+
{
172+
public string FullName { get; }
173+
public string NameFullName { get; } // Cecil's Assembly.Name.FullName
174+
public string MainModuleFileName { get; } // Cecil's Assembly.MainModule.FileName
175+
public string NameName { get; } // Cecil's Module.Name.Name
176+
177+
public MarshalMethodEntryAssemblyObject (string fullName, string nameFullName, string mainModuleFileName, string nameName)
178+
{
179+
FullName = fullName;
180+
NameFullName = nameFullName;
181+
MainModuleFileName = mainModuleFileName;
182+
NameName = nameName;
183+
}
184+
}
185+
186+
class MarshalMethodEntryModuleObject
187+
{
188+
public MarshalMethodEntryAssemblyObject Assembly { get; }
189+
190+
public MarshalMethodEntryModuleObject (MarshalMethodEntryAssemblyObject assembly)
191+
{
192+
Assembly = assembly;
193+
}
194+
}
195+
196+
class MarshalMethodEntryTypeObject
197+
{
198+
public string FullName { get; }
199+
public uint MetadataToken { get; }
200+
public MarshalMethodEntryModuleObject Module { get; }
201+
202+
public MarshalMethodEntryTypeObject (string fullName, uint metadataToken, MarshalMethodEntryModuleObject module)
203+
{
204+
FullName = fullName;
205+
MetadataToken = metadataToken;
206+
Module = module;
207+
}
208+
}
209+
210+
class MarshalMethodEntryMethodBaseObject
211+
{
212+
public string FullName { get; }
213+
214+
public MarshalMethodEntryMethodBaseObject (string fullName)
215+
{
216+
FullName = fullName;
217+
}
218+
}
219+
220+
class MarshalMethodEntryMethodObject : MarshalMethodEntryMethodBaseObject
221+
{
222+
public string Name { get; }
223+
public MarshalMethodEntryTypeObject DeclaringType { get; }
224+
public uint MetadataToken { get; }
225+
public List<MarshalMethodEntryMethodParameterObject> Parameters { get; }
226+
227+
public uint? AssemblyIndex { get; set; }
228+
public uint? ClassIndex { get; set; }
229+
public uint? MethodIndex { get; set; }
230+
231+
public bool HasParameters => Parameters.Count > 0;
232+
233+
public MarshalMethodEntryMethodObject (string name, string fullName, MarshalMethodEntryTypeObject declaringType, uint metadataToken, List<MarshalMethodEntryMethodParameterObject> parameters)
234+
: base (fullName)
235+
{
236+
Name = name;
237+
DeclaringType = declaringType;
238+
MetadataToken = metadataToken;
239+
Parameters = parameters;
240+
}
241+
}
242+
243+
class MarshalMethodEntryMethodParameterObject
244+
{
245+
public string Name { get; }
246+
public string ParameterTypeName { get; } // Cecil's ParameterDefinition.ParameterType.Name
247+
248+
public MarshalMethodEntryMethodParameterObject (string name, string parameterTypeName)
249+
{
250+
Name = name;
251+
ParameterTypeName = parameterTypeName;
252+
}
253+
}

0 commit comments

Comments
 (0)