Skip to content

Commit db2ec8e

Browse files
rolfbjarnerolfbjarne
authored
[tests] Tweak the AppSize test a bit. (##24306)
* Fail if a file was added to or removed from the app bundle. * Fail if the set of APIs that survived trimming changed. * Use Assert.Multiple to support multiple failing asserts. * Improve diagnostic output a bit.
1 parent 7bcb04b commit db2ec8e

File tree

3 files changed

+68
-33
lines changed

3 files changed

+68
-33
lines changed

‎tests/dotnet/UnitTests/AppSizeTest.cs‎

Lines changed: 66 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,11 @@ public void NativeAOT (ApplePlatform platform, string runtimeIdentifiers)
3333
// There's a tolerance in the test for minor app size variances, so if this test fails, the current change might not mean there's a big change,
3434
// there might just be many cumulative unnoticed/minor app size differences eventually triggering the test.
3535
// The test fails even if app size goes down; this way we can also keep track of good news! And additionally we won't miss it if the app size first goes down, then back up again.
36+
//
37+
// List of failure modes:
38+
// * Files added or removed from app bundle
39+
// * Total app size changed >10kb
40+
// * For those apps where assembly APIs can be compared, any API was added or removed.
3641
void Run (ApplePlatform platform, string runtimeIdentifiers, string configuration, string name, bool supportsAssemblyInspection, Dictionary<string, string>? extraProperties = null)
3742
{
3843
Configuration.IgnoreIfIgnoredPlatform (platform);
@@ -55,6 +60,17 @@ void Run (ApplePlatform platform, string runtimeIdentifiers, string configuratio
5560
var update = forceUpdate || !string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("WRITE_KNOWN_FAILURES"));
5661
var expectedDirectory = Path.Combine (Configuration.SourceRoot, "tests", "dotnet", "UnitTests", "expected");
5762

63+
Assert.Multiple (() => {
64+
AssertAppSize (platform, name, appPath, update, forceUpdate, expectedDirectory);
65+
66+
if (supportsAssemblyInspection)
67+
AssertAssemblyReport (platform, name, appPath, update, expectedDirectory);
68+
69+
});
70+
}
71+
72+
static void AssertAppSize (ApplePlatform platform, string name, string appPath, bool update, bool forceUpdate, string expectedDirectory)
73+
{
5874
// Compute the size of the app bundle, and compare it to the stored version on disk.
5975
var allFiles = Directory.GetFiles (appPath, "*", SearchOption.AllDirectories).
6076
Select (v => new FileInfo (v)).
@@ -75,24 +91,29 @@ void Run (ApplePlatform platform, string runtimeIdentifiers, string configuratio
7591
}
7692

7793
var appSizeDifference = appBundleSize - expectedAppBundleSize;
78-
if (appSizeDifference == 0 && !forceUpdate)
79-
return;
80-
8194
var toleranceInBytes = 1024 * 10; // 10kb
82-
if (toleranceInBytes >= Math.Abs (appSizeDifference)) {
83-
Console.WriteLine ($"App size difference is {FormatBytes (appSizeDifference)}, which is less than the tolerance ({toleranceInBytes}), so nothing will be reported.");
84-
if (!forceUpdate)
85-
return;
86-
}
95+
var withinTolerance = toleranceInBytes >= Math.Abs (appSizeDifference);
8796

88-
var msg = $"App size changed significantly ({FormatBytes (appSizeDifference, true)} different > tolerance of +-{FormatBytes (toleranceInBytes)}). Expected app size: {FormatBytes (expectedAppBundleSize)}, actual app size: {FormatBytes (appBundleSize)}.";
97+
string msg;
8998

90-
if (update) {
99+
if (appSizeDifference == 0) {
100+
msg = $"App size did not change. Expected app size: {FormatBytes (expectedAppBundleSize)}, actual app size: {FormatBytes (appBundleSize)}.";
101+
} else if (withinTolerance) {
102+
msg = $"App size changed, but not significantly: ({FormatBytes (appSizeDifference, true)} different <= tolerance of +-{FormatBytes (toleranceInBytes)}). Expected app size: {FormatBytes (expectedAppBundleSize)}, actual app size: {FormatBytes (appBundleSize)}.";
103+
} else {
104+
msg = $"App size changed significantly ({FormatBytes (appSizeDifference, true)} different > tolerance of +-{FormatBytes (toleranceInBytes)}). Expected app size: {FormatBytes (expectedAppBundleSize)}, actual app size: {FormatBytes (appBundleSize)}.";
105+
}
106+
107+
var updated = false;
108+
if (forceUpdate || (update && !withinTolerance)) {
91109
Directory.CreateDirectory (expectedDirectory);
92110
File.WriteAllText (expectedSizeReportPath, report.ToString ());
111+
msg += " Check the modified files for more information.";
112+
updated = true;
113+
} else if (!withinTolerance) {
114+
msg += " Set the environment variable WRITE_KNOWN_FAILURES=1, run the test again, and verify the modified files for more information.";
93115
}
94116

95-
msg += " Set the environment variable WRITE_KNOWN_FAILURES=1, run the test again, and verify the modified files for more information.";
96117
Console.WriteLine ($" {msg}");
97118

98119
var expectedLines = expectedSizeReport.SplitLines ().Skip (2).Where (v => v.IndexOf (':') >= 0).ToDictionary (v => v [..v.IndexOf (':')], v => v [(v.IndexOf (':') + 1)..]);
@@ -101,48 +122,60 @@ void Run (ApplePlatform platform, string runtimeIdentifiers, string configuratio
101122
foreach (var key in allKeys) {
102123
if (!expectedLines.TryGetValue (key, out var expectedLine)) {
103124
Console.WriteLine ($" File '{key}' was removed from app bundle: {actualLines [key]}");
125+
Assert.Fail ($"The file '{key}' was removed from the app bundle.");
104126
} else if (!actualLines.TryGetValue (key, out var actualLine)) {
105127
Console.WriteLine ($" File '{key}' was added to app bundle: {expectedLine}");
128+
Assert.Fail ($"The file '{key}' was added to the app bundle.");
106129
} else if (expectedLine != actualLine) {
107130
Console.WriteLine ($" File '{key}' changed in app bundle:");
108131
Console.WriteLine ($" -{expectedLine}");
109132
Console.WriteLine ($" +{actualLine}");
110133
}
111134
}
112135

113-
// Create a file with all the APIs that survived the trimmer; this can be useful to determine what is not trimmed away.
114-
// Note that any changes in this list when the test fails might be due to unrelated earlier changes, that didn't trigger the test
115-
// to fail, because the corresponding app size difference was within the tolerance for app size changes.
116-
if (supportsAssemblyInspection) {
117-
var asmDir = Path.Combine (appPath, GetRelativeAssemblyDirectory (platform));
118-
var preservedAPIs = new List<string> ();
119-
foreach (var dll in Directory.GetFiles (asmDir, "*.dll", SearchOption.AllDirectories)) {
120-
var relativePath = dll [(asmDir.Length + 1)..];
121-
using var ad = AssemblyDefinition.ReadAssembly (dll, new ReaderParameters { ReadingMode = ReadingMode.Deferred });
122-
foreach (var member in ad.EnumerateMembers ()) {
123-
preservedAPIs.Add ($"{relativePath}:{((ICustomAttributeProvider) member).AsFullName ()}");
124-
}
136+
if (!updated && !withinTolerance)
137+
Assert.Fail (msg);
138+
}
139+
140+
// Create a file with all the APIs that survived the trimmer; this can be useful to determine what is not trimmed away.
141+
// Note that any changes in this list when the test fails might be due to unrelated earlier changes, that didn't trigger the test
142+
// to fail, because the corresponding app size difference was within the tolerance for app size changes.
143+
void AssertAssemblyReport (ApplePlatform platform, string name, string appPath, bool update, string expectedDirectory)
144+
{
145+
var asmDir = Path.Combine (appPath, GetRelativeAssemblyDirectory (platform));
146+
var preservedAPIs = new List<string> ();
147+
foreach (var dll in Directory.GetFiles (asmDir, "*.dll", SearchOption.AllDirectories)) {
148+
var relativePath = dll [(asmDir.Length + 1)..];
149+
using var ad = AssemblyDefinition.ReadAssembly (dll, new ReaderParameters { ReadingMode = ReadingMode.Deferred });
150+
foreach (var member in ad.EnumerateMembers ()) {
151+
preservedAPIs.Add ($"{relativePath}:{((ICustomAttributeProvider) member).AsFullName ()}");
125152
}
126-
preservedAPIs.Sort ();
127-
var expectedFile = Path.Combine (expectedDirectory, $"{name}-preservedapis.txt");
128-
var expectedAPIs = File.ReadAllLines (expectedFile);
129-
var addedAPIs = preservedAPIs.Except (expectedAPIs);
130-
var removedAPIs = expectedAPIs.Except (preservedAPIs);
153+
}
154+
preservedAPIs.Sort ();
155+
var expectedFile = Path.Combine (expectedDirectory, $"{name}-preservedapis.txt");
156+
var expectedAPIs = File.ReadAllLines (expectedFile);
157+
var addedAPIs = preservedAPIs.Except (expectedAPIs).ToList ();
158+
var removedAPIs = expectedAPIs.Except (preservedAPIs).ToList ();
131159

160+
if (addedAPIs.Count () > 0) {
132161
Console.WriteLine ($" {addedAPIs.Count ()} additional APIs present:");
133162
foreach (var line in addedAPIs)
134163
Console.WriteLine ($" {line}");
164+
}
165+
if (removedAPIs.Count () > 0) {
135166
Console.WriteLine ($" {removedAPIs.Count ()} APIs not present anymore:");
136167
foreach (var line in removedAPIs)
137168
Console.WriteLine ($" {line}");
169+
}
138170

139-
if (update) {
140-
File.WriteAllLines (expectedFile, preservedAPIs);
141-
}
171+
if (update) {
172+
File.WriteAllLines (expectedFile, preservedAPIs);
142173
}
143174

144-
if (!update)
145-
Assert.Fail (msg);
175+
if (!update) {
176+
Assert.That (addedAPIs, Is.Empty, "No added APIs (set the environment variable WRITE_KNOWN_FAILURES=1 and run the test again to update the expected set of APIs)");
177+
Assert.That (removedAPIs, Is.Empty, "No removed APIs (set the environment variable WRITE_KNOWN_FAILURES=1 and run the test again to update the expected set of APIs)");
178+
}
146179
}
147180

148181
static string FormatBytes (long bytes, bool alwaysShowSign = false)

‎tests/dotnet/UnitTests/expected/iOS-MonoVM-interpreter-preservedapis.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -655,6 +655,7 @@ Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(ObjCRuntime.NativeHandle, Syst
655655
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean)
656656
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, System.Boolean, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean)
657657
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, System.Type, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean, System.Boolean, out System.Boolean&)
658+
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr)
658659
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.Boolean)
659660
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.IntPtr, System.RuntimeMethodHandle, System.Boolean, System.Boolean)
660661
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.IntPtr, System.RuntimeMethodHandle, System.Boolean)

‎tests/dotnet/UnitTests/expected/iOS-MonoVM-preservedapis.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -497,6 +497,7 @@ Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(ObjCRuntime.NativeHandle, Syst
497497
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean)
498498
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, System.Boolean, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean)
499499
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, System.Type, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean, System.Boolean, out System.Boolean&)
500+
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr)
500501
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.Boolean)
501502
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.IntPtr, System.RuntimeMethodHandle, System.Boolean, System.Boolean)
502503
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.IntPtr, System.RuntimeMethodHandle, System.Boolean)

0 commit comments

Comments
 (0)