Skip to content

[tests] Tweak the AppSize test a bit. #24306

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Nov 21, 2025
Merged

[tests] Tweak the AppSize test a bit. #24306

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d949d23
[tests] Tweak the AppSize test a bit.
rolfbjarne Nov 20, 2025
bec3da8
Simplify and tweak
rolfbjarne Nov 21, 2025
530689f
Merge remote-tracking branch 'origin/main' into dev/rolf/tests-appsiz…
rolfbjarne Nov 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 66 additions & 33 deletions tests/dotnet/UnitTests/AppSizeTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ public void NativeAOT (ApplePlatform platform, string runtimeIdentifiers)
// There's a tolerance in the test for minor app size variances, so if this test fails, the current change might not mean there's a big change,
// there might just be many cumulative unnoticed/minor app size differences eventually triggering the test.
// The test fails even if app size goes down; this way we can also keep track of good news! And additionally we won't miss it if the app size first goes down, then back up again.
//
// List of failure modes:
// * Files added or removed from app bundle
// * Total app size changed >10kb
// * For those apps where assembly APIs can be compared, any API was added or removed.
void Run (ApplePlatform platform, string runtimeIdentifiers, string configuration, string name, bool supportsAssemblyInspection, Dictionary<string, string>? extraProperties = null)
{
Configuration.IgnoreIfIgnoredPlatform (platform);
Expand All @@ -55,6 +60,17 @@ void Run (ApplePlatform platform, string runtimeIdentifiers, string configuratio
var update = forceUpdate || !string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("WRITE_KNOWN_FAILURES"));
var expectedDirectory = Path.Combine (Configuration.SourceRoot, "tests", "dotnet", "UnitTests", "expected");

Assert.Multiple (() => {
AssertAppSize (platform, name, appPath, update, forceUpdate, expectedDirectory);

if (supportsAssemblyInspection)
AssertAssemblyReport (platform, name, appPath, update, expectedDirectory);

});
}

static void AssertAppSize (ApplePlatform platform, string name, string appPath, bool update, bool forceUpdate, string expectedDirectory)
{
// Compute the size of the app bundle, and compare it to the stored version on disk.
var allFiles = Directory.GetFiles (appPath, "*", SearchOption.AllDirectories).
Select (v => new FileInfo (v)).
Expand All @@ -75,24 +91,29 @@ void Run (ApplePlatform platform, string runtimeIdentifiers, string configuratio
}

var appSizeDifference = appBundleSize - expectedAppBundleSize;
if (appSizeDifference == 0 && !forceUpdate)
return;

var toleranceInBytes = 1024 * 10; // 10kb
if (toleranceInBytes >= Math.Abs (appSizeDifference)) {
Console.WriteLine ($"App size difference is {FormatBytes (appSizeDifference)}, which is less than the tolerance ({toleranceInBytes}), so nothing will be reported.");
if (!forceUpdate)
return;
}
var withinTolerance = toleranceInBytes >= Math.Abs (appSizeDifference);

var msg = $"App size changed significantly ({FormatBytes (appSizeDifference, true)} different > tolerance of +-{FormatBytes (toleranceInBytes)}). Expected app size: {FormatBytes (expectedAppBundleSize)}, actual app size: {FormatBytes (appBundleSize)}.";
string msg;

if (update) {
if (appSizeDifference == 0) {
msg = $"App size did not change. Expected app size: {FormatBytes (expectedAppBundleSize)}, actual app size: {FormatBytes (appBundleSize)}.";
} else if (withinTolerance) {
msg = $"App size changed, but not significantly: ({FormatBytes (appSizeDifference, true)} different <= tolerance of +-{FormatBytes (toleranceInBytes)}). Expected app size: {FormatBytes (expectedAppBundleSize)}, actual app size: {FormatBytes (appBundleSize)}.";
} else {
msg = $"App size changed significantly ({FormatBytes (appSizeDifference, true)} different > tolerance of +-{FormatBytes (toleranceInBytes)}). Expected app size: {FormatBytes (expectedAppBundleSize)}, actual app size: {FormatBytes (appBundleSize)}.";
}

var updated = false;
if (forceUpdate || (update && !withinTolerance)) {
Directory.CreateDirectory (expectedDirectory);
File.WriteAllText (expectedSizeReportPath, report.ToString ());
msg += " Check the modified files for more information.";
updated = true;
} else if (!withinTolerance) {
msg += " Set the environment variable WRITE_KNOWN_FAILURES=1, run the test again, and verify the modified files for more information.";
}

msg += " Set the environment variable WRITE_KNOWN_FAILURES=1, run the test again, and verify the modified files for more information.";
Console.WriteLine ($" {msg}");

var expectedLines = expectedSizeReport.SplitLines ().Skip (2).Where (v => v.IndexOf (':') >= 0).ToDictionary (v => v [..v.IndexOf (':')], v => v [(v.IndexOf (':') + 1)..]);
Expand All @@ -101,48 +122,60 @@ void Run (ApplePlatform platform, string runtimeIdentifiers, string configuratio
foreach (var key in allKeys) {
if (!expectedLines.TryGetValue (key, out var expectedLine)) {
Console.WriteLine ($" File '{key}' was removed from app bundle: {actualLines [key]}");
Assert.Fail ($"The file '{key}' was removed from the app bundle.");
} else if (!actualLines.TryGetValue (key, out var actualLine)) {
Console.WriteLine ($" File '{key}' was added to app bundle: {expectedLine}");
Assert.Fail ($"The file '{key}' was added to the app bundle.");
} else if (expectedLine != actualLine) {
Console.WriteLine ($" File '{key}' changed in app bundle:");
Console.WriteLine ($" -{expectedLine}");
Console.WriteLine ($" +{actualLine}");
}
}

// Create a file with all the APIs that survived the trimmer; this can be useful to determine what is not trimmed away.
// Note that any changes in this list when the test fails might be due to unrelated earlier changes, that didn't trigger the test
// to fail, because the corresponding app size difference was within the tolerance for app size changes.
if (supportsAssemblyInspection) {
var asmDir = Path.Combine (appPath, GetRelativeAssemblyDirectory (platform));
var preservedAPIs = new List<string> ();
foreach (var dll in Directory.GetFiles (asmDir, "*.dll", SearchOption.AllDirectories)) {
var relativePath = dll [(asmDir.Length + 1)..];
using var ad = AssemblyDefinition.ReadAssembly (dll, new ReaderParameters { ReadingMode = ReadingMode.Deferred });
foreach (var member in ad.EnumerateMembers ()) {
preservedAPIs.Add ($"{relativePath}:{((ICustomAttributeProvider) member).AsFullName ()}");
}
if (!updated && !withinTolerance)
Assert.Fail (msg);
}

// Create a file with all the APIs that survived the trimmer; this can be useful to determine what is not trimmed away.
// Note that any changes in this list when the test fails might be due to unrelated earlier changes, that didn't trigger the test
// to fail, because the corresponding app size difference was within the tolerance for app size changes.
void AssertAssemblyReport (ApplePlatform platform, string name, string appPath, bool update, string expectedDirectory)
{
var asmDir = Path.Combine (appPath, GetRelativeAssemblyDirectory (platform));
var preservedAPIs = new List<string> ();
foreach (var dll in Directory.GetFiles (asmDir, "*.dll", SearchOption.AllDirectories)) {
var relativePath = dll [(asmDir.Length + 1)..];
using var ad = AssemblyDefinition.ReadAssembly (dll, new ReaderParameters { ReadingMode = ReadingMode.Deferred });
foreach (var member in ad.EnumerateMembers ()) {
preservedAPIs.Add ($"{relativePath}:{((ICustomAttributeProvider) member).AsFullName ()}");
}
preservedAPIs.Sort ();
var expectedFile = Path.Combine (expectedDirectory, $"{name}-preservedapis.txt");
var expectedAPIs = File.ReadAllLines (expectedFile);
var addedAPIs = preservedAPIs.Except (expectedAPIs);
var removedAPIs = expectedAPIs.Except (preservedAPIs);
}
preservedAPIs.Sort ();
var expectedFile = Path.Combine (expectedDirectory, $"{name}-preservedapis.txt");
var expectedAPIs = File.ReadAllLines (expectedFile);
var addedAPIs = preservedAPIs.Except (expectedAPIs).ToList ();
var removedAPIs = expectedAPIs.Except (preservedAPIs).ToList ();

if (addedAPIs.Count () > 0) {
Console.WriteLine ($" {addedAPIs.Count ()} additional APIs present:");
foreach (var line in addedAPIs)
Console.WriteLine ($" {line}");
}
if (removedAPIs.Count () > 0) {
Console.WriteLine ($" {removedAPIs.Count ()} APIs not present anymore:");
foreach (var line in removedAPIs)
Console.WriteLine ($" {line}");
}

if (update) {
File.WriteAllLines (expectedFile, preservedAPIs);
}
if (update) {
File.WriteAllLines (expectedFile, preservedAPIs);
}

if (!update)
Assert.Fail (msg);
if (!update) {
Assert.That (addedAPIs, Is.Empty, "No added APIs (set the environment variable WRITE_KNOWN_FAILURES=1 and run the test again to update the expected set of APIs)");
Assert.That (removedAPIs, Is.Empty, "No removed APIs (set the environment variable WRITE_KNOWN_FAILURES=1 and run the test again to update the expected set of APIs)");
}
}

static string FormatBytes (long bytes, bool alwaysShowSign = false)
Expand Down
1 change: 1 addition & 0 deletions tests/dotnet/UnitTests/expected/iOS-MonoVM-interpreter-preservedapis.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -655,6 +655,7 @@ Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(ObjCRuntime.NativeHandle, Syst
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean)
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, System.Boolean, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean)
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, System.Type, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean, System.Boolean, out System.Boolean&)
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr)
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.Boolean)
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.IntPtr, System.RuntimeMethodHandle, System.Boolean, System.Boolean)
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.IntPtr, System.RuntimeMethodHandle, System.Boolean)
Expand Down
1 change: 1 addition & 0 deletions tests/dotnet/UnitTests/expected/iOS-MonoVM-preservedapis.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,7 @@ Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(ObjCRuntime.NativeHandle, Syst
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean)
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, System.Boolean, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean)
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, System.Type, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean, System.Boolean, out System.Boolean&)
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr)
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.Boolean)
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.IntPtr, System.RuntimeMethodHandle, System.Boolean, System.Boolean)
Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.IntPtr, System.RuntimeMethodHandle, System.Boolean)
Expand Down