diff --git a/tests/dotnet/UnitTests/AppSizeTest.cs b/tests/dotnet/UnitTests/AppSizeTest.cs index 970814a3712..8db03128b43 100644 --- a/tests/dotnet/UnitTests/AppSizeTest.cs +++ b/tests/dotnet/UnitTests/AppSizeTest.cs @@ -33,6 +33,11 @@ public void NativeAOT (ApplePlatform platform, string runtimeIdentifiers) // There's a tolerance in the test for minor app size variances, so if this test fails, the current change might not mean there's a big change, // there might just be many cumulative unnoticed/minor app size differences eventually triggering the test. // The test fails even if app size goes down; this way we can also keep track of good news! And additionally we won't miss it if the app size first goes down, then back up again. + // + // List of failure modes: + // * Files added or removed from app bundle + // * Total app size changed >10kb + // * For those apps where assembly APIs can be compared, any API was added or removed. void Run (ApplePlatform platform, string runtimeIdentifiers, string configuration, string name, bool supportsAssemblyInspection, Dictionary? extraProperties = null) { Configuration.IgnoreIfIgnoredPlatform (platform); @@ -55,6 +60,17 @@ void Run (ApplePlatform platform, string runtimeIdentifiers, string configuratio var update = forceUpdate || !string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("WRITE_KNOWN_FAILURES")); var expectedDirectory = Path.Combine (Configuration.SourceRoot, "tests", "dotnet", "UnitTests", "expected"); + Assert.Multiple (() => { + AssertAppSize (platform, name, appPath, update, forceUpdate, expectedDirectory); + + if (supportsAssemblyInspection) + AssertAssemblyReport (platform, name, appPath, update, expectedDirectory); + + }); + } + + static void AssertAppSize (ApplePlatform platform, string name, string appPath, bool update, bool forceUpdate, string expectedDirectory) + { // Compute the size of the app bundle, and compare it to the stored version on disk. var allFiles = Directory.GetFiles (appPath, "*", SearchOption.AllDirectories). Select (v => new FileInfo (v)). @@ -75,24 +91,29 @@ void Run (ApplePlatform platform, string runtimeIdentifiers, string configuratio } var appSizeDifference = appBundleSize - expectedAppBundleSize; - if (appSizeDifference == 0 && !forceUpdate) - return; - var toleranceInBytes = 1024 * 10; // 10kb - if (toleranceInBytes >= Math.Abs (appSizeDifference)) { - Console.WriteLine ($"App size difference is {FormatBytes (appSizeDifference)}, which is less than the tolerance ({toleranceInBytes}), so nothing will be reported."); - if (!forceUpdate) - return; - } + var withinTolerance = toleranceInBytes >= Math.Abs (appSizeDifference); - var msg = $"App size changed significantly ({FormatBytes (appSizeDifference, true)} different > tolerance of +-{FormatBytes (toleranceInBytes)}). Expected app size: {FormatBytes (expectedAppBundleSize)}, actual app size: {FormatBytes (appBundleSize)}."; + string msg; - if (update) { + if (appSizeDifference == 0) { + msg = $"App size did not change. Expected app size: {FormatBytes (expectedAppBundleSize)}, actual app size: {FormatBytes (appBundleSize)}."; + } else if (withinTolerance) { + msg = $"App size changed, but not significantly: ({FormatBytes (appSizeDifference, true)} different <= tolerance of +-{FormatBytes (toleranceInBytes)}). Expected app size: {FormatBytes (expectedAppBundleSize)}, actual app size: {FormatBytes (appBundleSize)}."; + } else { + msg = $"App size changed significantly ({FormatBytes (appSizeDifference, true)} different > tolerance of +-{FormatBytes (toleranceInBytes)}). Expected app size: {FormatBytes (expectedAppBundleSize)}, actual app size: {FormatBytes (appBundleSize)}."; + } + + var updated = false; + if (forceUpdate || (update && !withinTolerance)) { Directory.CreateDirectory (expectedDirectory); File.WriteAllText (expectedSizeReportPath, report.ToString ()); + msg += " Check the modified files for more information."; + updated = true; + } else if (!withinTolerance) { + msg += " Set the environment variable WRITE_KNOWN_FAILURES=1, run the test again, and verify the modified files for more information."; } - msg += " Set the environment variable WRITE_KNOWN_FAILURES=1, run the test again, and verify the modified files for more information."; Console.WriteLine ($" {msg}"); var expectedLines = expectedSizeReport.SplitLines ().Skip (2).Where (v => v.IndexOf (':') >= 0).ToDictionary (v => v [..v.IndexOf (':')], v => v [(v.IndexOf (':') + 1)..]); @@ -101,8 +122,10 @@ void Run (ApplePlatform platform, string runtimeIdentifiers, string configuratio foreach (var key in allKeys) { if (!expectedLines.TryGetValue (key, out var expectedLine)) { Console.WriteLine ($" File '{key}' was removed from app bundle: {actualLines [key]}"); + Assert.Fail ($"The file '{key}' was removed from the app bundle."); } else if (!actualLines.TryGetValue (key, out var actualLine)) { Console.WriteLine ($" File '{key}' was added to app bundle: {expectedLine}"); + Assert.Fail ($"The file '{key}' was added to the app bundle."); } else if (expectedLine != actualLine) { Console.WriteLine ($" File '{key}' changed in app bundle:"); Console.WriteLine ($" -{expectedLine}"); @@ -110,39 +133,49 @@ void Run (ApplePlatform platform, string runtimeIdentifiers, string configuratio } } - // Create a file with all the APIs that survived the trimmer; this can be useful to determine what is not trimmed away. - // Note that any changes in this list when the test fails might be due to unrelated earlier changes, that didn't trigger the test - // to fail, because the corresponding app size difference was within the tolerance for app size changes. - if (supportsAssemblyInspection) { - var asmDir = Path.Combine (appPath, GetRelativeAssemblyDirectory (platform)); - var preservedAPIs = new List (); - foreach (var dll in Directory.GetFiles (asmDir, "*.dll", SearchOption.AllDirectories)) { - var relativePath = dll [(asmDir.Length + 1)..]; - using var ad = AssemblyDefinition.ReadAssembly (dll, new ReaderParameters { ReadingMode = ReadingMode.Deferred }); - foreach (var member in ad.EnumerateMembers ()) { - preservedAPIs.Add ($"{relativePath}:{((ICustomAttributeProvider) member).AsFullName ()}"); - } + if (!updated && !withinTolerance) + Assert.Fail (msg); + } + + // Create a file with all the APIs that survived the trimmer; this can be useful to determine what is not trimmed away. + // Note that any changes in this list when the test fails might be due to unrelated earlier changes, that didn't trigger the test + // to fail, because the corresponding app size difference was within the tolerance for app size changes. + void AssertAssemblyReport (ApplePlatform platform, string name, string appPath, bool update, string expectedDirectory) + { + var asmDir = Path.Combine (appPath, GetRelativeAssemblyDirectory (platform)); + var preservedAPIs = new List (); + foreach (var dll in Directory.GetFiles (asmDir, "*.dll", SearchOption.AllDirectories)) { + var relativePath = dll [(asmDir.Length + 1)..]; + using var ad = AssemblyDefinition.ReadAssembly (dll, new ReaderParameters { ReadingMode = ReadingMode.Deferred }); + foreach (var member in ad.EnumerateMembers ()) { + preservedAPIs.Add ($"{relativePath}:{((ICustomAttributeProvider) member).AsFullName ()}"); } - preservedAPIs.Sort (); - var expectedFile = Path.Combine (expectedDirectory, $"{name}-preservedapis.txt"); - var expectedAPIs = File.ReadAllLines (expectedFile); - var addedAPIs = preservedAPIs.Except (expectedAPIs); - var removedAPIs = expectedAPIs.Except (preservedAPIs); + } + preservedAPIs.Sort (); + var expectedFile = Path.Combine (expectedDirectory, $"{name}-preservedapis.txt"); + var expectedAPIs = File.ReadAllLines (expectedFile); + var addedAPIs = preservedAPIs.Except (expectedAPIs).ToList (); + var removedAPIs = expectedAPIs.Except (preservedAPIs).ToList (); + if (addedAPIs.Count () > 0) { Console.WriteLine ($" {addedAPIs.Count ()} additional APIs present:"); foreach (var line in addedAPIs) Console.WriteLine ($" {line}"); + } + if (removedAPIs.Count () > 0) { Console.WriteLine ($" {removedAPIs.Count ()} APIs not present anymore:"); foreach (var line in removedAPIs) Console.WriteLine ($" {line}"); + } - if (update) { - File.WriteAllLines (expectedFile, preservedAPIs); - } + if (update) { + File.WriteAllLines (expectedFile, preservedAPIs); } - if (!update) - Assert.Fail (msg); + if (!update) { + Assert.That (addedAPIs, Is.Empty, "No added APIs (set the environment variable WRITE_KNOWN_FAILURES=1 and run the test again to update the expected set of APIs)"); + Assert.That (removedAPIs, Is.Empty, "No removed APIs (set the environment variable WRITE_KNOWN_FAILURES=1 and run the test again to update the expected set of APIs)"); + } } static string FormatBytes (long bytes, bool alwaysShowSign = false) diff --git a/tests/dotnet/UnitTests/expected/iOS-MonoVM-interpreter-preservedapis.txt b/tests/dotnet/UnitTests/expected/iOS-MonoVM-interpreter-preservedapis.txt index ac862060822..6c440448d1d 100644 --- a/tests/dotnet/UnitTests/expected/iOS-MonoVM-interpreter-preservedapis.txt +++ b/tests/dotnet/UnitTests/expected/iOS-MonoVM-interpreter-preservedapis.txt @@ -655,6 +655,7 @@ Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(ObjCRuntime.NativeHandle, Syst Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean) Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, System.Boolean, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean) Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, System.Type, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean, System.Boolean, out System.Boolean&) +Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr) Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.Boolean) Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.IntPtr, System.RuntimeMethodHandle, System.Boolean, System.Boolean) Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.IntPtr, System.RuntimeMethodHandle, System.Boolean) diff --git a/tests/dotnet/UnitTests/expected/iOS-MonoVM-preservedapis.txt b/tests/dotnet/UnitTests/expected/iOS-MonoVM-preservedapis.txt index f7662dd6d02..013a74ad687 100644 --- a/tests/dotnet/UnitTests/expected/iOS-MonoVM-preservedapis.txt +++ b/tests/dotnet/UnitTests/expected/iOS-MonoVM-preservedapis.txt @@ -497,6 +497,7 @@ Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(ObjCRuntime.NativeHandle, Syst Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean) Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, System.Boolean, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean) Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr, System.Type, ObjCRuntime.Runtime/MissingCtorResolution, System.Boolean, System.Boolean, out System.Boolean&) +Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject(System.IntPtr) Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.Boolean) Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.IntPtr, System.RuntimeMethodHandle, System.Boolean, System.Boolean) Microsoft.iOS.dll:ObjCRuntime.Runtime.GetNSObject`1(System.IntPtr, System.IntPtr, System.RuntimeMethodHandle, System.Boolean)