added generate report only option in the generator

Author: Afroz Mohammed

buildtools/Build.ps1

@@ -125,6 +125,37 @@ try {
     else {
       throw "ERROR: Preview build is missing specific SDK artifacts."
     }
+    
+    # during preview, if there is overrides.xml, use the overrides.xml and generate auto-configuration for any missing operations that don't have errors.
+    # if buildconfig is reset, then overrides.xml will have empty config and a new report.xml will be generated.
+    # the report.xml will have all the operations that were in overrides.xml including new generated operations.
+    # if the report.xml is generated, rename it to overrides.xml so that auto-configured operations are included during cmdlet generation.
+    try {
+      Write-Host "Generating report.xml"
+      # build-report-only generates report.xml only when there is a new operation and
+      # there are no errors during auto-generation of the buildconfig
+      dotnet msbuild ./buildtools/build.proj /t:build-report-only `
+        /p:CleanSdkReferences=false `
+        /p:BreakOnNewOperations=false `
+        /p:Configuration=$Configuration
+    }
+    catch {
+      # don't error out. The same failure would be caught in the later step and reported.
+      Write-Host "failed generating report.xml"
+      Write-Host $_.Exception.Message
+    }
+
+    if (Test-Path 'report.xml') {
+      Write-Host "report.xml generated. renaming report.xml to overrides.xml"
+      if (Test-Path 'overrides.xml') {
+        Remove-Item 'overrides.xml'
+      }
+      Rename-Item -Path 'report.xml' -NewName 'overrides.xml'
+    }
+    else {
+      Write-Host "report.xml not generated"
+    }
+    
 
     dotnet msbuild ./buildtools/build.proj /t:preview-build `
       /p:RunTests=$RunTests `
@@ -159,6 +190,36 @@ try {
       throw "ERROR: Release build is missing specific SDK artifacts."
     }
 
+    # during release/dryrun, if there is overrides.xml, use the overrides.xml and generate auto-configuration for any missing operations that don't have errors.
+    # the report.xml will have all the operations that were in overrides.xml including new generated operations.
+    # if the report.xml is generated, rename it to overrides.xml so that auto-configured operations are included during cmdlet generation.
+    try {
+      Write-Host "Generating report.xml"
+      # build-report-only generates report.xml only when there is a new operation and
+      # there are no errors during auto-generation of the buildconfig
+      dotnet msbuild ./buildtools/build.proj /t:build-report-only `
+        /p:CleanSdkReferences=false `
+        /p:BreakOnNewOperations=false `
+        /p:Configuration=$Configuration
+    }
+    catch {
+      # don't error out. The same failure would be caught in the later step and reported.
+      Write-Host "failed generating report.xml"
+      Write-Host $_.Exception.Message
+    }
+
+    if (Test-Path 'report.xml') {
+      Write-Host "report.xml generated. renaming report.xml to overrides.xml"
+      if (Test-Path 'overrides.xml') {
+        Remove-Item 'overrides.xml'
+      }
+      Rename-Item -Path 'report.xml' -NewName 'overrides.xml'
+    }
+    else {
+      Write-Host "report.xml not generated"
+    }
+    
+    
     if ($RunAsStagingBuild -eq 'true') {
       msbuild ./buildtools/build.proj `
         /t:staging-build `

buildtools/build.proj

@@ -75,6 +75,11 @@
         <Message Text="Generates all cmdlets and then builds AWSPowerShell.NetCore deployment artifacts and powershell help files."/>
     </Target>
 
+    <Target Name="build-report-only" DependsOnTargets="clean-deployment;build-generator">
+        <Message Text="Generates report.xml only when there is a new operation and there are no errors during auto-generation of the buildconfig."/>
+        <Exec Command="dotnet $(GeneratorPath)/$(Generator) -sdk $(SDKAssembliesFolder) -rp $(RootPath) -t cmdlets -cnc true -scg false -bno false -ro true -vn $(PatchNumber)" />
+    </Target>
+
     <Target Name="staging-build" DependsOnTargets="clean-bin;clean-deployment;clean-sdk-references;build-generator;create-cmdlets">
         <Message Text="Generates and then builds all cmdlets and PowerShell deployment artifacts."/>
     </Target>

generator/AWSPSGenerator/CommandLineParser.cs

@@ -251,7 +251,16 @@ class ArgDeclaration
                 HasValue = true,
                 Parse = (arguments, argValue) => arguments.ParsedOptions.VersionNumber = argValue,
                 HelpText = "The four-components version number e.g. 4.0.0.0."
+            },
+            new ArgDeclaration
+            {
+                OptionName = "reportonly",
+                ShortName = "ro",
+                HasValue = true,
+                Parse = (arguments, argValue) => arguments.ParsedOptions.GenerateReportOnly = bool.Parse(argValue),
+                HelpText = "Only generate report.xml if there are operations not in config and no errors."
             }
+
         };
 
         static readonly char[] ArgumentPrefixes = { '-', '/' };

generator/AWSPSGeneratorLib/ConfigModel/XmlReportWriter.cs

@@ -14,7 +14,7 @@ namespace AWSPowerShellGenerator.ServiceConfig
 {
     class XmlReportWriter
     {
-        public static void SerializeReport(string folderPath, IEnumerable<ConfigModel> models)
+        public static void SerializeReport(string folderPath, IEnumerable<ConfigModel> models, bool generateReportOnly)
         {
             if (!Directory.Exists(folderPath))
             {
@@ -39,6 +39,11 @@ public static void SerializeReport(string folderPath, IEnumerable<ConfigModel> m
                         overrides.ContainsKey(configModel.C2jFilename) ||
                         configModel.ServiceOperationsList.Where(op => op.IsAutoConfiguring || op.AnalysisErrors.Any()).Any())
                     .ToArray();
+                bool hasErrors = models.Any(configModel =>
+                    configModel.AnalysisErrors.Any() ||
+                    configModel.ServiceOperationsList.Any(op => op.AnalysisErrors.Any()));
+
+                bool hasNewOperations = models.Any(model => model.ServiceOperationsList.Any(op => op.IsAutoConfiguring));
 
                 var doc = new XDocument();
 
@@ -179,7 +184,19 @@ public static void SerializeReport(string folderPath, IEnumerable<ConfigModel> m
                     }
                 }
 
-                doc.Save(filename);
+                if (!generateReportOnly)
+                {
+                    doc.Save(filename);
+                }
+                else if (hasNewOperations && !hasErrors)
+                {
+                    Console.WriteLine("New operations were auto-configured without errors and saved in report.xml");
+                    doc.Save(filename);
+                }
+                else
+                {
+                    Console.WriteLine($"Skipping saving report: hasNewOperations:{hasNewOperations}, hasErrors: {hasErrors} ");
+                }
             }
             catch (Exception e)
             {

generator/AWSPSGeneratorLib/Generator.cs

@@ -76,7 +76,7 @@ public void Execute(GeneratorOptions options)
 
             if (options.ShouldRunTask(GeneratorTasknames.GenerateCmdlets))
             {
-                Console.WriteLine("Executing task 'GenerateCmdlets'");
+                Console.WriteLine($"Executing task 'GenerateCmdlets' {(options.GenerateReportOnly ? "with reportonly option" : "")}");
 
                 var cmdletGenerator = new CmdletGenerator
                 {

generator/AWSPSGeneratorLib/GeneratorOptions.cs

@@ -37,6 +37,14 @@ public class GeneratorOptions
         /// </summary>
         public bool CreateNewCmdlets { get; set; } = true;
 
+        /// <summary>
+        /// If true, report.xml is generated when there is a new operation and
+        /// there are no errors during auto-generation of the buildconfig
+        /// When this is set, the generator won't throw exception for any analysis errors.
+        /// The cmdlets will not be generated when this is true.
+        /// </summary>
+        public bool GenerateReportOnly { get; set; } = false;
+
         /// <summary>
         /// If true (the default is false), the build will fail if new operations are
         /// present in the SDK. Set to true for release builds when all configurations
@@ -196,6 +204,7 @@ public GeneratorOptions(GeneratorOptions rhs)
             CNNorth1RegionDocsDomain = rhs.CNNorth1RegionDocsDomain;
             Edition = rhs.Edition;
             SkipCmdletGeneration = rhs.SkipCmdletGeneration;
+            GenerateReportOnly = rhs.GenerateReportOnly;
         }
     }
 }

generator/AWSPSGeneratorLib/Generators/CmdletGenerator.cs

@@ -207,7 +207,7 @@ protected override void GenerateHelper()
 
             CheckForServicePrefixDuplication();
 
-            if (!Options.SkipCmdletGeneration)
+            if (!Options.SkipCmdletGeneration || Options.GenerateReportOnly)
             {
                 //We clean and setup service folders, some services share a folder, project and module, so we use distinct
                 foreach (var project in ModelCollection.ConfigModels.Values
@@ -238,7 +238,7 @@ protected override void GenerateHelper()
                     LoadCurrentService(CurrentModel);
                     DocumentationUtils.CacheMemberDocumentationSummary(CurrentServiceNDoc);
 
-                    if (!Options.SkipCmdletGeneration)
+                    if (!Options.SkipCmdletGeneration || Options.GenerateReportOnly)
                     {
                         GenerateClientAndCmdlets();
                         // if the service contains any hand-maintained cmdlets, scan them to update the
@@ -259,7 +259,11 @@ protected override void GenerateHelper()
             var allFoundSdkAssemblies = GenerationSources.SDKFindAssemblyFilenames(SdkAssembliesFolder, Directory.EnumerateFiles);
             VerifyAllAssembliesHaveConfiguration(SdkVersionsUtils.ReadSdkVersionFile(SdkAssembliesFolder), ModelCollection, allFoundSdkAssemblies);
 
-            if (!Options.SkipCmdletGeneration)
+            if (Options.GenerateReportOnly)
+            {
+                WriteConfigurationChanges();
+            }
+            else if (!Options.SkipCmdletGeneration)
             {
                 ScanAdvancedCmdletsForCommonModule();
 
@@ -291,6 +295,15 @@ protected override void GenerateHelper()
 
             foreach (var configModel in ModelCollection.ConfigModels.Values)
             {
+                // if Options.GenerateReportOnly, clear AnalysisErrors at model level and ServiceOperations Level
+                if (Options.GenerateReportOnly)
+                {
+                    configModel.AnalysisErrors.Clear();
+                    foreach (var operation in configModel.ServiceOperationsList)
+                    {
+                        operation.AnalysisErrors.Clear();
+                    }
+                }
                 foreach (var error in configModel.AnalysisErrors.Concat(
                       configModel.ServiceOperationsList
                           .OrderBy(so => so.MethodName)
@@ -360,7 +373,7 @@ public static void VerifyAllAssembliesHaveConfiguration(JObject sdkVersionsJson,
 
         private void WriteConfigurationChanges()
         {
-            XmlReportWriter.SerializeReport(Options.RootPath, ModelCollection.ConfigModels.Values);
+            XmlReportWriter.SerializeReport(Options.RootPath, ModelCollection.ConfigModels.Values, Options.GenerateReportOnly);
         }
 
         private void CheckForServicePrefixDuplication()
@@ -445,29 +458,32 @@ private void GenerateClientAndCmdlets()
 
             var outputRoot = Path.Combine(CmdletsOutputPath, CurrentModel.AssemblyName);
 
-            try
+            if (!Options.GenerateReportOnly)
             {
-                using (var sw = new StringWriter())
+                try
                 {
-                    // if the service has operations requiring anonymous access, we'll generate two clients
-                    // one for regular authenticated calls and one using anonymous credentials
-                    using (var writer = new IndentedTextWriter(sw))
+                    using (var sw = new StringWriter())
                     {
-                        CmdletServiceClientWriter.Write(writer,
-                                                        CurrentModel,
-                                                        CurrentModel.ServiceName,
-                                                        GetServiceVersion(CurrentModel.ServiceNamespace, CurrentModel.ServiceClient),
-                                                        awsSignerAttributeTypeValue);
-                    }
+                        // if the service has operations requiring anonymous access, we'll generate two clients
+                        // one for regular authenticated calls and one using anonymous credentials
+                        using (var writer = new IndentedTextWriter(sw))
+                        {
+                            CmdletServiceClientWriter.Write(writer,
+                                CurrentModel,
+                                CurrentModel.ServiceName,
+                                GetServiceVersion(CurrentModel.ServiceNamespace, CurrentModel.ServiceClient),
+                                awsSignerAttributeTypeValue);
+                        }
 
-                    var fileContents = sw.ToString();
-                    var fileName = CurrentModel.GetServiceCmdletClassName(false) + "Cmdlet.cs";
-                    File.WriteAllText(Path.Combine(outputRoot, fileName), fileContents);
+                        var fileContents = sw.ToString();
+                        var fileName = CurrentModel.GetServiceCmdletClassName(false) + "Cmdlet.cs";
+                        File.WriteAllText(Path.Combine(outputRoot, fileName), fileContents);
+                    }
+                }
+                catch (Exception e)
+                {
+                    AnalysisError.ExceptionWhileWritingServiceClientCode(CurrentModel, e);
                 }
-            }
-            catch (Exception e)
-            {
-                AnalysisError.ExceptionWhileWritingServiceClientCode(CurrentModel, e);
             }
 
             // process the methods in order to make debugging more convenient
@@ -631,7 +647,7 @@ private void CreateCmdlet(MethodInfo method, ConfigModel configModel, string aws
                 AnalysisError.ExceptionWhileAnalyzingSDKLibrary(CurrentModel, CurrentOperation, e);
             }
 
-            if (serviceOperation.AnalysisErrors.Count == 0)
+            if (serviceOperation.AnalysisErrors.Count == 0 && !Options.GenerateReportOnly)
             {
                 // set file name and location
                 var filePath = Path.Combine(configModel.AssemblyName, GeneratedCmdletsFoldername, $"{serviceOperation.SelectedVerb}-{serviceOperation.SelectedNoun}-Cmdlet.cs");