Move Go Replace Detector to Prod (#1272)

* move goreplace from experiments to prod * json version * go replace detector * require string --------- Co-authored-by: Amitla Vannikumar <[email protected]>
microsoft · Oct 10, 2024 · 950576a · 950576a
1 parent ed4488a
commit 950576a
Show file tree

Hide file tree

Showing 7 changed files with 590 additions and 1,605 deletions.
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -46,7 +46,7 @@
     <PackageVersion Include="System.Memory" Version="4.5.5" />
     <PackageVersion Include="System.Reactive" Version="6.0.1" />
     <PackageVersion Include="System.Runtime.Loader" Version="4.3.0" />
-    <PackageVersion Include="System.Text.Json" Version="8.0.4" />
+    <PackageVersion Include="System.Text.Json" Version="8.0.5" />
     <PackageVersion Include="System.Threading.Tasks.Dataflow" Version="8.0.0" />
     <PackageVersion Include="Tomlyn.Signed" Version="0.17.0" />
     <PackageVersion Include="yamldotnet" Version="15.1.6" />

diff --git a/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs
@@ -19,6 +19,8 @@ namespace Microsoft.ComponentDetection.Detectors.Go;
 
 public class GoComponentDetector : FileComponentDetector
 {
+    private const string StartString = "require ";
+
     private static readonly Regex GoSumRegex = new(
         @"(?<name>.*)\s+(?<version>.*?)(/go\.mod)?\s+(?<hash>.*)",
         RegexOptions.Compiled | RegexOptions.ExplicitCapture | RegexOptions.IgnoreCase);
@@ -27,19 +29,22 @@ public class GoComponentDetector : FileComponentDetector
 
     private readonly ICommandLineInvocationService commandLineInvocationService;
     private readonly IEnvironmentVariableService envVarService;
+    private readonly IFileUtilityService fileUtilityService;
 
     public GoComponentDetector(
         IComponentStreamEnumerableFactory componentStreamEnumerableFactory,
         IObservableDirectoryWalkerFactory walkerFactory,
         ICommandLineInvocationService commandLineInvocationService,
         IEnvironmentVariableService envVarService,
-        ILogger<GoComponentDetector> logger)
+        ILogger<GoComponentDetector> logger,
+        IFileUtilityService fileUtilityService)
     {
         this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory;
         this.Scanner = walkerFactory;
         this.commandLineInvocationService = commandLineInvocationService;
         this.envVarService = envVarService;
         this.Logger = logger;
+        this.fileUtilityService = fileUtilityService;
     }
 
     public override string Id => "Go";
@@ -50,7 +55,7 @@ public GoComponentDetector(
 
     public override IEnumerable<ComponentType> SupportedComponentTypes { get; } = [ComponentType.Go];
 
-    public override int Version => 7;
+    public override int Version => 8;
 
     protected override Task<IObservable<ProcessRequest>> OnPrepareDetectionAsync(
         IObservable<ProcessRequest> processRequests,
@@ -246,7 +251,7 @@ private async Task<bool> UseGoCliToScanAsync(string location, ISingleFileCompone
             return false;
         }
 
-        this.RecordBuildDependencies(goDependenciesProcess.StdOut, singleFileComponentRecorder);
+        this.RecordBuildDependencies(goDependenciesProcess.StdOut, singleFileComponentRecorder, projectRootDirectory.FullName);
 
         var generateGraphProcess = await this.commandLineInvocationService.ExecuteCommandAsync("go", null, workingDirectory: projectRootDirectory, new List<string> { "mod", "graph" }.ToArray());
         if (generateGraphProcess.ExitCode == 0)
@@ -299,9 +304,9 @@ private async Task ParseGoModFileAsync(
 
                 // In go >= 1.17, direct dependencies are listed as "require x/y v1.2.3", and transitive dependencies
                 // are listed in the require () section
-                if (line.StartsWith("require "))
+                if (line.StartsWith(StartString))
                 {
-                    this.TryRegisterDependencyFromModLine(line[8..], singleFileComponentRecorder);
+                    this.TryRegisterDependencyFromModLine(line[StartString.Length..], singleFileComponentRecorder);
                 }
 
                 line = await reader.ReadLineAsync();
@@ -421,14 +426,16 @@ private bool IsModuleInBuildList(ISingleFileComponentRecorder singleFileComponen
         return singleFileComponentRecorder.GetComponent(component.Id) != null;
     }
 
-    private void RecordBuildDependencies(string goListOutput, ISingleFileComponentRecorder singleFileComponentRecorder)
+    private void RecordBuildDependencies(string goListOutput, ISingleFileComponentRecorder singleFileComponentRecorder, string projectRootDirectoryFullName)
     {
         var goBuildModules = new List<GoBuildModule>();
         var reader = new JsonTextReader(new StringReader(goListOutput))
         {
             SupportMultipleContent = true,
         };
 
+        using var record = new GoReplaceTelemetryRecord();
+
         while (reader.Read())
         {
             var serializer = new JsonSerializer();
@@ -439,13 +446,45 @@ private void RecordBuildDependencies(string goListOutput, ISingleFileComponentRe
 
         foreach (var dependency in goBuildModules)
         {
+            var dependencyName = $"{dependency.Path} {dependency.Version}";
+
             if (dependency.Main)
             {
                 // main is the entry point module (superfluous as we already have the file location)
                 continue;
             }
 
-            var goComponent = new GoComponent(dependency.Path, dependency.Version);
+            if (dependency.Replace?.Path != null && dependency.Replace.Version == null)
+            {
+                var dirName = projectRootDirectoryFullName;
+                var combinedPath = Path.Combine(dirName, dependency.Replace.Path, "go.mod");
+                var goModFilePath = Path.GetFullPath(combinedPath);
+                if (this.fileUtilityService.Exists(goModFilePath))
+                {
+                    this.Logger.LogInformation("go Module {GoModule} is being replaced with module at path {GoModFilePath}", dependencyName, goModFilePath);
+                    record.GoModPathAndVersion = dependencyName;
+                    record.GoModReplacement = goModFilePath;
+                    continue;
+                }
+
+                this.Logger.LogWarning("go.mod file {GoModFilePath} does not exist in the relative path given for replacement", goModFilePath);
+                record.GoModPathAndVersion = goModFilePath;
+                record.GoModReplacement = null;
+            }
+
+            GoComponent goComponent;
+            if (dependency.Replace?.Path != null && dependency.Replace.Version != null)
+            {
+                var dependencyReplacementName = $"{dependency.Replace.Path} {dependency.Replace.Version}";
+                goComponent = new GoComponent(dependency.Replace.Path, dependency.Replace.Version);
+                this.Logger.LogInformation("go Module {GoModule} being replaced with module {GoModuleReplacement}", dependencyName, dependencyReplacementName);
+                record.GoModPathAndVersion = dependencyName;
+                record.GoModReplacement = dependencyReplacementName;
+            }
+            else
+            {
+                goComponent = new GoComponent(dependency.Path, dependency.Version);
+            }
 
             if (dependency.Indirect)
             {
@@ -485,5 +524,7 @@ private class GoBuildModule
         public string Version { get; set; }
 
         public bool Indirect { get; set; }
+
+        public GoBuildModule Replace { get; set; }
     }
 }