Add a UV package manager (#327)

* Add a UV package manager

* The command is `uv pip install`

* Update logging

* Add noisy logs

* Fiddling with debug

* revert test results

* Use a different test output helper for this test suite

* Add debug traces

* Extra debug output

* Add an extra debug flag

* Refactor into util class for process spawning

* Run UV with the VIRTUAL_ENV environment variable

* Run UV in verbose mode

* Propagate uv cache variables

* Document UV support

* Refactor the process util to it's rightful place

.github/workflows/dotnet-ci.yml

@@ -69,6 +69,7 @@ jobs:
         working-directory: src
         env:
           PYTHON_VERSION: ${{ steps.installpython.outputs.python-version }}
+          UV_NO_CACHE: 1
 
       - name: Upload logs
         uses: actions/upload-artifact@v4

docs/environments.md

@@ -56,3 +56,23 @@ services
 
 `.WithPipInstaller()` takes an optional argument that specifies the path to the `requirements.txt` file. If you don't specify a path, it will look for a `requirements.txt` file in the virtual environment directory.
 
+## Installing dependencies with `uv`
+
+`uv` is an alternative to pip that can also install requirements from a file like `requirements.txt` or `pyproject.toml`. UV has a major benefit in a 10-100x speedup over pip, so your CSnakes applications will be faster to start.
+
+To use uv to install packages:
+
+```csharp
+...
+services
+    .WithPython()
+    .WithVirtualEnvironment(Path.Join(home, ".venv"))
+    .WithUvInstaller("requirements.txt"); // Optional - give the name of the requirements file, or pyproject.toml
+```
+
+Some other important notes about this implementation.
+
+- Only uses uv to install packages and does not use uv to create projects or virtual environments.
+- Must be used with `WithVirtualEnvironment()`, as pip requires a virtual environment to install packages into.
+- Will use the `UV_CACHE_DIR` environment variable to cache the packages in a directory if set.
+- Will disable the cache if the `UV_NO_CACHE` environment variable is set.

docs/index.md

@@ -25,7 +25,7 @@ Check out the [getting started](getting-started.md) guide or check out the [demo
 - Supports [nested sequence and mapping types (`tuple`, `dict`, `list`)](reference.md)
 - Supports [default values](reference.md#default-values)
 - Supports [Hot Reload](advanced.md#hot-reload) of Python code in Visual Studio and supported IDEs
-
+- Supports [UV](environments.md#uv) for fast installation of Python packages and dependencies
 
 ## Benefits
 

src/CSnakes.Runtime/PackageManagement/PipInstaller.cs

@@ -1,6 +1,5 @@
 using CSnakes.Runtime.EnvironmentManagement;
 using Microsoft.Extensions.Logging;
-using System.Diagnostics;
 using System.Runtime.InteropServices;
 
 namespace CSnakes.Runtime.PackageManagement;
@@ -11,12 +10,11 @@ internal class PipInstaller(ILogger<PipInstaller> logger, string requirementsFil
 
     public Task InstallPackages(string home, IEnvironmentManagement? environmentManager)
     {
-        // TODO:Allow overriding of the requirements file name.
         string requirementsPath = Path.GetFullPath(Path.Combine(home, requirementsFileName));
         if (File.Exists(requirementsPath))
         {
             logger.LogDebug("File {Requirements} was found.", requirementsPath);
-            InstallPackagesWithPip(home, environmentManager);
+            InstallPackagesWithPip(home, environmentManager, $"-r { requirementsFileName}", logger);
         }
         else
         {
@@ -26,54 +24,29 @@ public Task InstallPackages(string home, IEnvironmentManagement? environmentMana
         return Task.CompletedTask;
     }
 
-    private void InstallPackagesWithPip(string home, IEnvironmentManagement? environmentManager)
+    internal static void InstallPackagesWithPip(string home, IEnvironmentManagement? environmentManager, string requirements, ILogger logger)
     {
-        ProcessStartInfo startInfo = new()
-        {
-            WorkingDirectory = home,
-            FileName = pipBinaryName,
-            Arguments = $"install -r {requirementsFileName} --disable-pip-version-check"
-        };
+        string fileName = pipBinaryName;
+        string workingDirectory = home;
+        string path = "";
+        string arguments = $"install {requirements} --disable-pip-version-check";
 
         if (environmentManager is not null)
         {
             string virtualEnvironmentLocation = Path.GetFullPath(environmentManager.GetPath());
             logger.LogDebug("Using virtual environment at {VirtualEnvironmentLocation} to install packages with pip.", virtualEnvironmentLocation);
             string venvScriptPath = Path.Combine(virtualEnvironmentLocation, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "Scripts" : "bin");
             // TODO: Check that the pip executable exists, and if not, raise an exception with actionable steps.
-            startInfo.FileName = Path.Combine(venvScriptPath, pipBinaryName);
-            startInfo.EnvironmentVariables["PATH"] = $"{venvScriptPath};{Environment.GetEnvironmentVariable("PATH")}";
-        }
-
-        startInfo.RedirectStandardOutput = true;
-        startInfo.RedirectStandardError = true;
-
-        using Process process = new() { StartInfo = startInfo };
-        process.OutputDataReceived += (sender, e) =>
-        {
-            if (!string.IsNullOrEmpty(e.Data))
+            fileName = Path.Combine(venvScriptPath, pipBinaryName);
+            path = $"{venvScriptPath};{Environment.GetEnvironmentVariable("PATH")}";
+            IReadOnlyDictionary<string, string> extraEnv = new Dictionary<string, string>
             {
-                logger.LogDebug("{Data}", e.Data);
-            }
-        };
-
-        process.ErrorDataReceived += (sender, e) =>
-        {
-            if (!string.IsNullOrEmpty(e.Data))
-            {
-                logger.LogWarning("{Data}", e.Data);
-            }
-        };
-
-        process.Start();
-        process.BeginErrorReadLine();
-        process.BeginOutputReadLine();
-        process.WaitForExit();
-
-        if (process.ExitCode != 0)
+                { "VIRTUAL_ENV", virtualEnvironmentLocation }
+            };
+            ProcessUtils.ExecuteProcess(fileName, arguments, workingDirectory, path, logger, extraEnv);
+        } else
         {
-            logger.LogError("Failed to install packages.");
-            throw new InvalidOperationException("Failed to install packages.");
+            ProcessUtils.ExecuteProcess(fileName, arguments, workingDirectory, path, logger);
         }
     }
 }

src/CSnakes.Runtime/PackageManagement/UVInstaller.cs

@@ -0,0 +1,64 @@
+using CSnakes.Runtime.EnvironmentManagement;
+using Microsoft.Extensions.Logging;
+using System.Runtime.InteropServices;
+
+namespace CSnakes.Runtime.PackageManagement;
+
+internal class UVInstaller(ILogger<UVInstaller> logger, string requirementsFileName) : IPythonPackageInstaller
+{
+    static readonly string binaryName = $"uv{(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".exe" : "")}";
+
+    public Task InstallPackages(string home, IEnvironmentManagement? environmentManager)
+    {
+        string requirementsPath = Path.GetFullPath(Path.Combine(home, requirementsFileName));
+        if (File.Exists(requirementsPath))
+        {
+            logger.LogDebug("File {Requirements} was found.", requirementsPath);
+            InstallPackagesWithUv(home, environmentManager, $"-r {requirementsFileName} --verbose", logger);
+        }
+        else
+        {
+            logger.LogWarning("File {Requirements} was not found.", requirementsPath);
+        }
+
+        return Task.CompletedTask;
+    }
+
+    static internal void InstallPackagesWithUv(string home, IEnvironmentManagement? environmentManager, string requirements, ILogger logger)
+    {
+        string fileName = binaryName;
+        string workingDirectory = home;
+        string path = "";
+        string arguments = $"pip install {requirements}";
+
+        if (environmentManager is not null)
+        {
+            string virtualEnvironmentLocation = Path.GetFullPath(environmentManager.GetPath());
+            logger.LogDebug("Using virtual environment at {VirtualEnvironmentLocation} to install packages with uv.", virtualEnvironmentLocation);
+            string venvScriptPath = Path.Combine(virtualEnvironmentLocation, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "Scripts" : "bin");
+            string uvPath = Path.Combine(venvScriptPath, binaryName);
+
+            // Is UV installed?
+            if (!File.Exists(uvPath))
+            {
+                // Install it with pip
+                PipInstaller.InstallPackagesWithPip(home, environmentManager, "uv", logger);
+            }
+
+            fileName = uvPath;
+            path = $"{venvScriptPath};{Environment.GetEnvironmentVariable("PATH")}";
+            IReadOnlyDictionary<string, string?> extraEnv = new Dictionary<string, string?>
+            {
+                { "VIRTUAL_ENV", virtualEnvironmentLocation },
+                { "UV_CACHE_DIR", Environment.GetEnvironmentVariable("UV_CACHE_DIR") },
+                { "UV_NO_CACHE", Environment.GetEnvironmentVariable("UV_NO_CACHE") }
+            };
+
+            ProcessUtils.ExecuteProcess(fileName, arguments, workingDirectory, path, logger, extraEnv);
+        } else
+        {
+            ProcessUtils.ExecuteProcess(fileName, arguments, workingDirectory, path, logger);
+        }
+
+    }
+}

src/CSnakes.Runtime/ProcessUtils.cs

@@ -75,4 +75,55 @@ private static (Process proc, string? result, string? errors) ExecuteCommand(ILo
         process.WaitForExit();
         return (process, result, errors);
     }
+    internal static void ExecuteProcess(string fileName, string arguments, string workingDirectory, string path, ILogger logger, IReadOnlyDictionary<string, string?>? extraEnv = null)
+    {
+        ProcessStartInfo startInfo = new()
+        {
+            WorkingDirectory = workingDirectory,
+            FileName = fileName,
+            Arguments = arguments
+        };
+
+        if (!string.IsNullOrEmpty(path))
+            startInfo.EnvironmentVariables["PATH"] = path;
+        if (extraEnv is not null)
+        {
+            foreach (var kvp in extraEnv)
+            {
+                if (kvp.Value is not null)
+                    startInfo.EnvironmentVariables[kvp.Key] = kvp.Value;
+            }
+        }
+        startInfo.RedirectStandardOutput = true;
+        startInfo.RedirectStandardError = true;
+        logger.LogDebug($"Running {startInfo.FileName} with args {startInfo.Arguments} from {startInfo.WorkingDirectory}");
+
+        using Process process = new() { StartInfo = startInfo };
+        process.OutputDataReceived += (sender, e) =>
+        {
+            if (!string.IsNullOrEmpty(e.Data))
+            {
+                logger.LogDebug("{Data}", e.Data);
+            }
+        };
+
+        process.ErrorDataReceived += (sender, e) =>
+        {
+            if (!string.IsNullOrEmpty(e.Data))
+            {
+                logger.LogWarning("{Data}", e.Data);
+            }
+        };
+
+        process.Start();
+        process.BeginErrorReadLine();
+        process.BeginOutputReadLine();
+        process.WaitForExit();
+
+        if (process.ExitCode != 0)
+        {
+            logger.LogError("Failed to install packages.");
+            throw new InvalidOperationException("Failed to install packages.");
+        }
+    }
 }

src/CSnakes.Runtime/ServiceCollectionExtensions.cs

@@ -209,6 +209,24 @@ public static IPythonEnvironmentBuilder WithPipInstaller(this IPythonEnvironment
         return builder;
     }
 
+    /// <summary>
+    /// Adds a uv package installer to the service collection. If uv is not installed, it will be installed with pip.
+    /// </summary>
+    /// <param name="builder">The <see cref="IPythonEnvironmentBuilder"/> to add the installer to.</param>
+    /// <param name="requirementsPath">The path to the requirements file.</param>
+    /// <returns>The modified <see cref="IPythonEnvironmentBuilder"/>.</returns>
+    public static IPythonEnvironmentBuilder WithUvInstaller(this IPythonEnvironmentBuilder builder, string requirementsPath = "requirements.txt")
+    {
+        builder.Services.AddSingleton<IPythonPackageInstaller>(
+            sp =>
+            {
+                var logger = sp.GetRequiredService<ILogger<UVInstaller>>();
+                return new UVInstaller(logger, requirementsPath);
+            }
+        );
+        return builder;
+    }
+
     [GeneratedRegex("^(\\d+(\\.\\d+)*)")]
     private static partial Regex VersionParseExpr();
 }

src/Directory.Packages.props

@@ -12,6 +12,7 @@
    -->
   <ItemGroup>
     <PackageVersion Include="CommunityToolkit.HighPerformance" Version="8.3.0" />
+    <PackageVersion Include="Meziantou.Extensions.Logging.Xunit" Version="1.0.8" />
     <PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="3.3.4" />
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.9.2" />
     <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />

src/RedistributablePython.Tests/BasicTests.cs

@@ -1,11 +1,13 @@
+using Xunit.Abstractions;
+
 namespace RedistributablePython.Tests;
 
-public class BasicTests : RedistributablePythonTestBase
+public class BasicTests(ITestOutputHelper testOutputHelper) : RedistributablePythonTestBase(testOutputHelper)
 {
     [Fact]
-    public void TestSimpleImport()
+    public void TestSimpleRedistributableImport()
     {
-        var testModule = Env.TestSimple();
+        var testModule = Env.TestRedistImports();
         Assert.NotNull(testModule);
         testModule.TestNothing();
     }

src/RedistributablePython.Tests/RedistributablePython.Tests.csproj

@@ -9,13 +9,13 @@
 
   <ItemGroup>
     <PackageReference Include="coverlet.collector" />
+    <PackageReference Include="Meziantou.Extensions.Logging.Xunit" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="System.Net.Http" />
     <PackageReference Include="System.Text.RegularExpressions" />
     <PackageReference Include="xunit" />
     <PackageReference Include="xunit.runner.visualstudio" />
     <PackageReference Include="python" />
-    <PackageReference Include="MartinCostello.Logging.XUnit" />
   </ItemGroup>
 
   <ItemGroup>

src/RedistributablePython.Tests/RedistributablePythonTestBase.cs

@@ -1,15 +1,19 @@
+using Meziantou.Extensions.Logging.Xunit;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
+using Xunit.Abstractions;
 
 namespace RedistributablePython.Tests;
 public class RedistributablePythonTestBase : IDisposable
 {
     private readonly IPythonEnvironment env;
     private readonly IHost app;
+    private readonly ITestOutputHelper _testOutputHelper;
 
-    public RedistributablePythonTestBase()
+    public RedistributablePythonTestBase(ITestOutputHelper testOutputHelper)
     {
+        _testOutputHelper = testOutputHelper;
         string venvPath = Path.Join(Environment.CurrentDirectory, "python", ".venv");
         app = Host.CreateDefaultBuilder()
             .ConfigureServices((context, services) =>
@@ -18,10 +22,16 @@ public RedistributablePythonTestBase()
                 pb.WithHome(Path.Join(Environment.CurrentDirectory, "python"));
 
                 pb.FromRedistributable()
-                  .WithPipInstaller()
+                  .WithUvInstaller()
                   .WithVirtualEnvironment(venvPath);
 
-                services.AddLogging(builder => builder.AddXUnit());
+                services.AddSingleton<ILoggerProvider>(new XUnitLoggerProvider(_testOutputHelper, appendScope: true));
+
+            })
+            .ConfigureLogging(builder =>
+            {
+                builder.SetMinimumLevel(LogLevel.Debug);
+                builder.AddFilter(_ => true);
             })
             .Build();