From 606e88359664541c4adb71b45512514443eea711 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 13 Jan 2025 16:16:23 +1100 Subject: [PATCH] Add a UV package manager (#327) * Add a UV package manager * The command is `uv pip install` * Update logging * Add noisy logs * Fiddling with debug * revert test results * Use a different test output helper for this test suite * Add debug traces * Extra debug output * Add an extra debug flag * Refactor into util class for process spawning * Run UV with the VIRTUAL_ENV environment variable * Run UV in verbose mode * Propagate uv cache variables * Document UV support * Refactor the process util to it's rightful place --- .github/workflows/dotnet-ci.yml | 1 + docs/environments.md | 20 ++++++ docs/index.md | 2 +- .../PackageManagement/PipInstaller.cs | 55 ++++------------ .../PackageManagement/UVInstaller.cs | 64 +++++++++++++++++++ src/CSnakes.Runtime/ProcessUtils.cs | 51 +++++++++++++++ .../ServiceCollectionExtensions.cs | 18 ++++++ src/Directory.Packages.props | 1 + src/RedistributablePython.Tests/BasicTests.cs | 8 ++- .../RedistributablePython.Tests.csproj | 2 +- .../RedistributablePythonTestBase.cs | 16 ++++- ...{test_simple.py => test_redist_imports.py} | 0 12 files changed, 189 insertions(+), 49 deletions(-) create mode 100644 src/CSnakes.Runtime/PackageManagement/UVInstaller.cs rename src/RedistributablePython.Tests/python/{test_simple.py => test_redist_imports.py} (100%) diff --git a/.github/workflows/dotnet-ci.yml b/.github/workflows/dotnet-ci.yml index 0e34aff7..73f2aea0 100644 --- a/.github/workflows/dotnet-ci.yml +++ b/.github/workflows/dotnet-ci.yml @@ -69,6 +69,7 @@ jobs: working-directory: src env: PYTHON_VERSION: ${{ steps.installpython.outputs.python-version }} + UV_NO_CACHE: 1 - name: Upload logs uses: actions/upload-artifact@v4 diff --git a/docs/environments.md b/docs/environments.md index 5c415388..16631370 100644 --- a/docs/environments.md +++ b/docs/environments.md @@ -56,3 +56,23 @@ services `.WithPipInstaller()` takes an optional argument that specifies the path to the `requirements.txt` file. If you don't specify a path, it will look for a `requirements.txt` file in the virtual environment directory. +## Installing dependencies with `uv` + +`uv` is an alternative to pip that can also install requirements from a file like `requirements.txt` or `pyproject.toml`. UV has a major benefit in a 10-100x speedup over pip, so your CSnakes applications will be faster to start. + +To use uv to install packages: + +```csharp +... +services + .WithPython() + .WithVirtualEnvironment(Path.Join(home, ".venv")) + .WithUvInstaller("requirements.txt"); // Optional - give the name of the requirements file, or pyproject.toml +``` + +Some other important notes about this implementation. + +- Only uses uv to install packages and does not use uv to create projects or virtual environments. +- Must be used with `WithVirtualEnvironment()`, as pip requires a virtual environment to install packages into. +- Will use the `UV_CACHE_DIR` environment variable to cache the packages in a directory if set. +- Will disable the cache if the `UV_NO_CACHE` environment variable is set. diff --git a/docs/index.md b/docs/index.md index 09614af0..5779b171 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,7 +25,7 @@ Check out the [getting started](getting-started.md) guide or check out the [demo - Supports [nested sequence and mapping types (`tuple`, `dict`, `list`)](reference.md) - Supports [default values](reference.md#default-values) - Supports [Hot Reload](advanced.md#hot-reload) of Python code in Visual Studio and supported IDEs - +- Supports [UV](environments.md#uv) for fast installation of Python packages and dependencies ## Benefits diff --git a/src/CSnakes.Runtime/PackageManagement/PipInstaller.cs b/src/CSnakes.Runtime/PackageManagement/PipInstaller.cs index 927b8462..5a013c26 100644 --- a/src/CSnakes.Runtime/PackageManagement/PipInstaller.cs +++ b/src/CSnakes.Runtime/PackageManagement/PipInstaller.cs @@ -1,6 +1,5 @@ using CSnakes.Runtime.EnvironmentManagement; using Microsoft.Extensions.Logging; -using System.Diagnostics; using System.Runtime.InteropServices; namespace CSnakes.Runtime.PackageManagement; @@ -11,12 +10,11 @@ internal class PipInstaller(ILogger logger, string requirementsFil public Task InstallPackages(string home, IEnvironmentManagement? environmentManager) { - // TODO:Allow overriding of the requirements file name. string requirementsPath = Path.GetFullPath(Path.Combine(home, requirementsFileName)); if (File.Exists(requirementsPath)) { logger.LogDebug("File {Requirements} was found.", requirementsPath); - InstallPackagesWithPip(home, environmentManager); + InstallPackagesWithPip(home, environmentManager, $"-r { requirementsFileName}", logger); } else { @@ -26,14 +24,12 @@ public Task InstallPackages(string home, IEnvironmentManagement? environmentMana return Task.CompletedTask; } - private void InstallPackagesWithPip(string home, IEnvironmentManagement? environmentManager) + internal static void InstallPackagesWithPip(string home, IEnvironmentManagement? environmentManager, string requirements, ILogger logger) { - ProcessStartInfo startInfo = new() - { - WorkingDirectory = home, - FileName = pipBinaryName, - Arguments = $"install -r {requirementsFileName} --disable-pip-version-check" - }; + string fileName = pipBinaryName; + string workingDirectory = home; + string path = ""; + string arguments = $"install {requirements} --disable-pip-version-check"; if (environmentManager is not null) { @@ -41,39 +37,16 @@ private void InstallPackagesWithPip(string home, IEnvironmentManagement? environ logger.LogDebug("Using virtual environment at {VirtualEnvironmentLocation} to install packages with pip.", virtualEnvironmentLocation); string venvScriptPath = Path.Combine(virtualEnvironmentLocation, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "Scripts" : "bin"); // TODO: Check that the pip executable exists, and if not, raise an exception with actionable steps. - startInfo.FileName = Path.Combine(venvScriptPath, pipBinaryName); - startInfo.EnvironmentVariables["PATH"] = $"{venvScriptPath};{Environment.GetEnvironmentVariable("PATH")}"; - } - - startInfo.RedirectStandardOutput = true; - startInfo.RedirectStandardError = true; - - using Process process = new() { StartInfo = startInfo }; - process.OutputDataReceived += (sender, e) => - { - if (!string.IsNullOrEmpty(e.Data)) + fileName = Path.Combine(venvScriptPath, pipBinaryName); + path = $"{venvScriptPath};{Environment.GetEnvironmentVariable("PATH")}"; + IReadOnlyDictionary extraEnv = new Dictionary { - logger.LogDebug("{Data}", e.Data); - } - }; - - process.ErrorDataReceived += (sender, e) => - { - if (!string.IsNullOrEmpty(e.Data)) - { - logger.LogWarning("{Data}", e.Data); - } - }; - - process.Start(); - process.BeginErrorReadLine(); - process.BeginOutputReadLine(); - process.WaitForExit(); - - if (process.ExitCode != 0) + { "VIRTUAL_ENV", virtualEnvironmentLocation } + }; + ProcessUtils.ExecuteProcess(fileName, arguments, workingDirectory, path, logger, extraEnv); + } else { - logger.LogError("Failed to install packages."); - throw new InvalidOperationException("Failed to install packages."); + ProcessUtils.ExecuteProcess(fileName, arguments, workingDirectory, path, logger); } } } diff --git a/src/CSnakes.Runtime/PackageManagement/UVInstaller.cs b/src/CSnakes.Runtime/PackageManagement/UVInstaller.cs new file mode 100644 index 00000000..771ad65c --- /dev/null +++ b/src/CSnakes.Runtime/PackageManagement/UVInstaller.cs @@ -0,0 +1,64 @@ +using CSnakes.Runtime.EnvironmentManagement; +using Microsoft.Extensions.Logging; +using System.Runtime.InteropServices; + +namespace CSnakes.Runtime.PackageManagement; + +internal class UVInstaller(ILogger logger, string requirementsFileName) : IPythonPackageInstaller +{ + static readonly string binaryName = $"uv{(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".exe" : "")}"; + + public Task InstallPackages(string home, IEnvironmentManagement? environmentManager) + { + string requirementsPath = Path.GetFullPath(Path.Combine(home, requirementsFileName)); + if (File.Exists(requirementsPath)) + { + logger.LogDebug("File {Requirements} was found.", requirementsPath); + InstallPackagesWithUv(home, environmentManager, $"-r {requirementsFileName} --verbose", logger); + } + else + { + logger.LogWarning("File {Requirements} was not found.", requirementsPath); + } + + return Task.CompletedTask; + } + + static internal void InstallPackagesWithUv(string home, IEnvironmentManagement? environmentManager, string requirements, ILogger logger) + { + string fileName = binaryName; + string workingDirectory = home; + string path = ""; + string arguments = $"pip install {requirements}"; + + if (environmentManager is not null) + { + string virtualEnvironmentLocation = Path.GetFullPath(environmentManager.GetPath()); + logger.LogDebug("Using virtual environment at {VirtualEnvironmentLocation} to install packages with uv.", virtualEnvironmentLocation); + string venvScriptPath = Path.Combine(virtualEnvironmentLocation, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "Scripts" : "bin"); + string uvPath = Path.Combine(venvScriptPath, binaryName); + + // Is UV installed? + if (!File.Exists(uvPath)) + { + // Install it with pip + PipInstaller.InstallPackagesWithPip(home, environmentManager, "uv", logger); + } + + fileName = uvPath; + path = $"{venvScriptPath};{Environment.GetEnvironmentVariable("PATH")}"; + IReadOnlyDictionary extraEnv = new Dictionary + { + { "VIRTUAL_ENV", virtualEnvironmentLocation }, + { "UV_CACHE_DIR", Environment.GetEnvironmentVariable("UV_CACHE_DIR") }, + { "UV_NO_CACHE", Environment.GetEnvironmentVariable("UV_NO_CACHE") } + }; + + ProcessUtils.ExecuteProcess(fileName, arguments, workingDirectory, path, logger, extraEnv); + } else + { + ProcessUtils.ExecuteProcess(fileName, arguments, workingDirectory, path, logger); + } + + } +} diff --git a/src/CSnakes.Runtime/ProcessUtils.cs b/src/CSnakes.Runtime/ProcessUtils.cs index a94e3699..586e8f7d 100644 --- a/src/CSnakes.Runtime/ProcessUtils.cs +++ b/src/CSnakes.Runtime/ProcessUtils.cs @@ -75,4 +75,55 @@ private static (Process proc, string? result, string? errors) ExecuteCommand(ILo process.WaitForExit(); return (process, result, errors); } + internal static void ExecuteProcess(string fileName, string arguments, string workingDirectory, string path, ILogger logger, IReadOnlyDictionary? extraEnv = null) + { + ProcessStartInfo startInfo = new() + { + WorkingDirectory = workingDirectory, + FileName = fileName, + Arguments = arguments + }; + + if (!string.IsNullOrEmpty(path)) + startInfo.EnvironmentVariables["PATH"] = path; + if (extraEnv is not null) + { + foreach (var kvp in extraEnv) + { + if (kvp.Value is not null) + startInfo.EnvironmentVariables[kvp.Key] = kvp.Value; + } + } + startInfo.RedirectStandardOutput = true; + startInfo.RedirectStandardError = true; + logger.LogDebug($"Running {startInfo.FileName} with args {startInfo.Arguments} from {startInfo.WorkingDirectory}"); + + using Process process = new() { StartInfo = startInfo }; + process.OutputDataReceived += (sender, e) => + { + if (!string.IsNullOrEmpty(e.Data)) + { + logger.LogDebug("{Data}", e.Data); + } + }; + + process.ErrorDataReceived += (sender, e) => + { + if (!string.IsNullOrEmpty(e.Data)) + { + logger.LogWarning("{Data}", e.Data); + } + }; + + process.Start(); + process.BeginErrorReadLine(); + process.BeginOutputReadLine(); + process.WaitForExit(); + + if (process.ExitCode != 0) + { + logger.LogError("Failed to install packages."); + throw new InvalidOperationException("Failed to install packages."); + } + } } diff --git a/src/CSnakes.Runtime/ServiceCollectionExtensions.cs b/src/CSnakes.Runtime/ServiceCollectionExtensions.cs index 2de13513..4bd1caf5 100644 --- a/src/CSnakes.Runtime/ServiceCollectionExtensions.cs +++ b/src/CSnakes.Runtime/ServiceCollectionExtensions.cs @@ -209,6 +209,24 @@ public static IPythonEnvironmentBuilder WithPipInstaller(this IPythonEnvironment return builder; } + /// + /// Adds a uv package installer to the service collection. If uv is not installed, it will be installed with pip. + /// + /// The to add the installer to. + /// The path to the requirements file. + /// The modified . + public static IPythonEnvironmentBuilder WithUvInstaller(this IPythonEnvironmentBuilder builder, string requirementsPath = "requirements.txt") + { + builder.Services.AddSingleton( + sp => + { + var logger = sp.GetRequiredService>(); + return new UVInstaller(logger, requirementsPath); + } + ); + return builder; + } + [GeneratedRegex("^(\\d+(\\.\\d+)*)")] private static partial Regex VersionParseExpr(); } diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 458f2469..73d81324 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -12,6 +12,7 @@ --> + diff --git a/src/RedistributablePython.Tests/BasicTests.cs b/src/RedistributablePython.Tests/BasicTests.cs index 906b0fe0..e2460696 100644 --- a/src/RedistributablePython.Tests/BasicTests.cs +++ b/src/RedistributablePython.Tests/BasicTests.cs @@ -1,11 +1,13 @@ +using Xunit.Abstractions; + namespace RedistributablePython.Tests; -public class BasicTests : RedistributablePythonTestBase +public class BasicTests(ITestOutputHelper testOutputHelper) : RedistributablePythonTestBase(testOutputHelper) { [Fact] - public void TestSimpleImport() + public void TestSimpleRedistributableImport() { - var testModule = Env.TestSimple(); + var testModule = Env.TestRedistImports(); Assert.NotNull(testModule); testModule.TestNothing(); } diff --git a/src/RedistributablePython.Tests/RedistributablePython.Tests.csproj b/src/RedistributablePython.Tests/RedistributablePython.Tests.csproj index b48e8744..d3fec7ec 100644 --- a/src/RedistributablePython.Tests/RedistributablePython.Tests.csproj +++ b/src/RedistributablePython.Tests/RedistributablePython.Tests.csproj @@ -9,13 +9,13 @@ + - diff --git a/src/RedistributablePython.Tests/RedistributablePythonTestBase.cs b/src/RedistributablePython.Tests/RedistributablePythonTestBase.cs index aa7bfa4c..0b7875d2 100644 --- a/src/RedistributablePython.Tests/RedistributablePythonTestBase.cs +++ b/src/RedistributablePython.Tests/RedistributablePythonTestBase.cs @@ -1,15 +1,19 @@ +using Meziantou.Extensions.Logging.Xunit; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Xunit.Abstractions; namespace RedistributablePython.Tests; public class RedistributablePythonTestBase : IDisposable { private readonly IPythonEnvironment env; private readonly IHost app; + private readonly ITestOutputHelper _testOutputHelper; - public RedistributablePythonTestBase() + public RedistributablePythonTestBase(ITestOutputHelper testOutputHelper) { + _testOutputHelper = testOutputHelper; string venvPath = Path.Join(Environment.CurrentDirectory, "python", ".venv"); app = Host.CreateDefaultBuilder() .ConfigureServices((context, services) => @@ -18,10 +22,16 @@ public RedistributablePythonTestBase() pb.WithHome(Path.Join(Environment.CurrentDirectory, "python")); pb.FromRedistributable() - .WithPipInstaller() + .WithUvInstaller() .WithVirtualEnvironment(venvPath); - services.AddLogging(builder => builder.AddXUnit()); + services.AddSingleton(new XUnitLoggerProvider(_testOutputHelper, appendScope: true)); + + }) + .ConfigureLogging(builder => + { + builder.SetMinimumLevel(LogLevel.Debug); + builder.AddFilter(_ => true); }) .Build(); diff --git a/src/RedistributablePython.Tests/python/test_simple.py b/src/RedistributablePython.Tests/python/test_redist_imports.py similarity index 100% rename from src/RedistributablePython.Tests/python/test_simple.py rename to src/RedistributablePython.Tests/python/test_redist_imports.py