From 2998436fd7bf2e1a8197a2e3dee351e50b0b54d0 Mon Sep 17 00:00:00 2001 From: Katherine Fleming <2205659+kflemin@users.noreply.github.com> Date: Mon, 13 Apr 2026 15:35:43 -0600 Subject: [PATCH 01/11] new uv python management --- example_files/python_deps/.python-version | 1 + example_files/python_deps/dependencies.json | 9 - example_files/python_deps/install_python.ps1 | 96 ---- example_files/python_deps/install_python.sh | 136 ----- example_files/python_deps/pyproject.toml | 28 + lib/uo_cli.rb | 507 +++++-------------- spec/uo_cli_spec.rb | 27 +- 7 files changed, 147 insertions(+), 657 deletions(-) create mode 100644 example_files/python_deps/.python-version delete mode 100644 example_files/python_deps/dependencies.json delete mode 100644 example_files/python_deps/install_python.ps1 delete mode 100755 example_files/python_deps/install_python.sh create mode 100644 example_files/python_deps/pyproject.toml diff --git a/example_files/python_deps/.python-version b/example_files/python_deps/.python-version new file mode 100644 index 000000000..7c7a975f4 --- /dev/null +++ b/example_files/python_deps/.python-version @@ -0,0 +1 @@ +3.10 \ No newline at end of file diff --git a/example_files/python_deps/dependencies.json b/example_files/python_deps/dependencies.json deleted file mode 100644 index 056815632..000000000 --- a/example_files/python_deps/dependencies.json +++ /dev/null @@ -1,9 +0,0 @@ -[ - { "name": "ThermalNetwork", "version": "0.5.0"}, - { "name": "git+https://github.com/urbanopt/urbanopt-ditto-reader.git@numpy-update", "version": null}, - { "name": "NREL-disco", "version": "0.5.1"}, - { "name": "urbanopt-des", "version": "0.2.0"}, - { "name": "Shapely", "version": "1.8.5"}, - { "name": "urban-system-generator", "version": "0.1.1"}, - { "name": "numpy", "version": "2.2.6"} -] diff --git a/example_files/python_deps/install_python.ps1 b/example_files/python_deps/install_python.ps1 deleted file mode 100644 index c648b1039..000000000 --- a/example_files/python_deps/install_python.ps1 +++ /dev/null @@ -1,96 +0,0 @@ -param ( - [Parameter(Mandatory=$true)][string]$conda_version, - [Parameter(Mandatory=$true)][string]$python_version, - [Parameter()][string]$install_path = "." -) - -function Invoke-WebRequestExitOnError { - param([string]$url, [string]$filename) - Write-Debug "Invoke-WebRequest $url" - - try { - Invoke-WebRequest -OutFile $filename $url - } - catch { - throw "failed to download $url" - } -} - - -function Get-Python { - param([string]$conda_base_url, [string]$filename, [string]$python_version, [string]$install_path) - $path = Join-Path "." "${filename}" - $url = "https://repo.anaconda.com/miniconda/${filename}" - if (($FORCE_DOWNLOAD -eq 1) -and (Test-Path $path)) { - Remove-Item $path - } - if (!(Test-Path $path)) { - Invoke-WebRequestExitOnError $url $path - } - - $full_path = Resolve-Path $install_path - $dst = Join-Path $full_path "python-$python_version" - $cmd_args = "/InstallationType=JustMe /AddToPath=0 RegisterPython=0 /S /D=${dst}" - $result = Start-Process -FilePath ${path} -NoNewWindow -PassThru -Wait -ArgumentList $cmd_args - if ($FORCE_DOWNLOAD -eq 1) { - # This delay exists because we've observed cases where deleting the file fails because - # Windows says it is still in use. This seems to fix the issue. - Start-Sleep -Seconds 5 - Remove-Item $path - } - if ($result.ExitCode -ne 0) { - $msg = "Failed to run Python installer: ExitCode=${result.ExitCode}" - Write-Error $msg - exit $result.ExitCode - } -} - - -### MAIN ### -# -# Example usage: -# .\install_python.ps1 4.12.0 3.9 -# -# Anaconda recommends only running this distribution in an Anaconda shell. -# pip will fail with SSL errors in a non-Anaconda PowerShell. -# Anaconda says to workaround the issue by setting this environment variable: -# $env:CONDA_DLL_SEARCH_MODIFICATION_ENABLE = 1 -# Refer to https://github.com/conda/conda/issues/8273 -# To test the install run these commands: -# .\python-3.9\python --version -# .\python-3.9\Scripts\pip list -# -# To enable debug prints run this in the shell: -# $DebugPreference="Continue" -# -# To prevent re-download of the Python package set the environment variable -# FORCE_DOWNLOAD to 0. -# -# If you get the error "running scripts is disabled on this system" then follow -# the provided link or run the command below to change the security policy for -# the current shell: -# Set-ExecutionPolicy -ExecutionPolicy Unrestricted -Scope Process - -$python_version_fields = $python_version -split "\." -if ($python_version_fields.length -ne 2) { - $msg = "failed to run [$command]: ExitCode={0}" -f $result.ExitCode - Write-Error "Python version must be major.minor, such as '3.9'" - exit 1 -} -$python_major_minor = -join $python_version_fields[0..1] -$conda_base_url = "https://repo.anaconda.com/miniconda/" -$conda_package_name = "Miniconda3-py${python_major_minor}_${conda_version}-Windows-x86_64.exe" - -if (Test-Path env:FORCE_DOWNLOAD) { - $FORCE_DOWNLOAD = $env:FORCE_DOWNLOAD -} else { - $FORCE_DOWNLOAD = 1 -} -Write-Debug "FORCE_DOWNLOAD=${FORCE_DOWNLOAD}" - -if (!(Test-Path $install_path)) { - mkdir $install_path -} - -$ErrorActionPreference = "Stop" -Get-Python $conda_base_url $conda_package_name $python_version $install_path diff --git a/example_files/python_deps/install_python.sh b/example_files/python_deps/install_python.sh deleted file mode 100755 index 6fd056dae..000000000 --- a/example_files/python_deps/install_python.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/bash - -# Installs Python via Miniconda for Linux and MacOS. - -function log_message -{ - # Severity is first argument. - echo ${@:2} - echo "$(date) - $1 - ${@:2}" >> $LOG_FILE -} - -function debug -{ - if [ $VERBOSE -eq 1 ]; then - log_message "DEBUG" $@ - fi -} - -function info -{ - log_message "INFO" $@ -} - -function error -{ - log_message "ERROR" $@ -} - -function run_command -{ - debug "run command [$@]" - $@ > /dev/null - ret=$? - if [ $ret != 0 ]; then - error "command=[$@] failed return_code=$ret" - exit $ret - fi -} - -function show_help -{ - echo "Usage: $0 MINICONDA_VERSION PYTHON_VERSION PATH" -} - -### MAIN ### - -LOG_FILE="/tmp/install_python.log" -> $LOG_FILE - -if [ -z $VERBOSE ]; then - VERBOSE=0 -fi - -# Developers can set this to 0 to prevent repeated downloads. -# Normal operation is download the file every run and delete it afterwards. -if [ -z $FORCE_DOWNLOAD ]; then - FORCE_DOWNLOAD=1 -fi - -if [ -z $3 ]; then - show_help - exit 1 -fi - -CONDA_VERSION=$1 -PYTHON_FULL_VERSION=$2 -IFS="." read -ra VER_ARRAY <<< "$PYTHON_FULL_VERSION" -if [ ${#VER_ARRAY[@]} -lt 2 ] ; then - error "invalid python version: format x.y.z" - exit 1 -fi -PYTHON_MAJOR_MINOR="${VER_ARRAY[0]}${VER_ARRAY[1]}" -INSTALL_BASE=$3 - -if [ ! -d $INSTALL_BASE ]; then - error "path $INSTALL_BASE does not exist" - exit 1 -fi - -architecture=$(uname -m) - -echo "$architecture" - -# Handle multiple chip architectures (ARM & x86) as well as OS types (Linux & MacOS) -if [[ $architecture == "x86"* || $architecture == "i686" || $architecture == "i386" ]]; then - if [[ "$OSTYPE" == "linux-gnu" ]]; then - PLATFORM=Linux-x86_64 - elif [[ "$OSTYPE" == "darwin"* ]]; then - PLATFORM=MacOSX-x86_64 - else - error "unknown OS type $OSTYPE" - exit 1 - fi -elif [[ $architecture == "arm"* || $architecture == "aarch"* ]]; then - if [[ "$OSTYPE" == "linux-gnu" ]]; then - PLATFORM=Linux-aarch64 - elif [[ "$OSTYPE" == "darwin"* ]]; then - PLATFORM=MacOSX-arm64 - else - error "unknown OS type $OSTYPE" - exit 1 - fi -fi - -CONDA_PACKAGE_NAME=Miniconda3-py${PYTHON_MAJOR_MINOR}_${CONDA_VERSION}-${PLATFORM}.sh -CONDA_URL=https://repo.anaconda.com/miniconda/$CONDA_PACKAGE_NAME -CONDA_PACKAGE_PATH=/tmp/$CONDA_PACKAGE_NAME - -INSTALL_PATH=./Miniconda-${CONDA_VERSION} -PIP=$INSTALL_PATH/bin/pip -PYTHON=$INSTALL_PATH/bin/python - -debug "PYTHON_FULL_VERSION=$PYTHON_FULL_VERSION" -debug "PYTHON_MAJOR_MINOR=$PYTHON_MAJOR_MINOR" -debug "CONDA_VERSION=$CONDA_VERSION" -debug "CONDA_PACKAGE_NAME=$CONDA_PACKAGE_NAME" -debug "CONDA_URL=$CONDA_URL" -debug "CONDA_PACKAGE_PATH=$CONDA_PACKAGE_PATH" -debug "INSTALL_PATH=$INSTALL_PATH" -debug "PIP=$PIP" - -if [ $FORCE_DOWNLOAD -eq 1 ] && [ -f $CONDA_PACKAGE_PATH ]; then - run_command "rm -f $CONDA_PACKAGE_PATH" -fi - -if [ ! -f $CONDA_PACKAGE_PATH ]; then - run_command "curl $CONDA_URL -o $CONDA_PACKAGE_PATH" - debug "Finished downloading $CONDA_PACKAGE_NAME" -fi - -run_command "bash $CONDA_PACKAGE_PATH -b -p $INSTALL_PATH -u" -if [ $FORCE_DOWNLOAD -eq 1 ]; then - run_command "rm -rf $CONDA_PACKAGE_PATH" -fi - -debug "Finished installation of Python $PYTHON_FULL_VERSION" diff --git a/example_files/python_deps/pyproject.toml b/example_files/python_deps/pyproject.toml new file mode 100644 index 000000000..995c4452d --- /dev/null +++ b/example_files/python_deps/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "urbanopt" +version = "1.2.0" +description = "URBANopt python dependency management" +readme = "README.md" +requires-python = "==3.10.19" +dependencies = [] + +[dependency-groups] +disco = [ + "NREL-disco==0.5.1", +] +ditto-reader = [ + "urbanopt-ditto-reader==0.6.4" +] +thermalnetwork = [ + "thermalnetwork==0.5.0", +] +urbanopt-des = [ + "geojson-modelica-translator==0.13.0" +] +usg = [ + "urban-system-generator==0.1.1" +] + +# Note: dependency-groups are kept here as a version manifest. +# The CLI uses UV_TOOL_PACKAGES in uo_cli.rb to map groups to PyPI packages +# for use with `uv tool run --from `. diff --git a/lib/uo_cli.rb b/lib/uo_cli.rb index 78b9bd7e6..c401f58d8 100755 --- a/lib/uo_cli.rb +++ b/lib/uo_cli.rb @@ -24,7 +24,7 @@ module CLI class UrbanOptCLI COMMAND_MAP = { 'create' => 'Make new things - project directory or files', - 'install_python' => 'Install python and other dependencies to run OpenDSS, DISCO, GMT analysis', + 'install_python' => 'Install Python dependencies for OpenDSS, DISCO, DES, GHE, and USG tools (requires uv)', 'update' => 'Update files in an existing URBANopt project', 'run' => 'Use files in your directory to simulate district energy use', 'process' => 'Post-process URBANopt simulations for additional insights', @@ -160,6 +160,7 @@ def opt_create def opt_install_python @subopts = Optimist.options do banner "\nURBANopt install_python:\n \n" + banner "Uses uv to sync all Python dependency groups defined in pyproject.toml\n" opt :verbose, "\Verbose output \n" \ 'Example: uo install_python --verbose' @@ -1051,298 +1052,82 @@ def self.update_project(existing_project_folder, new_project_directory) end end - # Setup Python Variables for DiTTo and DISCO - def self.setup_python_variables - pvars = { - python_version: '3.10', - miniconda_version: '24.9.2-0', - python_install_path: nil, - python_path: nil, - pip_path: nil, - des_output_path: nil, - disco_path: nil, - ditto_path: nil, - ghe_path: nil, - gmt_path: nil, - usg_path: nil - } - - # get location - $LOAD_PATH.each do |path_item| - if path_item.to_s.end_with?('example_files') - # install python in cli gem's example_files/python_deps folder - # so it is accessible to all projects - pvars[:python_install_path] = File.join(path_item, 'python_deps') - pvars[:pip_path] = pvars[:python_install_path] - break - end - end - # look for config file and grab info - if File.exist? File.join(pvars[:python_install_path], 'python_config.json') - configs = JSON.parse(File.read(File.join(pvars[:python_install_path], 'python_config.json')), symbolize_names: true) - pvars[:python_path] = configs[:python_path] - pvars[:pip_path] = configs[:pip_path] - pvars[:des_output_path] = configs[:des_output_path] - pvars[:disco_path] = configs[:disco_path] - pvars[:ditto_path] = configs[:ditto_path] - pvars[:ghe_path] = configs[:ghe_path] - pvars[:gmt_path] = configs[:gmt_path] - pvars[:usg_path] = configs[:usg_path] - end - return pvars - end - - # Return UO python packages list from python_deps/dependencies.json - def self.get_python_deps - deps = [] - the_path = '' - $LOAD_PATH.each do |path_item| - if path_item.to_s.end_with?('example_files') - # install python in cli gem's example_files/python_deps folder - # so it is accessible to all projects - the_path = File.join(path_item, 'python_deps') - break - end - end - - if File.exist? File.join(the_path, 'dependencies.json') - deps = JSON.parse(File.read(File.join(the_path, 'dependencies.json')), symbolize_names: true) + # Mapping from dependency group name to PyPI package spec. + # These must match the [dependency-groups] entries in python_deps/pyproject.toml. + # Note: urbanopt-des uses geojson-modelica-translator directly because that + # package provides the `uo_des` CLI entry point. + UV_TOOL_PACKAGES = { + 'disco' => 'NREL-disco==0.5.1', + 'ditto-reader' => 'urbanopt-ditto-reader==0.6.4', + 'thermalnetwork' => 'thermalnetwork==0.5.0', + 'urbanopt-des' => 'geojson-modelica-translator==0.13.0', + 'usg' => 'urban-system-generator==0.1.1' + }.freeze + + # Python version constraint (must match pyproject.toml requires-python) + UV_PYTHON_VERSION = '3.10'.freeze + + # Recommended uv version and install instructions (update version here when changing) + UV_RECOMMENDED_VERSION = '0.11.6'.freeze + UV_INSTALL_URL = 'https://docs.astral.sh/uv/getting-started/installation/'.freeze + UV_INSTALL_MESSAGE = "\nERROR: uv is not installed or not on your PATH.\n" \ + "Please install uv (recommended version #{UV_RECOMMENDED_VERSION} or later): #{UV_INSTALL_URL}\n".freeze + + # Check that uv is available on the system PATH + def self.check_uv + puts 'Checking for uv...' + stdout, stderr, status = Open3.capture3('uv --version') + if status.success? + puts "...uv found: #{stdout.strip}" + return true + else + puts UV_INSTALL_MESSAGE + return false end - return deps end - # Check Python - def self.check_python(python_only: false) - results = { python: false, pvars: [], message: [], python_deps: false, result: false } - puts 'Checking system.....' - pvars = setup_python_variables - results[:pvars] = pvars - - # check vars - if pvars[:python_path].nil? || pvars[:pip_path].nil? - # need to install - results[:message] << 'Python paths have not yet been initialized with URBANopt.' - puts results[:message] - return results - end - - # check python - stdout, stderr, status = Open3.capture3("#{pvars[:python_path]} -V") - if stderr.empty? - puts "...python found at #{pvars[:python_path]}" - else - results[:message] << "ERROR installing python: #{stderr}" - puts results[:message] - return results - end + # Check for uv and abort if not found + def self.require_uv + abort(UV_INSTALL_MESSAGE) unless check_uv + end - # check pip - stdout, stderr, status = Open3.capture3("#{pvars[:pip_path]} -V") - if stderr.empty? - puts "...pip found at #{pvars[:pip_path]}" + # Run a Python tool via `uv tool run --from `. + # Each tool runs in an isolated ephemeral environment — no shared lockfile needed. + # +group+:: dependency group name (key in UV_TOOL_PACKAGES, e.g. 'ditto-reader') + # +command+:: the CLI command and arguments to run (e.g. 'ditto_reader_cli run-opendss ...') + # +use_system+:: if true, use system() for interactive output; if false, use Open3.capture3 + def self.run_uv_tool(group, command, use_system: true) + package = UV_TOOL_PACKAGES[group] + abort("\nERROR: Unknown tool group '#{group}'") if package.nil? + + full_command = "uv tool run --python #{UV_PYTHON_VERSION} --from \"#{package}\" #{command}" + puts "Running: #{full_command}" + if use_system + system(full_command) else - results[:message] << "ERROR finding pip: #{stderr}" - puts results[:message] - return results - end - - # python and pip installed correctly - results[:python] = true - - # now check dependencies (if python_only is false) - unless python_only - deps = get_python_deps - puts "DEPENDENCIES RETRIEVED FROM FILE: #{deps}" - errors = [] - deps.each do |dep| - puts "Checking for Python package: #{dep[:name]} (version: #{dep[:version]})" - # TODO: Update when there is a stable release for DISCO - if dep[:name].to_s.include? 'disco' - stdout, stderr, status = Open3.capture3("#{pvars[:pip_path]} show NREL-disco") - else - stdout, stderr, status = Open3.capture3("#{pvars[:pip_path]} show #{dep[:name]}") - end - if @opthash.subopts[:verbose] - puts dep[:name] - puts "stdout: #{stdout}" - puts "status: #{status}" - end - - if stderr.empty? - # check versions - m = stdout.match(/^Version: (\S{3,}$)/) - err = true - if m && m.size > 1 - if !dep[:version].nil? && dep[:version].to_s == m[1].to_s - puts "...#{dep[:name]} found with specified version #{dep[:version]}" - err = false - elsif dep[:version].nil? - err = false - puts "...#{dep[:name]} found (version #{m[1]})" - end - else - results[:message] << "could not determine version for #{dep[:name]}" - puts results[:message] - errors << stderr - end - if err - results[:message] << "incorrect version found for #{dep[:name]}...expecting version #{dep[:version]}" - puts results[:message] - errors << stderr - end - else - # ignore warnings - unless stderr.include? 'WARNING:' - results[:message] << stderr - puts results[:message] - errors << stderr - end - end - end - if errors.empty? - results[:python_deps] = true - end + stdout, stderr, status = Open3.capture3(full_command) + return stdout, stderr, status end - - # all is good if messages are empty - if results[:message].empty? - results[:result] = true - end - - return results end - # Install Python and Related Dependencies + # Install all Python tool dependencies (pre-caches each tool's environment) def self.install_python_dependencies - pvars = setup_python_variables - - # check if python and dependencies are already installed - results = check_python - - # install python if not installed - if !results[:python] - - # cd into script dir - wd = Dir.getwd - FileUtils.cd(pvars[:python_install_path]) - puts "Installing Python #{pvars[:python_version]}..." - if (/cygwin|mswin|mingw|bccwin|wince|emx/ =~ RUBY_PLATFORM).nil? - # not windows - script = File.join(pvars[:python_install_path], 'install_python.sh') - the_command = "cd #{pvars[:python_install_path]}; #{script} #{pvars[:miniconda_version]} #{pvars[:python_version]} #{pvars[:python_install_path]}" - stdout, stderr, status = Open3.capture3(the_command) - if (stderr && !stderr == '') || (stdout && stdout.include?('Usage')) - # error - puts "ERROR installing python dependencies: #{stderr}, #{stdout}" - return - end - # capture paths - mac_path_base = File.join(pvars[:python_install_path], "Miniconda-#{pvars[:miniconda_version]}") - pvars[:python_path] = File.join(mac_path_base, 'bin', 'python') - pvars[:pip_path] = File.join(mac_path_base, 'bin', 'pip') - pvars[:des_output_path] = File.join(mac_path_base, 'bin', 'des-output') - pvars[:disco_path] = File.join(mac_path_base, 'bin', 'disco') - pvars[:ditto_path] = File.join(mac_path_base, 'bin', 'ditto_reader_cli') - pvars[:ghe_path] = File.join(mac_path_base, 'bin', 'thermalnetwork') - pvars[:gmt_path] = File.join(mac_path_base, 'bin', 'uo_des') - pvars[:usg_path] = File.join(mac_path_base, 'bin', 'usg') - configs = { - python_path: pvars[:python_path], - pip_path: pvars[:pip_path], - des_output_path: pvars[:des_output_path], - disco_path: pvars[:disco_path], - ditto_path: pvars[:ditto_path], - ghe_path: pvars[:ghe_path], - gmt_path: pvars[:gmt_path], - usg_path: pvars[:usg_path] - } + errors = [] + UV_TOOL_PACKAGES.each do |group, package| + puts "Installing '#{group}' (#{package})..." + stdout, stderr, status = Open3.capture3("uv tool install --python #{UV_PYTHON_VERSION} \"#{package}\"") + if status.success? + puts "...#{group} installed successfully" else - # windows - script = File.join(pvars[:python_install_path], 'install_python.ps1') - - command_list = [ - 'powershell Set-ExecutionPolicy -ExecutionPolicy Unrestricted -Scope Process', - "powershell #{script} #{pvars[:miniconda_version]} #{pvars[:python_version]} #{pvars[:python_install_path]}", - 'powershell $env:CONDA_DLL_SEARCH_MODIFICATION_ENABLE = 1' - ] - - command_list.each do |command| - stdout, stderr, status = Open3.capture3(command) - if !stderr.empty? - puts "ERROR installing python dependencies: #{stderr}, #{stdout}" - break - end - end - - # capture paths - windows_path_base = File.join(pvars[:python_install_path], "python-#{pvars[:python_version]}") - pvars[:python_path] = File.join(windows_path_base, 'python.exe') - pvars[:pip_path] = File.join(windows_path_base, 'Scripts', 'pip.exe') - pvars[:des_output_path] = File.join(windows_path_base, 'Scripts', 'des-output.exe') - pvars[:disco_path] = File.join(windows_path_base, 'Scripts', 'disco.exe') - pvars[:ditto_path] = File.join(windows_path_base, 'Scripts', 'ditto_reader_cli.exe') - pvars[:ghe_path] = File.join(windows_path_base, 'Scripts', 'thermalnetwork.exe') - pvars[:gmt_path] = File.join(windows_path_base, 'Scripts', 'uo_des.exe') - pvars[:usg_path] = File.join(windows_path_base, 'Scripts', 'usg.exe') - - configs = { - python_path: pvars[:python_path], - pip_path: pvars[:pip_path], - des_output_path: pvars[:des_output_path], - disco_path: pvars[:disco_path], - ditto_path: pvars[:ditto_path], - ghe_path: pvars[:ghe_path], - gmt_path: pvars[:gmt_path], - usg_path: pvars[:usg_path] - } - end - - # get back to wd - FileUtils.cd(wd) - - # write config file - File.open(File.join(pvars[:python_install_path], 'python_config.json'), 'w') do |f| - f.write(JSON.pretty_generate(configs)) - end - end - - # install python dependencies if not installed - if !results[:python_deps] - deps = get_python_deps - deps.each do |dep| - puts "Installing #{dep[:name]} #{dep[:version]}" - the_command = '' - if dep[:version].nil? - the_command = "#{pvars[:pip_path]} install #{dep[:name]}" - else - the_command = "#{pvars[:pip_path]} install #{dep[:name]}==#{dep[:version]}" - end - - if @opthash.subopts[:verbose] - puts "INSTALL COMMAND: #{the_command}" - end - stdout, stderr, status = Open3.capture3(the_command) - if @opthash.subopts[:verbose] - puts "status: #{status}" - puts "stdout: #{stdout}" - end - if !stderr.empty? - puts "Error installing: #{stderr}" - end + puts "ERROR installing #{group}: #{stderr}" + errors << "#{group}: #{stderr}" end end - # double check python and dependencies have been installed now - if !results[:result] - # double check that everything has succeeded now - results = check_python - end - - if results[:result] - puts "Python and dependencies successfully installed in #{pvars[:python_install_path]}" + if errors.empty? + puts "\nAll Python tools successfully installed" else - # errors occurred - puts "Errors occurred when installing python and dependencies: #{results[:message]}" + puts "\nErrors occurred when installing tools:\n#{errors.join("\n")}" end end @@ -1437,6 +1222,7 @@ def self.install_python_dependencies # Install python and other dependencies if @opthash.command == 'install_python' puts "\nInstalling python and dependencies" + require_uv install_python_dependencies puts "\nDone\n" end @@ -1458,12 +1244,8 @@ def self.install_python_dependencies # Run OpenDSS simulation if @opthash.command == 'opendss' - # first check python - res = check_python - if res[:python] == false - puts "\nPython error: #{res[:message]}" - abort("\nPython dependencies are needed to run this workflow. Install with the CLI command: uo install_python \n") - end + # check that uv is available + require_uv # If a config file is supplied, use the data specified there. if @opthash.subopts[:config] @@ -1515,9 +1297,9 @@ def self.install_python_dependencies puts "\nERROR: #{e.message}" end - ditto_cli_root = "#{res[:pvars][:ditto_path]} run-opendss " + ditto_cli_addition = 'run-opendss ' if @opthash.subopts[:config] - ditto_cli_addition = "--config #{@opthash.subopts[:config]}" + ditto_cli_addition += "--config #{@opthash.subopts[:config]}" elsif @opthash.subopts[:scenario] && @opthash.subopts[:feature] ditto_cli_addition = "--scenario_file #{@opthash.subopts[:scenario]} --feature_file #{@opthash.subopts[:feature]}" if @opthash.subopts[:equipment] @@ -1551,8 +1333,7 @@ def self.install_python_dependencies abort("\nCommand must include ScenarioFile & FeatureFile, or a config file that specifies both. Please try again") end begin - puts "COMMAND: #{ditto_cli_root + ditto_cli_addition}" - system(ditto_cli_root + ditto_cli_addition) + run_uv_tool('ditto-reader', "ditto_reader_cli run-opendss #{ditto_cli_addition}") rescue FileNotFoundError abort("\nMust post-process results before running OpenDSS. We recommend 'process --default'." \ "Once OpenDSS is run, you may then 'process --opendss'") @@ -1564,14 +1345,8 @@ def self.install_python_dependencies # Run DISCO Simulation if @opthash.command == 'disco' - # first check python and python dependencies - res = check_python - if res[:result] == false - puts "\nPython error: #{res[:message]}" - abort("\nPython dependencies are needed to run this workflow. Install with the CLI command: uo install_python \n") - else - disco_path = res[:pvars][:disco_path] - end + # check that uv is available + require_uv # disco folder disco_folder = File.join(@root_dir, 'disco') @@ -1602,30 +1377,14 @@ def self.install_python_dependencies # call disco FileUtils.cd(run_folder) do - if (/cygwin|mswin|mingw|bccwin|wince|emx/ =~ RUBY_PLATFORM).nil? - # not windows - if Dir.exist?(File.join(run_folder, 'disco')) - # if disco results folder exists overwrite folder - commands = ["#{disco_path} upgrade-cost-analysis run config.json -o disco --console-log-level=warn --force"] - else - commands = ["#{disco_path} upgrade-cost-analysis run config.json -o disco --console-log-level=warn"] - end - else - # windows - if Dir.exist?(File.join(run_folder, 'disco')) - # if disco results folder exists overwrite folder) - commands = ['powershell $env:CONDA_DLL_SEARCH_MODIFICATION_ENABLE = 1', "#{disco_path} upgrade-cost-analysis run config.json -o disco --console-log-level=warn --force"] - else - commands = ['powershell $env:CONDA_DLL_SEARCH_MODIFICATION_ENABLE = 1', "#{disco_path} upgrade-cost-analysis run config.json -o disco --console-log-level=warn"] - end + disco_args = "upgrade-cost-analysis run config.json -o disco --console-log-level=warn" + if Dir.exist?(File.join(run_folder, 'disco')) + disco_args += ' --force' end puts 'Running DISCO...' - commands.each do |command| - # TODO: This will be updated so stderr only reports error/warnings at DISCO level - stdout, stderr, status = Open3.capture3(command) - if !stderr.empty? - puts "ERROR running DISCO: #{stderr}" - end + stdout, stderr, status = run_uv_tool('disco', "disco #{disco_args}", use_system: false) + if !stderr.empty? + puts "ERROR running DISCO: #{stderr}" end puts "Refer to detailed log file #{File.join(run_folder, 'disco', 'run_upgrade_cost_analysis.log')} for more information on the run." puts "Refer to the output summary file #{File.join(run_folder, 'disco', 'output_summary.json')} for a summary of the results." @@ -2173,16 +1932,12 @@ def self.install_python_dependencies if @opthash.command == 'des_params' - # first check python - res = check_python - if res[:python] == false - puts "\nPython error: #{res[:message]}" - abort("\nPython dependencies are needed to run this workflow. Install with the CLI command: uo install_python \n") - end + # check that uv is available + require_uv - des_cli_root = "#{res[:pvars][:gmt_path]} build-sys-param" + des_cli_addition = 'build-sys-param' if @opthash.subopts[:sys_param] - des_cli_addition = " #{@opthash.subopts[:sys_param]}" + des_cli_addition += " #{@opthash.subopts[:sys_param]}" if @opthash.subopts[:scenario] des_cli_addition += " #{@opthash.subopts[:scenario]}" end @@ -2210,7 +1965,7 @@ def self.install_python_dependencies abort("\nCommand must include new system parameter file name, ScenarioFile, & FeatureFile. Please try again") end begin - system(des_cli_root + des_cli_addition) + run_uv_tool('urbanopt-des', "uo_des #{des_cli_addition}") rescue FileNotFoundError abort("\nMust simulate using 'uo run' before preparing Modelica models.") rescue StandardError => e @@ -2220,16 +1975,12 @@ def self.install_python_dependencies if @opthash.command == 'des_create' - # first check python - res = check_python - if res[:python] == false - puts "\nPython error: #{res[:message]}" - abort("\nPython dependencies are needed to run this workflow. Install with the CLI command: uo install_python \n") - end + # check that uv is available + require_uv - des_cli_root = "#{res[:pvars][:gmt_path]} create-model" + des_cli_addition = 'create-model' if @opthash.subopts[:sys_param] - des_cli_addition = " #{@opthash.subopts[:sys_param]}" + des_cli_addition += " #{@opthash.subopts[:sys_param]}" if @opthash.subopts[:feature] des_cli_addition += " #{@opthash.subopts[:feature]}" end @@ -2244,7 +1995,7 @@ def self.install_python_dependencies abort("\nCommand must include system parameter file name and FeatureFile. Please try again") end begin - system(des_cli_root + des_cli_addition) + run_uv_tool('urbanopt-des', "uo_des #{des_cli_addition}") rescue FileNotFoundError abort("\nMust simulate using 'uo run' before preparing Modelica models.") rescue StandardError => e @@ -2254,16 +2005,12 @@ def self.install_python_dependencies if @opthash.command == 'des_run' - # first check python - res = check_python - if res[:python] == false - puts "\nPython error: #{res[:message]}" - abort("\nPython dependencies are needed to run this workflow. Install with the CLI command: uo install_python \n") - end + # check that uv is available + require_uv - des_cli_root = "#{res[:pvars][:gmt_path]} run-model" + des_cli_addition = 'run-model' if @opthash.subopts[:model] - des_cli_addition = " #{File.expand_path(@opthash.subopts[:model])}" + des_cli_addition += " #{File.expand_path(@opthash.subopts[:model])}" if @opthash.subopts[:start_time] des_cli_addition += " -a #{@opthash.subopts[:start_time]}" end @@ -2281,7 +2028,7 @@ def self.install_python_dependencies end begin - system(des_cli_root + des_cli_addition) + run_uv_tool('urbanopt-des', "uo_des #{des_cli_addition}") rescue FileNotFoundError abort("\nMust simulate using 'uo run' before preparing Modelica models.") rescue StandardError => e @@ -2290,20 +2037,17 @@ def self.install_python_dependencies end if @opthash.command == 'des_process' - # first check python - res = check_python - if res[:python] == false - puts "\nPython error: #{res[:message]}" - abort("\nPython dependencies are needed to run this workflow. Install with the CLI command: uo install_python \n") - end - des_cli_root = "#{res[:pvars][:gmt_path]} process-model" + # check that uv is available + require_uv + + des_cli_addition = 'process-model' if @opthash.subopts[:model] - des_cli_addition = " #{@opthash.subopts[:model]}" + des_cli_addition += " #{@opthash.subopts[:model]}" else abort("\nCommand must include Modelica model name. Please try again") end begin - system(des_cli_root + des_cli_addition) + run_uv_tool('urbanopt-des', "uo_des #{des_cli_addition}") rescue FileNotFoundError abort("\nMust simulate using 'uo run' before preparing Modelica models.") rescue StandardError => e @@ -2313,14 +2057,10 @@ def self.install_python_dependencies if @opthash.command == 'ghe_size' - # first check python - res = check_python - if res[:python] == false - puts "\nPython error: #{res[:message]}" - abort("\nPython dependencies are needed to run this workflow. Install with the CLI command: uo install_python \n") - end + # check that uv is available + require_uv - ghe_cli_root = res[:pvars][:ghe_path].to_s + ghe_cli_addition = '' if @opthash.subopts[:sys_param] ghe_cli_addition = " -y #{@opthash.subopts[:sys_param]}" @@ -2346,13 +2086,8 @@ def self.install_python_dependencies else abort("\nCommand must include ScenarioFile & FeatureFile. Please try again") end - # if @opthash.subopts[:verbose] - # puts "ghe_cli_root: #{ghe_cli_root}" - # puts "ghe_cli_addition: #{ghe_cli_addition}" - # puts "command: #{ghe_cli_root + ghe_cli_addition}" - # end begin - system(ghe_cli_root + ghe_cli_addition) + run_uv_tool('thermalnetwork', "thermalnetwork#{ghe_cli_addition}") rescue FileNotFoundError abort("\nFile Not Found Error Holder.") rescue StandardError => e @@ -2364,14 +2099,9 @@ def self.install_python_dependencies if @opthash.command == 'usg_preprocess' # Use the USG CLI to preprocess USG inputs. The output file will be automatically be named the same as the Geojson file + .csv - # first check python - res = check_python - if res[:python] == false - puts "\nPython error: #{res[:message]}" - abort("\nPython dependencies are needed to run this workflow. Install with the CLI command: uo install_python \n") - end + # check that uv is available + require_uv - usg_cli_root = "#{res[:pvars][:usg_path].to_s} geojson2csv" usg_cli_addition = '' if @opthash.subopts[:feature] @@ -2379,8 +2109,7 @@ def self.install_python_dependencies end begin - puts "\nRunning system command: #{usg_cli_root + usg_cli_addition}\n" - system(usg_cli_root + usg_cli_addition) + run_uv_tool('usg', "usg geojson2csv#{usg_cli_addition}") rescue FileNotFoundError abort("\nFeature File #{@opthash.subopts[:feature]} not Found. Please check the file path and try again.") rescue StandardError => e @@ -2392,15 +2121,10 @@ def self.install_python_dependencies if @opthash.command == 'usg_complete' # Use the USG CLI to complete USG simulations. The input file will be the same as the Geojson file + .csv - # first check python - res = check_python - if res[:python] == false - puts "\nPython error: #{res[:message]}" - abort("\nPython dependencies are needed to run this workflow. Install with the CLI command: uo install_python \n") - end + # check that uv is available + require_uv # Step 1 Complete - usg_cli_root = "#{res[:pvars][:usg_path].to_s} complete" usg_cli_addition = '' if @opthash.subopts[:input] @@ -2419,8 +2143,7 @@ def self.install_python_dependencies end begin - puts "\nRunning system command: #{usg_cli_root + usg_cli_addition}\n" - system(usg_cli_root + usg_cli_addition) + run_uv_tool('usg', "usg complete#{usg_cli_addition}") rescue FileNotFoundError abort("\nInput CSV File #{@opthash.subopts[:input]} not found. Please check the file path and try again.") rescue StandardError => e @@ -2429,20 +2152,18 @@ def self.install_python_dependencies # Step 2 - Post Process # this is a temporary step needed to convert headers to newer ResStock schemas - usg_cli_root2 = "#{res[:pvars][:usg_path].to_s} process" - usg_cli_addition = '' + usg_cli_addition2 = '' # using --no-reports to not write reports - usg_cli_addition += " -i #{output_file}" - usg_cli_addition += " -o #{output_file.sub('.csv', '_converted.csv')} --no-reports" + usg_cli_addition2 += " -i #{output_file}" + usg_cli_addition2 += " -o #{output_file.sub('.csv', '_converted.csv')} --no-reports" if @opthash.subopts[:feature] - usg_cli_addition += " -g #{@opthash.subopts[:feature]}" + usg_cli_addition2 += " -g #{@opthash.subopts[:feature]}" end begin - puts "\nRunning system command: #{usg_cli_root2 + usg_cli_addition}\n" - system(usg_cli_root2 + usg_cli_addition) + run_uv_tool('usg', "usg process#{usg_cli_addition2}") rescue FileNotFoundError abort("\nCSV File #{output_file} not found. Please check the file path and try again.") rescue StandardError => e diff --git a/spec/uo_cli_spec.rb b/spec/uo_cli_spec.rb index be66760a6..a51f2f156 100644 --- a/spec/uo_cli_spec.rb +++ b/spec/uo_cli_spec.rb @@ -55,16 +55,6 @@ def delete_directory_or_file(dir_or_file) end end - # Find Python version - # Returns Python version as a list of strings for major, minor, and patch - def find_python_version - version_output, status = Open3.capture2e('python3 --version') - if status.success? - version = version_output.split(' ')[1] - return version.split('.') - end - end - # Look through the workflow file and activate certain measures # params\ # +test_dir+:: _path_ Path to the test directory being used @@ -322,20 +312,11 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw', end context 'Install python dependencies' do - it 'successfully installs python and dependencies' do - config = example_dir / 'python_deps' / 'config.json' - FileUtils.rm_rf(config) if config.exist? + it 'successfully installs python dependencies via uv' do system("#{call_cli} install_python") - python_config = example_dir / 'python_deps' / 'python_config.json' - expect(python_config.exist?).to be true - - configs = JSON.parse(File.read(python_config)) - expect(configs['python_path']).not_to be_falsey - expect(configs['pip_path']).not_to be_falsey - expect(configs['ditto_path']).not_to be_falsey - expect(configs['gmt_path']).not_to be_falsey - expect(configs['disco_path']).not_to be_falsey - expect(configs['ghe_path']).not_to be_falsey + # Verify uv is available and sync succeeds + uv_version, status = Open3.capture2e('uv --version') + expect(status.success?).to be true end end From d71ff4f330a17dacbab50b56437d1e2571bf18a4 Mon Sep 17 00:00:00 2001 From: Katherine Fleming <2205659+kflemin@users.noreply.github.com> Date: Tue, 14 Apr 2026 13:04:41 -0600 Subject: [PATCH 02/11] install uv in ci --- .github/workflows/nightly_ci_build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/nightly_ci_build.yml b/.github/workflows/nightly_ci_build.yml index 3d34051b0..417da246d 100644 --- a/.github/workflows/nightly_ci_build.yml +++ b/.github/workflows/nightly_ci_build.yml @@ -41,6 +41,9 @@ jobs: ruby --version bundle install bundle update + - name: Install uv + if: ${{ matrix.simulation-type == 'electric' || matrix.simulation-type == 'GHE' }} + run: curl -LsSf https://astral.sh/uv/install.sh | sh - name: Install python dependencies if: ${{ matrix.simulation-type == 'electric' || matrix.simulation-type == 'GHE' }} run: bundle exec rspec -e 'Install python dependencies' From 6c033933be08afe72d686fdd432a69459cbbb293 Mon Sep 17 00:00:00 2001 From: Katherine Fleming <2205659+kflemin@users.noreply.github.com> Date: Tue, 14 Apr 2026 16:20:51 -0600 Subject: [PATCH 03/11] update tests and copilot issues --- example_files/python_deps/pyproject.toml | 2 +- lib/uo_cli.rb | 36 +++++++++++++++--------- spec/uo_cli_spec.rb | 13 +++++++-- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/example_files/python_deps/pyproject.toml b/example_files/python_deps/pyproject.toml index 995c4452d..fb672b851 100644 --- a/example_files/python_deps/pyproject.toml +++ b/example_files/python_deps/pyproject.toml @@ -3,7 +3,7 @@ name = "urbanopt" version = "1.2.0" description = "URBANopt python dependency management" readme = "README.md" -requires-python = "==3.10.19" +requires-python = "==3.10.*" dependencies = [] [dependency-groups] diff --git a/lib/uo_cli.rb b/lib/uo_cli.rb index c401f58d8..b9fa2421a 100755 --- a/lib/uo_cli.rb +++ b/lib/uo_cli.rb @@ -17,6 +17,7 @@ require 'json' require 'openssl' require 'open3' +require 'shellwords' require 'yaml' module URBANopt @@ -1076,7 +1077,7 @@ def self.update_project(existing_project_folder, new_project_directory) # Check that uv is available on the system PATH def self.check_uv puts 'Checking for uv...' - stdout, stderr, status = Open3.capture3('uv --version') + stdout, stderr, status = Open3.capture3('uv', '--version') if status.success? puts "...uv found: #{stdout.strip}" return true @@ -1100,12 +1101,15 @@ def self.run_uv_tool(group, command, use_system: true) package = UV_TOOL_PACKAGES[group] abort("\nERROR: Unknown tool group '#{group}'") if package.nil? - full_command = "uv tool run --python #{UV_PYTHON_VERSION} --from \"#{package}\" #{command}" - puts "Running: #{full_command}" + base_args = ['uv', 'tool', 'run', '--python', UV_PYTHON_VERSION, '--from', package] + cmd_args = Shellwords.shellsplit(command) + full_args = base_args + cmd_args + + puts "Running: #{full_args.shelljoin}" if use_system - system(full_command) + system(*full_args) else - stdout, stderr, status = Open3.capture3(full_command) + stdout, stderr, status = Open3.capture3(*full_args) return stdout, stderr, status end end @@ -1115,19 +1119,21 @@ def self.install_python_dependencies errors = [] UV_TOOL_PACKAGES.each do |group, package| puts "Installing '#{group}' (#{package})..." - stdout, stderr, status = Open3.capture3("uv tool install --python #{UV_PYTHON_VERSION} \"#{package}\"") + stdout, stderr, status = Open3.capture3('uv', 'tool', 'install', '--python', UV_PYTHON_VERSION, package) if status.success? puts "...#{group} installed successfully" else - puts "ERROR installing #{group}: #{stderr}" - errors << "#{group}: #{stderr}" + puts "ERROR installing #{group}:" + puts " stdout: #{stdout}" unless stdout.strip.empty? + puts " stderr: #{stderr}" unless stderr.strip.empty? + errors << group end end if errors.empty? puts "\nAll Python tools successfully installed" else - puts "\nErrors occurred when installing tools:\n#{errors.join("\n")}" + abort("\nThe following tools failed to install: #{errors.join(', ')}") end end @@ -1297,9 +1303,9 @@ def self.install_python_dependencies puts "\nERROR: #{e.message}" end - ditto_cli_addition = 'run-opendss ' + ditto_cli_addition = '' if @opthash.subopts[:config] - ditto_cli_addition += "--config #{@opthash.subopts[:config]}" + ditto_cli_addition = "--config #{@opthash.subopts[:config]}" elsif @opthash.subopts[:scenario] && @opthash.subopts[:feature] ditto_cli_addition = "--scenario_file #{@opthash.subopts[:scenario]} --feature_file #{@opthash.subopts[:feature]}" if @opthash.subopts[:equipment] @@ -1383,8 +1389,12 @@ def self.install_python_dependencies end puts 'Running DISCO...' stdout, stderr, status = run_uv_tool('disco', "disco #{disco_args}", use_system: false) - if !stderr.empty? - puts "ERROR running DISCO: #{stderr}" + if !status.success? + puts "ERROR running DISCO (exit code #{status.exitstatus}):" + puts stderr unless stderr.empty? + puts stdout unless stdout.empty? + elsif !stderr.empty? + puts "DISCO warnings: #{stderr}" end puts "Refer to detailed log file #{File.join(run_folder, 'disco', 'run_upgrade_cost_analysis.log')} for more information on the run." puts "Refer to the output summary file #{File.join(run_folder, 'disco', 'output_summary.json')} for a summary of the results." diff --git a/spec/uo_cli_spec.rb b/spec/uo_cli_spec.rb index a51f2f156..5ab3d5677 100644 --- a/spec/uo_cli_spec.rb +++ b/spec/uo_cli_spec.rb @@ -313,10 +313,19 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw', context 'Install python dependencies' do it 'successfully installs python dependencies via uv' do - system("#{call_cli} install_python") - # Verify uv is available and sync succeeds + result = system("#{call_cli} install_python") + expect(result).to be true + + # Verify uv is available uv_version, status = Open3.capture2e('uv --version') expect(status.success?).to be true + + # Verify all expected tools are installed + tool_list, tool_status = Open3.capture2e('uv tool list') + expect(tool_status.success?).to be true + %w[disco ditto-reader thermalnetwork urbanopt-des usg].each do |tool| + expect(tool_list).to include(tool), "Expected '#{tool}' to be installed but it was not found in uv tool list" + end end end From dd9c658c3bcfc89aa388e016e5fd2f5fe473013a Mon Sep 17 00:00:00 2001 From: Katherine Fleming <2205659+kflemin@users.noreply.github.com> Date: Thu, 14 May 2026 16:42:42 -0600 Subject: [PATCH 04/11] update version --- lib/uo_cli/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/uo_cli/version.rb b/lib/uo_cli/version.rb index d79494ad0..0fe913f9e 100644 --- a/lib/uo_cli/version.rb +++ b/lib/uo_cli/version.rb @@ -5,6 +5,6 @@ module URBANopt module CLI - VERSION = '1.2.0'.freeze + VERSION = '1.2.0.dev1'.freeze end end From bdb20d09c31ac753a1501b8a3206134a2e5d0b74 Mon Sep 17 00:00:00 2001 From: Katherine Fleming <2205659+kflemin@users.noreply.github.com> Date: Tue, 19 May 2026 15:08:06 -0600 Subject: [PATCH 05/11] update district types --- lib/uo_cli.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/uo_cli.rb b/lib/uo_cli.rb index b9fa2421a..f8a5f810f 100755 --- a/lib/uo_cli.rb +++ b/lib/uo_cli.rb @@ -419,12 +419,12 @@ def opt_des_params "Example: uo des_params --sys-param path/to/sys_params.json --feature path/to/example_project.json\n", type: String, required: true, short: :f opt :model_type, "\nSelection for which kind of DES simulation to perform\n" \ - "Valid choices: 'time_series']\n" \ + "Valid choice: 'time_series'\n" \ 'If not specified, the default time_series simulation type will be used', type: String, short: :m opt :district_type, "\nSelection for which kind of district system parameters to generate\n" \ "Example: uo des_params --sys-param path/to/sys_params.json --feature path/to/example_project.json --district-type 5G_ghe\n" \ - "Available options are: ['4G', '5G_ghe']\n" \ + "Available options are: ['steam', '4G', '5G', '5G_ghe']. Defaults to '4G'.\n" \ 'If not specified, the default 4G district type will be used', type: String, short: :t opt :overwrite, "\nDelete and rebuild existing sys-param file\n", short: :o From 8d742e2530f94e052ca47438961b2214d64a6ad7 Mon Sep 17 00:00:00 2001 From: Katherine Fleming <2205659+kflemin@users.noreply.github.com> Date: Tue, 19 May 2026 15:23:25 -0600 Subject: [PATCH 06/11] fix space in filename --- example_files/python_deps/.python-version | 1 - lib/uo_cli/version.rb | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 example_files/python_deps/.python-version diff --git a/example_files/python_deps/.python-version b/example_files/python_deps/.python-version deleted file mode 100644 index 7c7a975f4..000000000 --- a/example_files/python_deps/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.10 \ No newline at end of file diff --git a/lib/uo_cli/version.rb b/lib/uo_cli/version.rb index 0fe913f9e..12251e90b 100644 --- a/lib/uo_cli/version.rb +++ b/lib/uo_cli/version.rb @@ -5,6 +5,6 @@ module URBANopt module CLI - VERSION = '1.2.0.dev1'.freeze + VERSION = '1.2.0.dev2'.freeze end end From 5febcb3c4d2c8a279b524b57b5009d475f6aeb2e Mon Sep 17 00:00:00 2001 From: Katherine Fleming <2205659+kflemin@users.noreply.github.com> Date: Thu, 28 May 2026 15:34:00 -0600 Subject: [PATCH 07/11] fix gmt dependency...should be urbanoptopt-des --- example_files/python_deps/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example_files/python_deps/pyproject.toml b/example_files/python_deps/pyproject.toml index fb672b851..1ad0dc980 100644 --- a/example_files/python_deps/pyproject.toml +++ b/example_files/python_deps/pyproject.toml @@ -17,7 +17,7 @@ thermalnetwork = [ "thermalnetwork==0.5.0", ] urbanopt-des = [ - "geojson-modelica-translator==0.13.0" + "urbanopt-des==0.2.0" ] usg = [ "urban-system-generator==0.1.1" From 6b38323fa326a92d0b2476b5eb783795ba246b64 Mon Sep 17 00:00:00 2001 From: Katherine Fleming <2205659+kflemin@users.noreply.github.com> Date: Fri, 5 Jun 2026 15:40:03 -0600 Subject: [PATCH 08/11] python dependency cleanup --- example_files/python_deps/pyproject.toml | 6 +- lib/uo_cli.rb | 191 ++++++++++++++++++++--- 2 files changed, 172 insertions(+), 25 deletions(-) diff --git a/example_files/python_deps/pyproject.toml b/example_files/python_deps/pyproject.toml index 1ad0dc980..752ffd26d 100644 --- a/example_files/python_deps/pyproject.toml +++ b/example_files/python_deps/pyproject.toml @@ -23,6 +23,6 @@ usg = [ "urban-system-generator==0.1.1" ] -# Note: dependency-groups are kept here as a version manifest. -# The CLI uses UV_TOOL_PACKAGES in uo_cli.rb to map groups to PyPI packages -# for use with `uv tool run --from `. +# Note: dependency-groups are the source of truth. +# The CLI reads these entries from pyproject.toml and uses the first package in +# each group for `uv tool install` and `uv tool run --from `. diff --git a/lib/uo_cli.rb b/lib/uo_cli.rb index f8a5f810f..291d167d5 100755 --- a/lib/uo_cli.rb +++ b/lib/uo_cli.rb @@ -25,7 +25,7 @@ module CLI class UrbanOptCLI COMMAND_MAP = { 'create' => 'Make new things - project directory or files', - 'install_python' => 'Install Python dependencies for OpenDSS, DISCO, DES, GHE, and USG tools (requires uv)', + 'install_python' => 'Pre-install Python tool environments from pyproject.toml dependency-groups (requires uv)', 'update' => 'Update files in an existing URBANopt project', 'run' => 'Use files in your directory to simulate district energy use', 'process' => 'Post-process URBANopt simulations for additional insights', @@ -161,9 +161,10 @@ def opt_create def opt_install_python @subopts = Optimist.options do banner "\nURBANopt install_python:\n \n" - banner "Uses uv to sync all Python dependency groups defined in pyproject.toml\n" + banner "Pre-installs and caches uv tool environments for dependency-groups in pyproject.toml\n" + banner "Runtime commands (opendss/disco/des/ghe/usg) use uv tool run and may still resolve on demand\n" - opt :verbose, "\Verbose output \n" \ + opt :verbose, "\nVerbose output\n" \ 'Example: uo install_python --verbose' end end @@ -1053,20 +1054,17 @@ def self.update_project(existing_project_folder, new_project_directory) end end - # Mapping from dependency group name to PyPI package spec. - # These must match the [dependency-groups] entries in python_deps/pyproject.toml. - # Note: urbanopt-des uses geojson-modelica-translator directly because that - # package provides the `uo_des` CLI entry point. - UV_TOOL_PACKAGES = { - 'disco' => 'NREL-disco==0.5.1', - 'ditto-reader' => 'urbanopt-ditto-reader==0.6.4', - 'thermalnetwork' => 'thermalnetwork==0.5.0', - 'urbanopt-des' => 'geojson-modelica-translator==0.13.0', - 'usg' => 'urban-system-generator==0.1.1' - }.freeze - - # Python version constraint (must match pyproject.toml requires-python) - UV_PYTHON_VERSION = '3.10'.freeze + # Tool groups expected in [dependency-groups] in python_deps/pyproject.toml. + UV_TOOL_GROUPS = [ + 'disco', + 'ditto-reader', + 'thermalnetwork', + 'urbanopt-des', + 'usg' + ].freeze + + # Fallback Python version when requires-python cannot be parsed. + UV_PYTHON_VERSION_FALLBACK = '3.10'.freeze # Recommended uv version and install instructions (update version here when changing) UV_RECOMMENDED_VERSION = '0.11.6'.freeze @@ -1074,6 +1072,153 @@ def self.update_project(existing_project_folder, new_project_directory) UV_INSTALL_MESSAGE = "\nERROR: uv is not installed or not on your PATH.\n" \ "Please install uv (recommended version #{UV_RECOMMENDED_VERSION} or later): #{UV_INSTALL_URL}\n".freeze + # Locate the installed python_deps directory under the loaded CLI example_files path. + def self.setup_python_variables + pvars = { + python_install_path: nil + } + + $LOAD_PATH.each do |path_item| + if path_item.to_s.end_with?('example_files') + pvars[:python_install_path] = File.join(path_item, 'python_deps') + break + end + end + + if pvars[:python_install_path].nil? + abort("\nERROR: Could not locate example_files/python_deps in LOAD_PATH\n") + end + + pvars + end + + # Return full path to python_deps/pyproject.toml. + def self.uv_pyproject_path + pvars = setup_python_variables + File.join(pvars[:python_install_path], 'pyproject.toml') + end + + # Return python version used for uv commands, derived from pyproject requires-python. + # Example supported specs: "==3.10.*", ">=3.10,<3.12", "~=3.10". + def self.uv_python_version + pyproject_path = uv_pyproject_path + unless File.exist?(pyproject_path) + abort("\nERROR: Could not find pyproject.toml at #{pyproject_path}\n") + end + + requires_python = nil + File.readlines(pyproject_path, chomp: true).each do |raw_line| + line = raw_line.strip + next if line.empty? || line.start_with?('#') + + match = line.match(/^requires-python\s*=\s*"([^"]+)"/) + if match + requires_python = match[1] + break + end + end + + if requires_python.nil? + puts "WARNING: requires-python not found in pyproject.toml; using fallback #{UV_PYTHON_VERSION_FALLBACK}" + return UV_PYTHON_VERSION_FALLBACK + end + + version_match = requires_python.match(/(\d+\.\d+)/) + if version_match.nil? + puts "WARNING: could not parse requires-python '#{requires_python}'; using fallback #{UV_PYTHON_VERSION_FALLBACK}" + return UV_PYTHON_VERSION_FALLBACK + end + + version_match[1] + end + + # Return dependency-groups hash parsed from python_deps/pyproject.toml. + # Expected shape: { 'group-name' => ['package-spec', ...], ... } + def self.load_uv_dependency_groups + pyproject_path = uv_pyproject_path + + unless File.exist?(pyproject_path) + abort("\nERROR: Could not find pyproject.toml at #{pyproject_path}\n") + end + + groups = {} + in_dependency_groups = false + current_group = nil + current_specs = [] + + File.readlines(pyproject_path, chomp: true).each do |raw_line| + line = raw_line.strip + next if line.empty? || line.start_with?('#') + + section_match = line.match(/^\[([^\]]+)\]$/) + if section_match + if in_dependency_groups && !current_group.nil? + groups[current_group] = current_specs.dup + current_group = nil + current_specs = [] + end + in_dependency_groups = section_match[1] == 'dependency-groups' + next + end + + next unless in_dependency_groups + + unless current_group.nil? + if line.start_with?(']') + groups[current_group] = current_specs.dup + current_group = nil + current_specs = [] + else + line.scan(/"([^"]+)"/) { |match| current_specs << match[0] } + end + next + end + + group_match = line.match(/^([A-Za-z0-9_-]+)\s*=\s*\[(.*)$/) + next if group_match.nil? + + group_name = group_match[1] + remainder = group_match[2].strip + inline_specs = [] + remainder.scan(/"([^"]+)"/) { |match| inline_specs << match[0] } + + if remainder.include?(']') + groups[group_name] = inline_specs + else + current_group = group_name + current_specs = inline_specs + end + end + + if in_dependency_groups && !current_group.nil? + groups[current_group] = current_specs.dup + end + + groups + end + + # Return map of tool group to package spec. + # The first package listed in each group is used as the uv tool package. + def self.uv_tool_packages + dependency_groups = load_uv_dependency_groups + package_map = {} + + UV_TOOL_GROUPS.each do |group| + specs = dependency_groups[group] + if specs.nil? || specs.empty? + abort("\nERROR: Missing dependency group '#{group}' in pyproject.toml\n") + end + + if specs.length > 1 + puts "WARNING: dependency group '#{group}' has multiple package specs; using first one for uv tool commands" + end + + package_map[group] = specs.first + end + + package_map + end + # Check that uv is available on the system PATH def self.check_uv puts 'Checking for uv...' @@ -1094,14 +1239,15 @@ def self.require_uv # Run a Python tool via `uv tool run --from `. # Each tool runs in an isolated ephemeral environment — no shared lockfile needed. - # +group+:: dependency group name (key in UV_TOOL_PACKAGES, e.g. 'ditto-reader') + # +group+:: dependency group name (key in pyproject [dependency-groups], e.g. 'ditto-reader') # +command+:: the CLI command and arguments to run (e.g. 'ditto_reader_cli run-opendss ...') # +use_system+:: if true, use system() for interactive output; if false, use Open3.capture3 def self.run_uv_tool(group, command, use_system: true) - package = UV_TOOL_PACKAGES[group] + package = uv_tool_packages[group] abort("\nERROR: Unknown tool group '#{group}'") if package.nil? - base_args = ['uv', 'tool', 'run', '--python', UV_PYTHON_VERSION, '--from', package] + python_version = uv_python_version + base_args = ['uv', 'tool', 'run', '--python', python_version, '--from', package] cmd_args = Shellwords.shellsplit(command) full_args = base_args + cmd_args @@ -1117,9 +1263,10 @@ def self.run_uv_tool(group, command, use_system: true) # Install all Python tool dependencies (pre-caches each tool's environment) def self.install_python_dependencies errors = [] - UV_TOOL_PACKAGES.each do |group, package| + python_version = uv_python_version + uv_tool_packages.each do |group, package| puts "Installing '#{group}' (#{package})..." - stdout, stderr, status = Open3.capture3('uv', 'tool', 'install', '--python', UV_PYTHON_VERSION, package) + stdout, stderr, status = Open3.capture3('uv', 'tool', 'install', '--python', python_version, package) if status.success? puts "...#{group} installed successfully" else From 3cdf6f60a622190b2d6c47b360912bf6c0ecfebd Mon Sep 17 00:00:00 2001 From: Katherine Fleming <2205659+kflemin@users.noreply.github.com> Date: Fri, 5 Jun 2026 16:03:28 -0600 Subject: [PATCH 09/11] temporarily remove disco dependency for this release --- .gitignore | 2 +- lib/uo_cli.rb | 34 ++++++++++++++++------------------ 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 53cd74e3a..85cfc2815 100644 --- a/.gitignore +++ b/.gitignore @@ -32,5 +32,5 @@ example_files/python_deps/python_config.json example_files/python_deps/python example_files/python_deps/python-3.10 - +*.venv reopt_project diff --git a/lib/uo_cli.rb b/lib/uo_cli.rb index 291d167d5..df575edab 100755 --- a/lib/uo_cli.rb +++ b/lib/uo_cli.rb @@ -32,7 +32,7 @@ class UrbanOptCLI 'visualize' => 'Visualize and compare results for features and scenarios', 'validate' => 'Validate results with custom rules', 'opendss' => 'Run OpenDSS simulation', - 'disco' => 'Run DISCO analysis', + 'disco' => 'Run DISCO analysis (temporarily unavailable in this version)', 'rnm' => 'Run RNM simulation', 'delete' => 'Delete simulations for a specified scenario', 'des_params' => 'Make a DES system parameters config file', @@ -256,6 +256,7 @@ def opt_opendss def opt_disco @subopts = Optimist.options do banner "\nURBANopt disco:\n\n" + banner "DISCO is temporarily unavailable in this version and will be restored in the next installer.\n" opt :scenario, "\nRun DISCO simulations for \n" \ "Requires --feature also be specified\n" \ @@ -1055,8 +1056,8 @@ def self.update_project(existing_project_folder, new_project_directory) end # Tool groups expected in [dependency-groups] in python_deps/pyproject.toml. + # TODO: restore DISCO once it is working again UV_TOOL_GROUPS = [ - 'disco', 'ditto-reader', 'thermalnetwork', 'urbanopt-des', @@ -1072,7 +1073,8 @@ def self.update_project(existing_project_folder, new_project_directory) UV_INSTALL_MESSAGE = "\nERROR: uv is not installed or not on your PATH.\n" \ "Please install uv (recommended version #{UV_RECOMMENDED_VERSION} or later): #{UV_INSTALL_URL}\n".freeze - # Locate the installed python_deps directory under the loaded CLI example_files path. + # Locate python_deps from the loaded example_files path, or fall back to the + # repo/gem-relative example_files directory if it is not present on $LOAD_PATH. def self.setup_python_variables pvars = { python_install_path: nil @@ -1085,6 +1087,14 @@ def self.setup_python_variables end end + if pvars[:python_install_path].nil? + fallback_example_files = File.expand_path('../example_files', __dir__) + fallback_python_deps = File.join(fallback_example_files, 'python_deps') + if Dir.exist?(fallback_python_deps) + pvars[:python_install_path] = fallback_python_deps + end + end + if pvars[:python_install_path].nil? abort("\nERROR: Could not locate example_files/python_deps in LOAD_PATH\n") end @@ -1498,6 +1508,8 @@ def self.install_python_dependencies # Run DISCO Simulation if @opthash.command == 'disco' + abort("\nDISCO is not included in this version due to a temporary dependency issue. It will be restored in the next version.\n") + # check that uv is available require_uv @@ -1628,21 +1640,7 @@ def self.install_python_dependencies abort("\nNo OpenDSS results available in folder '#{opendss_folder}'\n") end elsif @opthash.subopts[:disco] == true - puts "\nPost-processing DISCO results\n" - disco_folder = File.join(@root_dir, 'run', @scenario_name.downcase, 'disco') - if File.directory?(disco_folder) - disco_folder_name = File.basename(disco_folder) - disco_post_processor = URBANopt::Scenario::DISCOPostProcessor.new( - scenario_report, - disco_results_dir_name = disco_folder_name - ) - disco_post_processor.run - puts "\nDone\n" - results << { process_type: 'disco', status: 'Complete', timestamp: Time.now.strftime('%Y-%m-%dT%k:%M:%S.%L') } - else - results << { process_type: 'disco', status: 'failed', timestamp: Time.now.strftime('%Y-%m-%dT%k:%M:%S.%L') } - abort("\nNo DISCO results available in folder '#{opendss_folder}'\n") - end + abort("\nDISCO post-processing is not available in this version because the DISCO dependency is temporarily excluded. It will be restored in the next version.\n") elsif (@opthash.subopts[:reopt_scenario] == true) || (@opthash.subopts[:reopt_feature] == true) || (@opthash.subopts[:reopt_backup_power] == true) # --- REOPT Scenarios --- From 56f750324b42ad7f8993839e35b9f83216e97f35 Mon Sep 17 00:00:00 2001 From: Katherine Fleming <2205659+kflemin@users.noreply.github.com> Date: Fri, 5 Jun 2026 16:08:56 -0600 Subject: [PATCH 10/11] temp remove disco and disco tests --- lib/uo_cli.rb | 2 +- spec/uo_cli_spec.rb | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/uo_cli.rb b/lib/uo_cli.rb index df575edab..c80334f77 100755 --- a/lib/uo_cli.rb +++ b/lib/uo_cli.rb @@ -1640,7 +1640,7 @@ def self.install_python_dependencies abort("\nNo OpenDSS results available in folder '#{opendss_folder}'\n") end elsif @opthash.subopts[:disco] == true - abort("\nDISCO post-processing is not available in this version because the DISCO dependency is temporarily excluded. It will be restored in the next version.\n") + abort("\nDISCO post-processing is not available in this version due to a temporary dependency issue. It will be restored in the next version.\n") elsif (@opthash.subopts[:reopt_scenario] == true) || (@opthash.subopts[:reopt_feature] == true) || (@opthash.subopts[:reopt_backup_power] == true) # --- REOPT Scenarios --- diff --git a/spec/uo_cli_spec.rb b/spec/uo_cli_spec.rb index 5ab3d5677..a3b2d452b 100644 --- a/spec/uo_cli_spec.rb +++ b/spec/uo_cli_spec.rb @@ -323,7 +323,8 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw', # Verify all expected tools are installed tool_list, tool_status = Open3.capture2e('uv tool list') expect(tool_status.success?).to be true - %w[disco ditto-reader thermalnetwork urbanopt-des usg].each do |tool| + # TODO: restore disco assertion when DISCO dependency is re-enabled. + %w[ditto-reader thermalnetwork urbanopt-des usg].each do |tool| expect(tool_list).to include(tool), "Expected '#{tool}' to be installed but it was not found in uv tool list" end end @@ -798,6 +799,7 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw', end it 'successfully runs disco simulation', :electric do + skip('DISCO is temporarily unavailable and will be restored in the next installer release.') # This test requires the 'runs an electrical network scenario' be run first system("#{call_cli} disco --scenario #{test_scenario_elec} --feature #{test_feature_elec}") expect((test_directory_elec / 'run' / 'electrical_scenario' / 'disco').exist?).to be true From c22561728cd40606c35cb46aef746c39f31b2c3e Mon Sep 17 00:00:00 2001 From: Katherine Fleming <2205659+kflemin@users.noreply.github.com> Date: Fri, 5 Jun 2026 16:20:38 -0600 Subject: [PATCH 11/11] readme update --- README.md | 128 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 122 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 49bcda1fa..a4efa956a 100644 --- a/README.md +++ b/README.md @@ -186,13 +186,129 @@ uo --version Python dependencies are currently versioned as follows: -| Python Package | Version | -| --------------------------- | ------- | -| urbanopt-ditto-reader | 0.6.4 | -| NREL-disco | 0.5.1 | -| geojson-modelica-translator | 0.8.0 | -| ThermalNetwork | 0.3.0 | +| Python Package | Version | Notes | +| --------------------------- | ------- | ----- | +| urbanopt-ditto-reader | 0.6.4 | | +| NREL-disco | 0.5.1 |Currently excluded due to dependency issue. Will be restored in next version | +| urbanopt-des | 0.2.0 | This includes the Geojson Modelica Translator | +| ThermalNetwork | 0.5.0 | This includes GHEDesigner | +| Urban System Generator (usg) | 0.1.1 | | + ## Development To install this gem onto your local machine, clone this repo and run `bundle exec rake install`. If you make changes to this repo, update the version number in `lib/version.rb` in your first commit. When ready to release, [follow the documentation](https://docs.urbanopt.net/developer_resources/release_instructions.html). + + +## Python Dependency Refactor - uv + +Starting with version 1.3.0, there has been a major python dependency refactor. +The CLI now uses `example_files/python_deps/pyproject.toml` as the source of truth for Python tool dependencies and uv for python package management. + +The CLI: +1. Reads `[dependency-groups]` from `pyproject.toml`. +2. Reads `requires-python` from `pyproject.toml` and derives a major.minor version for uv (for example `3.10` from `==3.10.*`). +3. Uses `uv tool install --python ` during `uo install_python`. +4. Uses `uv tool run --python --from ` at runtime. + + +### For troubleshooting only: How to Update a Python Dependency in an Installed URBANopt Installer + +If you need to manually update a python dependency directly in the URBANopt CLI installer, follow the steps below. + +#### Step 1: Locate Installed pyproject.toml + +Find the installed gem location: + +```bash +gem contents urbanopt-cli | grep example_files/python_deps/pyproject.toml +``` + +If your installer is at `/Applications/URBANoptCLI_1.2.0`, the file is typically under that install's embedded Ruby gem path, ending with: + +```text +.../gems/urbanopt-cli-/example_files/python_deps/pyproject.toml +``` + +#### Step 2: Edit the Dependency in pyproject.toml + +Open `pyproject.toml` and edit the package spec in `[dependency-groups]`. + +Example: + +```toml +[dependency-groups] +thermalnetwork = [ + "thermalnetwork==0.5.0", +] +``` + +Update to: + +```toml +thermalnetwork = [ + "thermalnetwork==0.6.0", +] +``` + +Notes: +1. Keep valid TOML syntax. +2. The CLI uses the first package entry in each group for uv tool install/run. +3. If you add multiple entries in one group, the CLI warns and uses only the first one for uv tool commands. + +#### Step 3: Reinstall Python Tool Environments via CLI + +From an environment where `uo` resolves to the installed CLI, run: + +```bash +uo install_python +``` + +This command now: +1. Checks `uv` availability. +2. Loads dependency groups from installed `pyproject.toml`. +3. Determines Python version from `requires-python`. +4. Installs each active tool with `uv tool install`. + +There is no separate `uv sync` step required for CLI behavior. + +#### Step 4: Verify with an End-to-End CLI Command + +Use a command that exercises the updated tool. + +Examples: +1. `ditto-reader`: run `uo opendss ...` +2. `thermalnetwork`: run `uo ghe_size ...` +3. `urbanopt-des`: run `uo des_params ...` or other `des_*` command +4. `usg`: run `uo usg_preprocess ...` + +Because runtime uses `uv tool run --from `, this is the most reliable verification path. + +#### For Manual uv Testing (Optional) + +If you want to test outside `uo`, mirror CLI behavior directly: + +```bash +uv tool install --python 3.10 "thermalnetwork==0.6.0" +uv tool run --python 3.10 --from "thermalnetwork==0.6.0" python -c "import thermalnetwork; print(thermalnetwork.__version__)" +``` + +Use the Python version derived from `requires-python` in installed `pyproject.toml`. + +#### Troubleshooting + +`ERROR: uv is not installed or not on your PATH`: +1. Install uv and retry `uo install_python`. + +`Missing dependency group '' in pyproject.toml`: +1. Ensure group names match expected active groups exactly. +2. Ensure `[dependency-groups]` section is valid TOML. + +`requires-python not found` or parse warning: +1. Add/fix `requires-python` in `[project]` (for example `==3.10.*`). +2. If parsing fails, CLI falls back to Python `3.10`. + +Command still appears to use old behavior: +1. Confirm you edited the installed gem's `pyproject.toml`, not a source checkout copy. +2. Re-run `uo install_python` after editing. +