From 070811d91c992691f16bbf7b97d9de2310fc7398 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 9 Aug 2025 07:20:11 -0600 Subject: [PATCH 01/37] refactoring gurobi interfaces --- .../solver/solvers/gurobi_persistent.py | 156 ++++++++++++------ 1 file changed, 108 insertions(+), 48 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi_persistent.py index ea3693c1c70..899b7915e80 100644 --- a/pyomo/contrib/solver/solvers/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi_persistent.py @@ -30,7 +30,7 @@ from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types from pyomo.repn import generate_standard_repn from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability +from pyomo.contrib.solver.common.base import PersistentSolverBase, SolverBase, Availability from pyomo.contrib.solver.common.results import ( Results, TerminationCondition, @@ -233,44 +233,15 @@ def __init__(self): self.var2 = None -class GurobiPersistent( - GurobiSolverMixin, - PersistentSolverMixin, - PersistentSolverUtils, - PersistentSolverBase, -): - """ - Interface to Gurobi persistent - """ - +class GurobiBase(SolverBase): CONFIG = GurobiConfig() _gurobipy_available = gurobipy_available def __init__(self, **kwds): - treat_fixed_vars_as_params = kwds.pop('treat_fixed_vars_as_params', True) - PersistentSolverBase.__init__(self, **kwds) - PersistentSolverUtils.__init__( - self, treat_fixed_vars_as_params=treat_fixed_vars_as_params - ) + super().__init__(**kwds) self._register_env_client() self._solver_model = None - self._symbol_map = SymbolMap() - self._labeler = None - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._solver_con_to_pyomo_con_map = {} - self._pyomo_sos_to_solver_sos_map = {} - self._range_constraints = OrderedSet() - self._mutable_helpers = {} - self._mutable_bounds = {} - self._mutable_quadratic_helpers = {} - self._mutable_objective = None - self._needs_updated = True self._callback = None - self._callback_func = None - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._last_results_object: Optional[Results] = None def release_license(self): self._reinit() @@ -280,12 +251,10 @@ def __del__(self): if not python_is_shutting_down(): self._release_env_client() - @property - def symbol_map(self): - return self._symbol_map + def _mipstart(self): + raise NotImplementedError('should be implemented by derived classes') - def _solve(self): - config = self._active_config + def _solve(self, config): timer = config.timer ostreams = [io.StringIO()] + config.tee @@ -304,13 +273,7 @@ def _solve(self): self._solver_model.setParam('MIPGapAbs', config.abs_gap) if config.use_mipstart: - for ( - pyomo_var_id, - gurobi_var, - ) in self._pyomo_var_to_solver_var_map.items(): - pyomo_var = self._vars[pyomo_var_id][0] - if pyomo_var.is_integer() and pyomo_var.value is not None: - self.set_var_attr(pyomo_var, 'Start', pyomo_var.value) + self._mipstart() for key, option in options.items(): self._solver_model.setParam(key, option) @@ -319,18 +282,99 @@ def _solve(self): self._solver_model.optimize(self._callback) timer.stop('optimize') - self._needs_updated = False res = self._postsolve(timer) res.solver_config = config res.solver_name = 'Gurobi' res.solver_version = self.version() res.solver_log = ostreams[0].getvalue() return res + + +class GurobiDirect(GurobiBase): + def __init__(self, **kwds): + super().__init__(**kwds) + + +class GurobiQuadraticBase(GurobiBase): + def __init__(self, **kwds): + super().__init__(**kwds) + self._vars = {} # from id(v) to v + self._symbol_map = SymbolMap() + self._labeler = None + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + + @property + def symbol_map(self): + return self._symbol_map + + def _mipstart(self): + for ( + pyomo_var_id, + gurobi_var, + ) in self._pyomo_var_to_solver_var_map.items(): + pyomo_var = self._vars[pyomo_var_id] + if pyomo_var.is_integer() and pyomo_var.value is not None: + gurobi_var.setAttr('Start', pyomo_var.value) + + def _proces_domain_and_bounds(self, var): + lb, ub, step = var.domain.get_interval() + if lb is None: + lb = -gurobipy.GRB.INFINITY + if ub is None: + ub = gurobipy.GRB.INFINITY + if step == 0: + vtype = gurobipy.GRB.CONTINUOUS + elif step == 1: + if lb == 0 and ub == 1: + vtype = gurobipy.GRB.BINARY + else: + vtype = gurobipy.GRB.INTEGER + else: + raise ValueError( + f'Unrecognized domain step: {step} (should be either 0 or 1)' + ) + if var.fixed: + lb = var.value + ub = lb + else: + lb = max(lb, value(var._lb)) + ub = min(ub, value(var._ub)) + return lb, ub, vtype + + +class GurobiDirectQuadratic(GurobiQuadraticBase): + def __init__(self, **kwds): + super().__init__(**kwds) + + +class GurobiPersistentQuadratic(GurobiQuadraticBase): + def __init__(self, **kwds): + super().__init__(**kwds) + self._solver_con_to_pyomo_con_map = {} + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._callback_func = None + self._last_results_object: Optional[Results] = None + + def _solve(self, config): + super()._solve(config) + self._needs_updated = False def _process_domain_and_bounds( self, var, var_id, mutable_lbs, mutable_ubs, ndx, gurobipy_var ): - _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[id(var)] + _lb = var._lb + _ub = var._ub + _fixed = var.fixed + _domain_interval = var.domain.get_interval() + _value = var.value lb, ub, step = _domain_interval if lb is None: lb = -gurobipy.GRB.INFINITY @@ -372,6 +416,24 @@ def _process_domain_and_bounds( return lb, ub, vtype + +class _GurobiPersistent( + GurobiSolverMixin, + PersistentSolverMixin, + PersistentSolverUtils, + PersistentSolverBase, +): + """ + Interface to Gurobi persistent + """ + + + def __init__(self, **kwds): + PersistentSolverBase.__init__(self, **kwds) + PersistentSolverUtils.__init__( + self, treat_fixed_vars_as_params=treat_fixed_vars_as_params + ) + def _add_variables(self, variables: List[VarData]): var_names = [] vtypes = [] @@ -522,7 +584,6 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobipy_con = self._solver_model.addRange( gurobi_expr, lhs_val, rhs_val, name=conname ) - self._range_constraints.add(con) if not is_constant(lhs_expr) or not is_constant(rhs_expr): mutable_range_constant = _MutableRangeConstant() mutable_range_constant.lhs_expr = lhs_expr @@ -654,7 +715,6 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._symbol_map.removeSymbol(con) del self._pyomo_con_to_solver_con_map[con] del self._solver_con_to_pyomo_con_map[id(solver_con)] - self._range_constraints.discard(con) self._mutable_helpers.pop(con, None) self._mutable_quadratic_helpers.pop(con, None) self._needs_updated = True From d70dbb52a28aeb3500bdd430919f43f94c7a862c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 10 Aug 2025 15:36:39 -0600 Subject: [PATCH 02/37] revert_gurobi_persistent --- .../solver/solvers/gurobi_persistent.py | 156 ++++++------------ 1 file changed, 48 insertions(+), 108 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi_persistent.py index 899b7915e80..ea3693c1c70 100644 --- a/pyomo/contrib/solver/solvers/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi_persistent.py @@ -30,7 +30,7 @@ from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types from pyomo.repn import generate_standard_repn from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, SolverBase, Availability +from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability from pyomo.contrib.solver.common.results import ( Results, TerminationCondition, @@ -233,15 +233,44 @@ def __init__(self): self.var2 = None -class GurobiBase(SolverBase): +class GurobiPersistent( + GurobiSolverMixin, + PersistentSolverMixin, + PersistentSolverUtils, + PersistentSolverBase, +): + """ + Interface to Gurobi persistent + """ + CONFIG = GurobiConfig() _gurobipy_available = gurobipy_available def __init__(self, **kwds): - super().__init__(**kwds) + treat_fixed_vars_as_params = kwds.pop('treat_fixed_vars_as_params', True) + PersistentSolverBase.__init__(self, **kwds) + PersistentSolverUtils.__init__( + self, treat_fixed_vars_as_params=treat_fixed_vars_as_params + ) self._register_env_client() self._solver_model = None + self._symbol_map = SymbolMap() + self._labeler = None + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._solver_con_to_pyomo_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + self._range_constraints = OrderedSet() + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True self._callback = None + self._callback_func = None + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._last_results_object: Optional[Results] = None def release_license(self): self._reinit() @@ -251,10 +280,12 @@ def __del__(self): if not python_is_shutting_down(): self._release_env_client() - def _mipstart(self): - raise NotImplementedError('should be implemented by derived classes') + @property + def symbol_map(self): + return self._symbol_map - def _solve(self, config): + def _solve(self): + config = self._active_config timer = config.timer ostreams = [io.StringIO()] + config.tee @@ -273,7 +304,13 @@ def _solve(self, config): self._solver_model.setParam('MIPGapAbs', config.abs_gap) if config.use_mipstart: - self._mipstart() + for ( + pyomo_var_id, + gurobi_var, + ) in self._pyomo_var_to_solver_var_map.items(): + pyomo_var = self._vars[pyomo_var_id][0] + if pyomo_var.is_integer() and pyomo_var.value is not None: + self.set_var_attr(pyomo_var, 'Start', pyomo_var.value) for key, option in options.items(): self._solver_model.setParam(key, option) @@ -282,99 +319,18 @@ def _solve(self, config): self._solver_model.optimize(self._callback) timer.stop('optimize') + self._needs_updated = False res = self._postsolve(timer) res.solver_config = config res.solver_name = 'Gurobi' res.solver_version = self.version() res.solver_log = ostreams[0].getvalue() return res - - -class GurobiDirect(GurobiBase): - def __init__(self, **kwds): - super().__init__(**kwds) - - -class GurobiQuadraticBase(GurobiBase): - def __init__(self, **kwds): - super().__init__(**kwds) - self._vars = {} # from id(v) to v - self._symbol_map = SymbolMap() - self._labeler = None - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._pyomo_sos_to_solver_sos_map = {} - - @property - def symbol_map(self): - return self._symbol_map - - def _mipstart(self): - for ( - pyomo_var_id, - gurobi_var, - ) in self._pyomo_var_to_solver_var_map.items(): - pyomo_var = self._vars[pyomo_var_id] - if pyomo_var.is_integer() and pyomo_var.value is not None: - gurobi_var.setAttr('Start', pyomo_var.value) - - def _proces_domain_and_bounds(self, var): - lb, ub, step = var.domain.get_interval() - if lb is None: - lb = -gurobipy.GRB.INFINITY - if ub is None: - ub = gurobipy.GRB.INFINITY - if step == 0: - vtype = gurobipy.GRB.CONTINUOUS - elif step == 1: - if lb == 0 and ub == 1: - vtype = gurobipy.GRB.BINARY - else: - vtype = gurobipy.GRB.INTEGER - else: - raise ValueError( - f'Unrecognized domain step: {step} (should be either 0 or 1)' - ) - if var.fixed: - lb = var.value - ub = lb - else: - lb = max(lb, value(var._lb)) - ub = min(ub, value(var._ub)) - return lb, ub, vtype - - -class GurobiDirectQuadratic(GurobiQuadraticBase): - def __init__(self, **kwds): - super().__init__(**kwds) - - -class GurobiPersistentQuadratic(GurobiQuadraticBase): - def __init__(self, **kwds): - super().__init__(**kwds) - self._solver_con_to_pyomo_con_map = {} - self._mutable_helpers = {} - self._mutable_bounds = {} - self._mutable_quadratic_helpers = {} - self._mutable_objective = None - self._needs_updated = True - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._callback_func = None - self._last_results_object: Optional[Results] = None - - def _solve(self, config): - super()._solve(config) - self._needs_updated = False def _process_domain_and_bounds( self, var, var_id, mutable_lbs, mutable_ubs, ndx, gurobipy_var ): - _lb = var._lb - _ub = var._ub - _fixed = var.fixed - _domain_interval = var.domain.get_interval() - _value = var.value + _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[id(var)] lb, ub, step = _domain_interval if lb is None: lb = -gurobipy.GRB.INFINITY @@ -416,24 +372,6 @@ def _process_domain_and_bounds( return lb, ub, vtype - -class _GurobiPersistent( - GurobiSolverMixin, - PersistentSolverMixin, - PersistentSolverUtils, - PersistentSolverBase, -): - """ - Interface to Gurobi persistent - """ - - - def __init__(self, **kwds): - PersistentSolverBase.__init__(self, **kwds) - PersistentSolverUtils.__init__( - self, treat_fixed_vars_as_params=treat_fixed_vars_as_params - ) - def _add_variables(self, variables: List[VarData]): var_names = [] vtypes = [] @@ -584,6 +522,7 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobipy_con = self._solver_model.addRange( gurobi_expr, lhs_val, rhs_val, name=conname ) + self._range_constraints.add(con) if not is_constant(lhs_expr) or not is_constant(rhs_expr): mutable_range_constant = _MutableRangeConstant() mutable_range_constant.lhs_expr = lhs_expr @@ -715,6 +654,7 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._symbol_map.removeSymbol(con) del self._pyomo_con_to_solver_con_map[con] del self._solver_con_to_pyomo_con_map[id(solver_con)] + self._range_constraints.discard(con) self._mutable_helpers.pop(con, None) self._mutable_quadratic_helpers.pop(con, None) self._needs_updated = True From 5b1d3f9cfb551598599a8f2ecf99313517bf06f7 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 10 Aug 2025 20:43:45 -0600 Subject: [PATCH 03/37] refactoring gurobi interfaces --- .../contrib/solver/solvers/gurobi/__init__.py | 0 .../solver/solvers/gurobi/gurobi_direct.py | 201 +++++++++++ .../solvers/gurobi/gurobi_direct_base.py | 328 ++++++++++++++++++ 3 files changed, 529 insertions(+) create mode 100644 pyomo/contrib/solver/solvers/gurobi/__init__.py create mode 100644 pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py create mode 100644 pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py diff --git a/pyomo/contrib/solver/solvers/gurobi/__init__.py b/pyomo/contrib/solver/solvers/gurobi/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py new file mode 100644 index 00000000000..5c36372ef72 --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -0,0 +1,201 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import datetime +import io +import math +import operator +import os + +from pyomo.common.collections import ComponentMap, ComponentSet +from pyomo.common.config import ConfigValue +from pyomo.common.dependencies import attempt_import +from pyomo.common.enums import ObjectiveSense +from pyomo.common.errors import MouseTrap, ApplicationError +from pyomo.common.shutdown import python_is_shutting_down +from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.timing import HierarchicalTimer +from pyomo.core.staleflag import StaleFlagManager + +from pyomo.contrib.solver.common.base import SolverBase, Availability +from pyomo.contrib.solver.common.config import BranchAndBoundConfig +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, + NoDualsError, + NoReducedCostsError, + NoSolutionError, + IncompatibleModelError, +) +from pyomo.contrib.solver.common.results import ( + Results, + SolutionStatus, + TerminationCondition, +) +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +import logging +from .gurobi_direct_base import GurobiDirectBase, gurobipy + + +class GurobiDirectSolutionLoader(SolutionLoaderBase): + def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars): + self._grb_model = grb_model + self._grb_cons = grb_cons + self._grb_vars = grb_vars + self._pyo_cons = pyo_cons + self._pyo_vars = pyo_vars + GurobiDirectBase._register_env_client() + + def __del__(self): + if python_is_shutting_down(): + return + # Free the associated model + if self._grb_model is not None: + self._grb_cons = None + self._grb_vars = None + self._pyo_cons = None + self._pyo_vars = None + # explicitly release the model + self._grb_model.dispose() + self._grb_model = None + # Release the gurobi license if this is the last reference to + # the environment (either through a results object or solver + # interface) + GurobiDirectBase._release_env_client() + + def load_vars(self, vars_to_load=None, solution_number=0): + assert solution_number == 0 + if self._grb_model.SolCount == 0: + raise NoSolutionError() + + iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) + if vars_to_load: + vars_to_load = ComponentSet(vars_to_load) + iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) + for p_var, g_var in iterator: + p_var.set_value(g_var, skip_validation=True) + StaleFlagManager.mark_all_as_stale(delayed=True) + + def get_primals(self, vars_to_load=None, solution_number=0): + assert solution_number == 0 + if self._grb_model.SolCount == 0: + raise NoSolutionError() + + iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) + if vars_to_load: + vars_to_load = ComponentSet(vars_to_load) + iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) + return ComponentMap(iterator) + + def get_duals(self, cons_to_load=None): + if self._grb_model.Status != gurobipy.GRB.OPTIMAL: + raise NoDualsError() + + def dedup(_iter): + last = None + for con_info_dual in _iter: + if not con_info_dual[1] and con_info_dual[0][0] is last: + continue + last = con_info_dual[0][0] + yield con_info_dual + + iterator = dedup(zip(self._pyo_cons, self._grb_cons.getAttr('Pi').tolist())) + if cons_to_load: + cons_to_load = set(cons_to_load) + iterator = filter( + lambda con_info_dual: con_info_dual[0][0] in cons_to_load, iterator + ) + return {con_info[0]: dual for con_info, dual in iterator} + + def get_reduced_costs(self, vars_to_load=None): + if self._grb_model.Status != gurobipy.GRB.OPTIMAL: + raise NoReducedCostsError() + + iterator = zip(self._pyo_vars, self._grb_vars.getAttr('Rc').tolist()) + if vars_to_load: + vars_to_load = ComponentSet(vars_to_load) + iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) + return ComponentMap(iterator) + + +class GurobiDirect(GurobiDirectBase): + def __init__(self, **kwds): + super().__init__(**kwds) + self._gurobi_vars = None + self._pyomo_vars = None + + def _pyomo_gurobi_var_iter(self): + return zip(self._pyomo_vars, self._gurobi_vars.tolist()) + + def _create_solver_model(self, pyomo_model, config): + timer = config.timer + + timer.start('compile_model') + repn = LinearStandardFormCompiler().write( + pyomo_model, mixed_form=True, set_sense=None + ) + timer.stop('compile_model') + + if len(repn.objectives) > 1: + raise IncompatibleModelError( + f"The {self.__class__.__name__} solver only supports models " + f"with zero or one objectives (received {len(repn.objectives)})." + ) + + timer.start('prepare_matrices') + inf = float('inf') + ninf = -inf + bounds = list(map(operator.attrgetter('bounds'), repn.columns)) + lb = [ninf if _b is None else _b for _b in map(operator.itemgetter(0), bounds)] + ub = [inf if _b is None else _b for _b in map(operator.itemgetter(1), bounds)] + CON = gurobipy.GRB.CONTINUOUS + BIN = gurobipy.GRB.BINARY + INT = gurobipy.GRB.INTEGER + vtype = [ + ( + CON + if v.is_continuous() + else BIN if v.is_binary() else INT if v.is_integer() else '?' + ) + for v in repn.columns + ] + sense_type = list('=<>') # Note: ordering matches 0, 1, -1 + sense = [sense_type[r[1]] for r in repn.rows] + timer.stop('prepare_matrices') + + gurobi_model = gurobipy.Model(env=self.env()) + + timer.start('transfer_model') + x = gurobi_model.addMVar( + len(repn.columns), + lb=lb, + ub=ub, + obj=repn.c.todense()[0] if repn.c.shape[0] else 0, + vtype=vtype, + ) + A = gurobi_model.addMConstr(repn.A, x, sense, repn.rhs) + if repn.c.shape[0]: + gurobi_model.setAttr('ObjCon', repn.c_offset[0]) + gurobi_model.setAttr('ModelSense', int(repn.objectives[0].sense)) + # Note: calling gurobi_model.update() here is not + # necessary (it will happen as part of optimize()): + # gurobi_model.update() + timer.stop('transfer_model') + + self._pyomo_vars = repn.columns + self._gurobi_vars = x + + solution_loader = GurobiDirectSolutionLoader( + gurobi_model, A, x, repn.rows, repn.columns + ) + has_obj = len(repn.objectives) > 0 + + return gurobi_model, solution_loader, has_obj diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py new file mode 100644 index 00000000000..01c91b8b2ed --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -0,0 +1,328 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import datetime +import io +import math +import operator +import os + +from pyomo.common.collections import ComponentMap, ComponentSet +from pyomo.common.config import ConfigValue +from pyomo.common.dependencies import attempt_import +from pyomo.common.enums import ObjectiveSense +from pyomo.common.errors import MouseTrap, ApplicationError +from pyomo.common.shutdown import python_is_shutting_down +from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.timing import HierarchicalTimer +from pyomo.core.staleflag import StaleFlagManager + +from pyomo.contrib.solver.common.base import SolverBase, Availability +from pyomo.contrib.solver.common.config import BranchAndBoundConfig +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, + NoDualsError, + NoReducedCostsError, + NoSolutionError, + IncompatibleModelError, +) +from pyomo.contrib.solver.common.results import ( + Results, + SolutionStatus, + TerminationCondition, +) +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +import logging + + +logger = logging.getLogger(__name__) + + +gurobipy, gurobipy_available = attempt_import('gurobipy') + + +class GurobiConfig(BranchAndBoundConfig): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + BranchAndBoundConfig.__init__( + self, + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + self.use_mipstart: bool = self.declare( + 'use_mipstart', + ConfigValue( + default=False, + domain=bool, + description="If True, the current values of the integer variables " + "will be passed to Gurobi.", + ), + ) + + +class GurobiDirectBase(SolverBase): + + _num_gurobipy_env_clients = 0 + _gurobipy_env = None + _available = None + _gurobipy_available = gurobipy_available + _tc_map = None + + CONFIG = GurobiConfig() + + def __init__(self, **kwds): + super().__init__(**kwds) + self._register_env_client() + + def __del__(self): + if not python_is_shutting_down(): + self._release_env_client() + + def available(self): + if self._available is None: + # this triggers the deferred import, and for the persistent + # interface, may update the _available flag + # + # Note that we set the _available flag on the *most derived + # class* and not on the instance, or on the base class. That + # allows different derived interfaces to have different + # availability (e.g., persistent has a minimum version + # requirement that the direct interface doesn't - is that true?) + if not self._gurobipy_available: + if self._available is None: + self.__class__._available = Availability.NotFound + else: + self.__class__._available = self._check_license() + return self._available + + @staticmethod + def release_license(): + if GurobiDirectBase._gurobipy_env is None: + return + if GurobiDirectBase._num_gurobipy_env_clients: + logger.warning( + "Call to GurobiDirectBase.release_license() with %s remaining " + "environment clients." % (GurobiDirectBase._num_gurobipy_env_clients,) + ) + GurobiDirectBase._gurobipy_env.close() + GurobiDirectBase._gurobipy_env = None + + @staticmethod + def env(): + if GurobiDirectBase._gurobipy_env is None: + with capture_output(capture_fd=True): + GurobiDirectBase._gurobipy_env = gurobipy.Env() + return GurobiDirectBase._gurobipy_env + + @staticmethod + def _register_env_client(): + GurobiDirectBase._num_gurobipy_env_clients += 1 + + @staticmethod + def _release_env_client(): + GurobiDirectBase._num_gurobipy_env_clients -= 1 + if GurobiDirectBase._num_gurobipy_env_clients <= 0: + # Note that _num_gurobipy_env_clients should never be <0, + # but if it is, release_license will issue a warning (that + # we want to know about) + GurobiDirectBase.release_license() + + def _check_license(self): + try: + model = gurobipy.Model(env=self.env()) + except gurobipy.GurobiError: + return Availability.BadLicense + + model.setParam('OutputFlag', 0) + try: + model.addVars(range(2001)) + model.optimize() + return Availability.FullLicense + except gurobipy.GurobiError: + return Availability.LimitedLicense + finally: + model.dispose() + + def version(self): + version = ( + gurobipy.GRB.VERSION_MAJOR, + gurobipy.GRB.VERSION_MINOR, + gurobipy.GRB.VERSION_TECHNICAL, + ) + return version + + def _create_solver_model(self, pyomo_model, config): + # should return gurobi_model, solution_loader, has_objective + raise NotImplementedError('should be implemented by derived classes') + + def _pyomo_gurobi_var_iter(self): + # generator of tuples (pyomo_var, gurobi_var) + raise NotImplementedError('should be implemented by derived classes') + + def _mipstart(self): + for pyomo_var, gurobi_var in self._pyomo_gurobi_var_iter(): + if pyomo_var.is_integer() and pyomo_var.value is not None: + gurobi_var.setAttr('Start', pyomo_var.value) + + def solve(self, model, **kwds) -> Results: + start_timestamp = datetime.datetime.now(datetime.timezone.utc) + config: GurobiConfig = self.config( + value=kwds, + preserve_implicit=True, + ) + if not self.available(): + c = self.__class__ + raise ApplicationError( + f'Solver {c.__module__}.{c.__qualname__} is not available ' + f'({self.available()}).' + ) + if config.timer is None: + config.timer = HierarchicalTimer() + timer = config.timer + + StaleFlagManager.mark_all_as_stale() + ostreams = [io.StringIO()] + config.tee + + orig_cwd = os.getcwd() + try: + if config.working_dir: + os.chdir(config.working_dir) + with capture_output(TeeStream(*ostreams), capture_fd=False): + gurobi_model, solution_loader, has_obj = self._create_solver_model(model, config) + options = config.solver_options + + gurobi_model.setParam('LogToConsole', 1) + + if config.threads is not None: + gurobi_model.setParam('Threads', config.threads) + if config.time_limit is not None: + gurobi_model.setParam('TimeLimit', config.time_limit) + if config.rel_gap is not None: + gurobi_model.setParam('MIPGap', config.rel_gap) + if config.abs_gap is not None: + gurobi_model.setParam('MIPGapAbs', config.abs_gap) + + if config.use_mipstart: + self._mipstart() + + for key, option in options.items(): + gurobi_model.setParam(key, option) + + timer.start('optimize') + gurobi_model.optimize() + timer.stop('optimize') + finally: + os.chdir(orig_cwd) + + res = self._postsolve( + grb_model=gurobi_model, + config=config, + ) + + res.solution_loader = solution_loader + res.solver_log = ostreams[0].getvalue() + end_timestamp = datetime.datetime.now(datetime.timezone.utc) + res.timing_info.start_timestamp = start_timestamp + res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() + res.timing_info.timer = timer + return res + + def _get_tc_map(self): + if GurobiDirectBase._tc_map is None: + grb = gurobipy.GRB + tc = TerminationCondition + GurobiDirectBase._tc_map = { + grb.LOADED: tc.unknown, # problem is loaded, but no solution + grb.OPTIMAL: tc.convergenceCriteriaSatisfied, + grb.INFEASIBLE: tc.provenInfeasible, + grb.INF_OR_UNBD: tc.infeasibleOrUnbounded, + grb.UNBOUNDED: tc.unbounded, + grb.CUTOFF: tc.objectiveLimit, + grb.ITERATION_LIMIT: tc.iterationLimit, + grb.NODE_LIMIT: tc.iterationLimit, + grb.TIME_LIMIT: tc.maxTimeLimit, + grb.SOLUTION_LIMIT: tc.unknown, + grb.INTERRUPTED: tc.interrupted, + grb.NUMERIC: tc.unknown, + grb.SUBOPTIMAL: tc.unknown, + grb.USER_OBJ_LIMIT: tc.objectiveLimit, + } + return GurobiDirectBase._tc_map + + def _postsolve(self, grb_model, config, has_obj): + status = grb_model.Status + + results = Results() + results.timing_info.gurobi_time = grb_model.Runtime + + if grb_model.SolCount > 0: + if status == gurobipy.GRB.OPTIMAL: + results.solution_status = SolutionStatus.optimal + else: + results.solution_status = SolutionStatus.feasible + else: + results.solution_status = SolutionStatus.noSolution + + results.termination_condition = self._get_tc_map().get( + status, TerminationCondition.unknown + ) + + if ( + results.termination_condition + != TerminationCondition.convergenceCriteriaSatisfied + and config.raise_exception_on_nonoptimal_result + ): + raise NoOptimalSolutionError() + + if has_obj: + try: + if math.isfinite(grb_model.ObjVal): + results.incumbent_objective = grb_model.ObjVal + else: + results.incumbent_objective = None + except (gurobipy.GurobiError, AttributeError): + results.incumbent_objective = None + try: + results.objective_bound = grb_model.ObjBound + except (gurobipy.GurobiError, AttributeError): + if grb_model.ModelSense == ObjectiveSense.minimize: + results.objective_bound = -math.inf + else: + results.objective_bound = math.inf + else: + results.incumbent_objective = None + results.objective_bound = None + + results.iteration_count = grb_model.getAttr('IterCount') + + config.timer.start('load solution') + if config.load_solutions: + if grb_model.SolCount > 0: + results.solution_loader.load_vars() + else: + raise NoFeasibleSolutionError() + config.timer.stop('load solution') + + results.solver_config = config + results.solver_name = self.name + results.solver_version = self.version() + + return results From 4818130badff45f8873326b32710bfba2e87e7fd Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 11 Aug 2025 06:23:20 -0600 Subject: [PATCH 04/37] refactoring gurobi interfaces --- .../solver/solvers/gurobi/gurobi_direct.py | 7 +- .../solvers/gurobi/gurobi_direct_base.py | 72 ++-- .../solvers/gurobi/gurobi_persistent.py | 335 ++++++++++++++++++ 3 files changed, 381 insertions(+), 33 deletions(-) create mode 100644 pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 5c36372ef72..f4a33e2cc54 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -24,6 +24,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer from pyomo.core.staleflag import StaleFlagManager +from pyomo.repn.plugins.standard_form import LinearStandardFormCompiler from pyomo.contrib.solver.common.base import SolverBase, Availability from pyomo.contrib.solver.common.config import BranchAndBoundConfig @@ -127,6 +128,8 @@ def get_reduced_costs(self, vars_to_load=None): class GurobiDirect(GurobiDirectBase): + _minimum_version = (9, 0, 0) + def __init__(self, **kwds): super().__init__(**kwds) self._gurobi_vars = None @@ -135,8 +138,8 @@ def __init__(self, **kwds): def _pyomo_gurobi_var_iter(self): return zip(self._pyomo_vars, self._gurobi_vars.tolist()) - def _create_solver_model(self, pyomo_model, config): - timer = config.timer + def _create_solver_model(self, pyomo_model): + timer = self.config.timer timer.start('compile_model') repn = LinearStandardFormCompiler().write( diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 01c91b8b2ed..b314a39b49a 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -85,12 +85,14 @@ class GurobiDirectBase(SolverBase): _available = None _gurobipy_available = gurobipy_available _tc_map = None + _minimum_version = (0, 0, 0) CONFIG = GurobiConfig() def __init__(self, **kwds): super().__init__(**kwds) self._register_env_client() + self._callback = None def __del__(self): if not python_is_shutting_down(): @@ -111,6 +113,8 @@ def available(self): self.__class__._available = Availability.NotFound else: self.__class__._available = self._check_license() + if self.version() < self._minimum_version: + self.__class__._available = Availability.BadVersion return self._available @staticmethod @@ -169,7 +173,7 @@ def version(self): ) return version - def _create_solver_model(self, pyomo_model, config): + def _create_solver_model(self, pyomo_model): # should return gurobi_model, solution_loader, has_objective raise NotImplementedError('should be implemented by derived classes') @@ -184,29 +188,30 @@ def _mipstart(self): def solve(self, model, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) - config: GurobiConfig = self.config( - value=kwds, - preserve_implicit=True, - ) - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - if config.timer is None: - config.timer = HierarchicalTimer() - timer = config.timer - - StaleFlagManager.mark_all_as_stale() - ostreams = [io.StringIO()] + config.tee - + orig_config = self.config orig_cwd = os.getcwd() try: + self.config = config = self.config( + value=kwds, + preserve_implicit=True, + ) + if not self.available(): + c = self.__class__ + raise ApplicationError( + f'Solver {c.__module__}.{c.__qualname__} is not available ' + f'({self.available()}).' + ) + if config.timer is None: + config.timer = HierarchicalTimer() + timer = config.timer + + StaleFlagManager.mark_all_as_stale() + ostreams = [io.StringIO()] + config.tee + if config.working_dir: os.chdir(config.working_dir) with capture_output(TeeStream(*ostreams), capture_fd=False): - gurobi_model, solution_loader, has_obj = self._create_solver_model(model, config) + gurobi_model, solution_loader, has_obj = self._create_solver_model(model) options = config.solver_options gurobi_model.setParam('LogToConsole', 1) @@ -227,15 +232,16 @@ def solve(self, model, **kwds) -> Results: gurobi_model.setParam(key, option) timer.start('optimize') - gurobi_model.optimize() + gurobi_model.optimize(self._callback) timer.stop('optimize') + + res = self._postsolve( + grb_model=gurobi_model, + has_obj=has_obj, + ) finally: os.chdir(orig_cwd) - - res = self._postsolve( - grb_model=gurobi_model, - config=config, - ) + self.config = orig_config res.solution_loader = solution_loader res.solver_log = ostreams[0].getvalue() @@ -267,7 +273,7 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map - def _postsolve(self, grb_model, config, has_obj): + def _postsolve(self, grb_model, has_obj): status = grb_model.Status results = Results() @@ -288,7 +294,7 @@ def _postsolve(self, grb_model, config, has_obj): if ( results.termination_condition != TerminationCondition.convergenceCriteriaSatisfied - and config.raise_exception_on_nonoptimal_result + and self.config.raise_exception_on_nonoptimal_result ): raise NoOptimalSolutionError() @@ -313,15 +319,19 @@ def _postsolve(self, grb_model, config, has_obj): results.iteration_count = grb_model.getAttr('IterCount') - config.timer.start('load solution') - if config.load_solutions: + self.config.timer.start('load solution') + if self.config.load_solutions: if grb_model.SolCount > 0: results.solution_loader.load_vars() else: raise NoFeasibleSolutionError() - config.timer.stop('load solution') + self.config.timer.stop('load solution') - results.solver_config = config + # self.config gets copied a the beginning of + # solve and restored at the end, so modifying + # results.solver_config will not actually + # modify self.config + results.solver_config = self.config results.solver_name = self.name results.solver_version = self.version() diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py new file mode 100644 index 00000000000..fa269c1d3c5 --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -0,0 +1,335 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import io +import logging +import math +from typing import List, Optional +from collections.abc import Iterable + +from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet +from pyomo.common.dependencies import attempt_import +from pyomo.common.errors import ApplicationError +from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.timing import HierarchicalTimer +from pyomo.common.shutdown import python_is_shutting_down +from pyomo.core.kernel.objective import minimize, maximize +from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler +from pyomo.core.base.var import VarData +from pyomo.core.base.constraint import ConstraintData, Constraint +from pyomo.core.base.sos import SOSConstraintData, SOSConstraint +from pyomo.core.base.param import ParamData +from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types +from pyomo.repn import generate_standard_repn +from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression +from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability +from pyomo.contrib.solver.common.results import ( + Results, + TerminationCondition, + SolutionStatus, +) +from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig +from pyomo.contrib.solver.solvers.gurobi_direct import ( + GurobiConfigMixin, + GurobiSolverMixin, +) +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, + NoDualsError, + NoReducedCostsError, + NoSolutionError, + IncompatibleModelError, +) +from pyomo.contrib.solver.common.persistent import ( + PersistentSolverUtils, + PersistentSolverMixin, +) +from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader +from pyomo.core.staleflag import StaleFlagManager +from .gurobi_direct_base import GurobiConfig, GurobiDirectBase, gurobipy +from pyomo.contrib.solver.common.util import get_objective +from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor + + +logger = logging.getLogger(__name__) + + +class GurobiSolutionLoader(PersistentSolutionLoader): + def load_vars(self, vars_to_load=None, solution_number=0): + self._assert_solution_still_valid() + self._solver._load_vars( + vars_to_load=vars_to_load, solution_number=solution_number + ) + + def get_primals(self, vars_to_load=None, solution_number=0): + self._assert_solution_still_valid() + return self._solver._get_primals( + vars_to_load=vars_to_load, solution_number=solution_number + ) + + +class _MutableLowerBound: + def __init__(self, var_id, expr, var_map): + self.var_id = var_id + self.expr = expr + self.var_map = var_map + + def update(self): + self.var_map[self.var_id].setAttr('lb', value(self.expr)) + + +class _MutableUpperBound: + def __init__(self, var_id, expr, var_map): + self.var_id = var_id + self.expr = expr + self.var_map = var_map + + def update(self): + self.var_map[self.var_id].setAttr('ub', value(self.expr)) + + +class _MutableLinearCoefficient: + def __init__(self): + self.expr = None + self.var = None + self.con = None + self.gurobi_model = None + + def update(self): + self.gurobi_model.chgCoeff(self.con, self.var, value(self.expr)) + + +class _MutableRangeConstant: + def __init__(self): + self.lhs_expr = None + self.rhs_expr = None + self.con = None + self.slack_name = None + self.gurobi_model = None + + def update(self): + rhs_val = value(self.rhs_expr) + lhs_val = value(self.lhs_expr) + self.con.rhs = rhs_val + slack = self.gurobi_model.getVarByName(self.slack_name) + slack.ub = rhs_val - lhs_val + + +class _MutableConstant: + def __init__(self): + self.expr = None + self.con = None + + def update(self): + self.con.rhs = value(self.expr) + + +class _MutableQuadraticConstraint: + def __init__( + self, gurobi_model, gurobi_con, constant, linear_coefs, quadratic_coefs + ): + self.con = gurobi_con + self.gurobi_model = gurobi_model + self.constant = constant + self.last_constant_value = value(self.constant.expr) + self.linear_coefs = linear_coefs + self.last_linear_coef_values = [value(i.expr) for i in self.linear_coefs] + self.quadratic_coefs = quadratic_coefs + self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + + def get_updated_expression(self): + gurobi_expr = self.gurobi_model.getQCRow(self.con) + for ndx, coef in enumerate(self.linear_coefs): + current_coef_value = value(coef.expr) + incremental_coef_value = ( + current_coef_value - self.last_linear_coef_values[ndx] + ) + gurobi_expr += incremental_coef_value * coef.var + self.last_linear_coef_values[ndx] = current_coef_value + for ndx, coef in enumerate(self.quadratic_coefs): + current_coef_value = value(coef.expr) + incremental_coef_value = ( + current_coef_value - self.last_quadratic_coef_values[ndx] + ) + gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 + self.last_quadratic_coef_values[ndx] = current_coef_value + return gurobi_expr + + def get_updated_rhs(self): + return value(self.constant.expr) + + +class _MutableObjective: + def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): + self.gurobi_model = gurobi_model + self.constant = constant + self.linear_coefs = linear_coefs + self.quadratic_coefs = quadratic_coefs + self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + + def get_updated_expression(self): + for ndx, coef in enumerate(self.linear_coefs): + coef.var.obj = value(coef.expr) + self.gurobi_model.ObjCon = value(self.constant.expr) + + gurobi_expr = None + for ndx, coef in enumerate(self.quadratic_coefs): + if value(coef.expr) != self.last_quadratic_coef_values[ndx]: + if gurobi_expr is None: + self.gurobi_model.update() + gurobi_expr = self.gurobi_model.getObjective() + current_coef_value = value(coef.expr) + incremental_coef_value = ( + current_coef_value - self.last_quadratic_coef_values[ndx] + ) + gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 + self.last_quadratic_coef_values[ndx] = current_coef_value + return gurobi_expr + + +class _MutableQuadraticCoefficient: + def __init__(self): + self.expr = None + self.var1 = None + self.var2 = None + + +class GurobiDirectQuadratic(GurobiDirectBase): + _minimum_version = (7, 0, 0) + + def __init__(self, **kwds): + super().__init__(**kwds) + self._solver_model = None + self._vars = {} # from id(v) to v + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + + def _create_solver_model(self, pyomo_model): + self._clear() + self._solver_model = gurobipy.Model(env=self.env()) + cons = list(pyomo_model.component_data_objects(Constraint, descend_into=True, active=True)) + self._add_constraints(cons) + sos = list(pyomo_model.component_data_objects(SOSConstraint, descend_into=True, active=True)) + self._add_sos_constraints(sos) + obj = get_objective(pyomo_model) + self._set_objective(obj) + + def _clear(self): + self._solver_model = None + self._vars = {} + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + + def _pyomo_gurobi_var_iter(self): + for vid, v in self._vars.items(): + yield v, self._pyomo_var_to_solver_var_map[vid] + + def _process_domain_and_bounds(self, var): + lb, ub, step = var.domain.get_interval() + if lb is None: + lb = -gurobipy.GRB.INFINITY + if ub is None: + ub = gurobipy.GRB.INFINITY + if step == 0: + vtype = gurobipy.GRB.CONTINUOUS + elif step == 1: + if lb == 0 and ub == 1: + vtype = gurobipy.GRB.BINARY + else: + vtype = gurobipy.GRB.INTEGER + else: + raise ValueError( + f'Unrecognized domain: {var.domain}' + ) + if var.fixed: + lb = var.value + ub = lb + else: + lb = max(lb, value(var._lb)) + ub = min(ub, value(var._ub)) + return lb, ub, vtype + + def _add_variables(self, variables: List[VarData]): + vtypes = [] + lbs = [] + ubs = [] + for ndx, var in enumerate(variables): + self._vars[id(var)] = var + lb, ub, vtype = self._process_domain_and_bounds(var) + vtypes.append(vtype) + lbs.append(lb) + ubs.append(ub) + + gurobi_vars = self._solver_model.addVars( + len(variables), lb=lbs, ub=ubs, vtype=vtypes + ) + + for pyomo_var, gurobi_var in zip(variables, gurobi_vars): + self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var + + def _get_expr_from_pyomo_expr(self, expr): + repn = generate_standard_repn(expr, quadratic=True, compute_values=True) + + if repn.nonlinear_expr is not None: + raise IncompatibleModelError( + f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' + ) + + if len(repn.linear_vars) > 0: + missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] + self._add_variables(missing_vars) + new_expr = gurobipy.LinExpr( + repn.linear_coefs, + [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars], + ) + else: + new_expr = 0.0 + + for coef, v in zip(repn.quadratic_coefs, repn.quadratic_vars): + x, y = v + gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] + gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] + new_expr += coef * gurobi_x * gurobi_y + + return new_expr, repn.constant + + def _add_constraints(self, cons: List[ConstraintData]): + gurobi_expr_list = [] + for con in cons: + lb, body, ub = con.to_bounded_expression(evaluate_bounds=True) + gurobi_expr, repn_constant = self._get_expr_from_pyomo_expr(body) + if lb is None and ub is None: + raise ValueError( + "Constraint does not have a lower " + f"or an upper bound: {con} \n" + ) + elif lb is None: + gurobi_expr_list.append(gurobi_expr <= ub - repn_constant) + elif ub is None: + gurobi_expr_list.append(lb - repn_constant <= gurobi_expr) + elif lb == ub: + gurobi_expr_list.append(gurobi_expr == lb - repn_constant) + else: + gurobi_expr_list.append(gurobi_expr == [lb-repn_constant, ub-repn_constant]) + + gurobi_cons = self._solver_model.addConstrs(gurobi_expr_list) + self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) + self._pyomo_con_to_solver_con_map[con] = gurobipy_con + self._solver_con_to_pyomo_con_map[id(gurobipy_con)] = con + self._constraints_added_since_update.update(cons) + self._needs_updated = True + + +class GurobiPersistentQuadratic(GurobiDirectQuadratic): + _minimum_version = (7, 0, 0) From 7998fda55861cde77931d14d58b27444d9f3a501 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 10:04:55 -0600 Subject: [PATCH 05/37] refactoring gurobi interfaces --- pyomo/contrib/solver/plugins.py | 8 +- .../solvers/gurobi/gurobi_direct_base.py | 132 ++++++++++- .../solvers/gurobi/gurobi_persistent.py | 220 ++++++++++++++++-- .../solver/tests/solvers/test_solvers.py | 26 ++- 4 files changed, 350 insertions(+), 36 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 86c05f2bd70..7630c614aa2 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -13,7 +13,8 @@ from .common.factory import SolverFactory from .solvers.ipopt import Ipopt from .solvers.gurobi_persistent import GurobiPersistent -from .solvers.gurobi_direct import GurobiDirect +from .solvers.gurobi.gurobi_direct import GurobiDirect +from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic from .solvers.highs import Highs @@ -31,6 +32,11 @@ def load(): legacy_name='gurobi_direct_v2', doc='Direct (scipy-based) interface to Gurobi', )(GurobiDirect) + SolverFactory.register( + name='gurobi_direct_quadratic', + legacy_name='gurobi_direct_quadratic_v2', + doc='Direct interface to Gurobi', + )(GurobiDirect) SolverFactory.register( name='highs', legacy_name='highs', doc='Persistent interface to HiGHS' )(Highs) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index b314a39b49a..d26dbf54c83 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -78,6 +78,122 @@ def __init__( ) +def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_number): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + solution_number: int + """ + if ( + solver_model.getAttr('NumIntVars') == 0 + and solver_model.getAttr('NumBinVars') == 0 + ): + raise ValueError( + 'Cannot obtain suboptimal solutions for a continuous model' + ) + original_solution_number = solver_model.getParamInfo('SolutionNumber')[2] + solver_model.setParam('SolutionNumber', solution_number) + gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + vals = solver_model.getAttr("Xn", gurobi_vars_to_load) + res = ComponentMap() + for var, val in zip(vars_to_load, vals): + res[var] = val + solver_model.setParam('SolutionNumber', original_solution_number) + return res + + +def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + solution_number: int + """ + for v, val in _get_primals( + solver_model=solver_model, + var_map=var_map, + vars_to_load=vars_to_load, + solution_number=solution_number, + ).items(): + v.set_value(val, skip_validation=True) + StaleFlagManager.mark_all_as_stale(delayed=True) + + +def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + solution_number: int + """ + if solver_model.SolCount == 0: + raise NoSolutionError() + + if solution_number != 0: + return _load_suboptimal_mip_solution( + solver_model=solver_model, + var_map=var_map, + vars_to_load=vars_to_load, + solution_number=solution_number, + ) + + gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + vals = solver_model.getAttr("X", gurobi_vars_to_load) + + res = ComponentMap() + for var, val in zip(vars_to_load, vals): + res[var] = val + return res + + +def _get_reduced_costs(solver_model, var_map, vars_to_load): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + """ + if solver_model.Status != gurobipy.GRB.OPTIMAL: + raise NoReducedCostsError() + + res = ComponentMap() + gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + vals = solver_model.getAttr("Rc", gurobi_vars_to_load) + + for var, val in zip(vars_to_load, vals): + res[var] = val + + return res + + +def _get_duals(solver_model, con_map, linear_cons_to_load, quadratic_cons_to_load): + """ + solver_model: gurobipy.Model + con_map: Dict[ConstraintData, gurobipy.Constr] + Maps the pyomo constraint to the gurobipy constraint + linear_cons_to_load: List[ConstraintData] + quadratic_cons_to_load: List[ConstraintData] + """ + if solver_model.Status != gurobipy.GRB.OPTIMAL: + raise NoDualsError() + + linear_gurobi_cons = [con_map[c] for c in linear_cons_to_load] + quadratic_gurobi_cons = [con_map[c] for c in quadratic_cons_to_load] + linear_vals = solver_model.getAttr("Pi", linear_gurobi_cons) + quadratic_vals = solver_model.getAttr("QCPi", quadratic_gurobi_cons) + + duals = {} + for c, val in zip(linear_cons_to_load, linear_vals): + duals[c] = val + for c, val in zip(quadratic_cons_to_load, quadratic_vals): + duals[c] = val + return duals + + class GurobiDirectBase(SolverBase): _num_gurobipy_env_clients = 0 @@ -191,10 +307,15 @@ def solve(self, model, **kwds) -> Results: orig_config = self.config orig_cwd = os.getcwd() try: - self.config = config = self.config( + config = self.config( value=kwds, preserve_implicit=True, ) + + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = config + object.__setattr__(self, 'config', config) + if not self.available(): c = self.__class__ raise ApplicationError( @@ -237,13 +358,17 @@ def solve(self, model, **kwds) -> Results: res = self._postsolve( grb_model=gurobi_model, + solution_loader=solution_loader, has_obj=has_obj, ) finally: os.chdir(orig_cwd) + + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = orig_config + object.__setattr__(self, 'config', orig_config) self.config = orig_config - res.solution_loader = solution_loader res.solver_log = ostreams[0].getvalue() end_timestamp = datetime.datetime.now(datetime.timezone.utc) res.timing_info.start_timestamp = start_timestamp @@ -273,10 +398,11 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map - def _postsolve(self, grb_model, has_obj): + def _postsolve(self, grb_model, solution_loader, has_obj): status = grb_model.Status results = Results() + results.solution_loader = solution_loader results.timing_info.gurobi_time = grb_model.Runtime if grb_model.SolCount > 0: diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index fa269c1d3c5..3ae6e86526c 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -12,7 +12,7 @@ import io import logging import math -from typing import List, Optional +from typing import Dict, List, NoReturn, Optional, Sequence, Mapping from collections.abc import Iterable from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet @@ -53,9 +53,18 @@ PersistentSolverUtils, PersistentSolverMixin, ) -from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader +from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader, SolutionLoaderBase from pyomo.core.staleflag import StaleFlagManager -from .gurobi_direct_base import GurobiConfig, GurobiDirectBase, gurobipy +from .gurobi_direct_base import ( + GurobiConfig, + GurobiDirectBase, + gurobipy, + _load_suboptimal_mip_solution, + _load_vars, + _get_primals, + _get_duals, + _get_reduced_costs, +) from pyomo.contrib.solver.common.util import get_objective from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor @@ -63,7 +72,87 @@ logger = logging.getLogger(__name__) -class GurobiSolutionLoader(PersistentSolutionLoader): +class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): + def __init__( + self, + solver_model, + var_id_map, + var_map, + con_map, + linear_cons, + quadratic_cons, + ) -> None: + super().__init__() + self._solver_model = solver_model + self._vars = var_id_map + self._var_map = var_map + self._con_map = con_map + self._linear_cons = linear_cons + self._quadratic_cons = quadratic_cons + + def load_vars( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=0, + ) -> None: + if vars_to_load is None: + vars_to_load = list(self._vars.values()) + _load_vars( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + solution_number=solution_id, + ) + + def get_primals( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=0, + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = list(self._vars.values()) + return _get_primals( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + solution_number=solution_id, + ) + + def get_reduced_costs( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = list(self._vars.values()) + return _get_reduced_costs( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + ) + + def get_duals( + self, + cons_to_load: Optional[Sequence[ConstraintData]] = None, + ) -> Dict[ConstraintData, float]: + if cons_to_load is None: + cons_to_load = list(self._con_map.keys()) + linear_cons_to_load = [] + quadratic_cons_to_load = [] + for c in cons_to_load: + if c in self._linear_cons: + linear_cons_to_load.append(c) + else: + assert c in self._quadratic_cons + quadratic_cons_to_load.append(c) + return _get_duals( + solver_model=self._solver_model, + con_map=self._con_map, + linear_cons_to_load=linear_cons_to_load, + quadratic_cons_to_load=quadratic_cons_to_load, + ) + + +class GurobiPersistentSolutionLoader(PersistentSolutionLoader): def load_vars(self, vars_to_load=None, solution_number=0): self._assert_solution_still_valid() self._solver._load_vars( @@ -212,23 +301,50 @@ def __init__(self, **kwds): self._vars = {} # from id(v) to v self._pyomo_var_to_solver_var_map = {} self._pyomo_con_to_solver_con_map = {} + self._linear_cons = set() + self._quadratic_cons = set() self._pyomo_sos_to_solver_sos_map = {} def _create_solver_model(self, pyomo_model): + timer = self.config.timer + timer.start('create gurobipy model') self._clear() self._solver_model = gurobipy.Model(env=self.env()) + timer.start('collect constraints') cons = list(pyomo_model.component_data_objects(Constraint, descend_into=True, active=True)) + timer.stop('collect constraints') + timer.start('translate constraints') self._add_constraints(cons) + timer.stop('translate constraints') + timer.start('sos') sos = list(pyomo_model.component_data_objects(SOSConstraint, descend_into=True, active=True)) self._add_sos_constraints(sos) + timer.stop('sos') + timer.start('get objective') obj = get_objective(pyomo_model) + timer.stop('get objective') + timer.start('translate objective') self._set_objective(obj) + timer.stop('translate objective') + has_obj = obj is not None + solution_loader = GurobiDirectQuadraticSolutionLoader( + solver_model=self._solver_model, + var_id_map=self._vars, + var_map=self._pyomo_var_to_solver_var_map, + con_map=self._pyomo_con_to_solver_con_map, + linear_cons=self._linear_cons, + quadratic_cons=self._quadratic_cons, + ) + timer.stop('create gurobipy model') + return self._solver_model, solution_loader, has_obj def _clear(self): self._solver_model = None self._vars = {} self._pyomo_var_to_solver_var_map = {} self._pyomo_con_to_solver_con_map = {} + self._linear_cons = set() + self._quadratic_cons = set() self._pyomo_sos_to_solver_sos_map = {} def _pyomo_gurobi_var_iter(self): @@ -256,8 +372,10 @@ def _process_domain_and_bounds(self, var): lb = var.value ub = lb else: - lb = max(lb, value(var._lb)) - ub = min(ub, value(var._ub)) + if var._lb is not None: + lb = max(lb, value(var._lb)) + if var._ub is not None: + ub = min(ub, value(var._ub)) return lb, ub, vtype def _add_variables(self, variables: List[VarData]): @@ -273,14 +391,12 @@ def _add_variables(self, variables: List[VarData]): gurobi_vars = self._solver_model.addVars( len(variables), lb=lbs, ub=ubs, vtype=vtypes - ) + ).values() for pyomo_var, gurobi_var in zip(variables, gurobi_vars): self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var - def _get_expr_from_pyomo_expr(self, expr): - repn = generate_standard_repn(expr, quadratic=True, compute_values=True) - + def _get_expr_from_pyomo_repn(self, repn): if repn.nonlinear_expr is not None: raise IncompatibleModelError( f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' @@ -289,18 +405,26 @@ def _get_expr_from_pyomo_expr(self, expr): if len(repn.linear_vars) > 0: missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] self._add_variables(missing_vars) + vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr( repn.linear_coefs, - [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars], + vlist, ) else: new_expr = 0.0 - for coef, v in zip(repn.quadratic_coefs, repn.quadratic_vars): - x, y = v - gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] - gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] - new_expr += coef * gurobi_x * gurobi_y + if len(repn.quadratic_vars) > 0: + missing_vars = {} + for x, y in repn.quadratic_vars: + for v in [x, y]: + vid = id(v) + if vid not in self._vars: + missing_vars[vid] = v + self._add_variables(list(missing_vars.values())) + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] + gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] + new_expr += coef * gurobi_x * gurobi_y return new_expr, repn.constant @@ -308,26 +432,72 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobi_expr_list = [] for con in cons: lb, body, ub = con.to_bounded_expression(evaluate_bounds=True) - gurobi_expr, repn_constant = self._get_expr_from_pyomo_expr(body) + repn = generate_standard_repn(body, quadratic=True, compute_values=True) + if len(repn.quadratic_vars) > 0: + self._quadratic_cons.add(con) + else: + self._linear_cons.add(con) + gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) if lb is None and ub is None: raise ValueError( "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: - gurobi_expr_list.append(gurobi_expr <= ub - repn_constant) + gurobi_expr_list.append(gurobi_expr <= float(ub - repn_constant)) elif ub is None: - gurobi_expr_list.append(lb - repn_constant <= gurobi_expr) + gurobi_expr_list.append(float(lb - repn_constant) <= gurobi_expr) elif lb == ub: - gurobi_expr_list.append(gurobi_expr == lb - repn_constant) + gurobi_expr_list.append(gurobi_expr == float(lb - repn_constant)) else: - gurobi_expr_list.append(gurobi_expr == [lb-repn_constant, ub-repn_constant]) + gurobi_expr_list.append(gurobi_expr == [float(lb-repn_constant), float(ub-repn_constant)]) - gurobi_cons = self._solver_model.addConstrs(gurobi_expr_list) + gurobi_cons = self._solver_model.addConstrs((gurobi_expr_list[i] for i in range(len(gurobi_expr_list)))).values() self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) - self._pyomo_con_to_solver_con_map[con] = gurobipy_con - self._solver_con_to_pyomo_con_map[id(gurobipy_con)] = con - self._constraints_added_since_update.update(cons) + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + for con in cons: + level = con.level + if level == 1: + sos_type = gurobipy.GRB.SOS_TYPE1 + elif level == 2: + sos_type = gurobipy.GRB.SOS_TYPE2 + else: + raise ValueError( + f"Solver does not support SOS level {level} constraints" + ) + + gurobi_vars = [] + weights = [] + + missing_vars = {id(v): v for v, w in con.get_items() if id(v) not in self._vars} + self._add_variables(list(missing_vars.values())) + + for v, w in con.get_items(): + v_id = id(v) + gurobi_vars.append(self._pyomo_var_to_solver_var_map[v_id]) + weights.append(w) + + gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) + self._pyomo_sos_to_solver_sos_map[con] = gurobipy_con + + def _set_objective(self, obj): + if obj is None: + sense = gurobipy.GRB.MINIMIZE + gurobi_expr = 0 + repn_constant = 0 + else: + if obj.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + elif obj.sense == maximize: + sense = gurobipy.GRB.MAXIMIZE + else: + raise ValueError(f'Objective sense is not recognized: {obj.sense}') + + repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=True) + gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) + + self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) self._needs_updated = True diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 5ab36554061..748e0127151 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -30,8 +30,9 @@ from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.solvers.ipopt import Ipopt -from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent -from pyomo.contrib.solver.solvers.gurobi_direct import GurobiDirect +# from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -47,20 +48,31 @@ raise unittest.SkipTest('Parameterized is not available.') all_solvers = [ - ('gurobi_persistent', GurobiPersistent), + # ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('highs', Highs), ] mip_solvers = [ - ('gurobi_persistent', GurobiPersistent), + # ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), ] -nlp_solvers = [('ipopt', Ipopt)] -qcp_solvers = [('gurobi_persistent', GurobiPersistent), ('ipopt', Ipopt)] +nlp_solvers = [ + ('ipopt', Ipopt), +] +qcp_solvers = [ + # ('gurobi_persistent', GurobiPersistent), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), + ('ipopt', Ipopt), +] qp_solvers = qcp_solvers + [("highs", Highs)] -miqcqp_solvers = [('gurobi_persistent', GurobiPersistent)] +miqcqp_solvers = [ + # ('gurobi_persistent', GurobiPersistent), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), +] nl_solvers = [('ipopt', Ipopt)] nl_solvers_set = {i[0] for i in nl_solvers} From 909be8815c6c585a5c2c700b9444599868c26598 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:02:49 -0600 Subject: [PATCH 06/37] refactoring gurobi interfaces --- pyomo/contrib/observer/model_observer.py | 69 +- pyomo/contrib/solver/plugins.py | 3 +- .../solvers/gurobi/gurobi_persistent.py | 934 +++++++++++++++++- .../solver/tests/solvers/test_solvers.py | 11 +- 4 files changed, 925 insertions(+), 92 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 422eb1da574..8f7238c2ee9 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -162,9 +162,6 @@ def __init__( class Observer(abc.ABC): - def __init__(self): - pass - @abc.abstractmethod def add_variables(self, variables: List[VarData]): pass @@ -255,7 +252,7 @@ def set_instance(self, model): self._model = model self._add_block(model) - def _add_variables(self, variables: List[VarData]): + def add_variables(self, variables: List[VarData]): for v in variables: if id(v) in self._referenced_variables: raise ValueError(f'Variable {v.name} has already been added') @@ -271,7 +268,7 @@ def _add_variables(self, variables: List[VarData]): for obs in self._observers: obs.add_variables(variables) - def _add_parameters(self, params: List[ParamData]): + def add_parameters(self, params: List[ParamData]): for p in params: pid = id(p) if pid in self._referenced_params: @@ -287,7 +284,7 @@ def _check_for_new_vars(self, variables: List[VarData]): v_id = id(v) if v_id not in self._referenced_variables: new_vars[v_id] = v - self._add_variables(list(new_vars.values())) + self.add_variables(list(new_vars.values())) def _check_to_remove_vars(self, variables: List[VarData]): vars_to_remove = {} @@ -296,7 +293,7 @@ def _check_to_remove_vars(self, variables: List[VarData]): ref_cons, ref_sos, ref_obj = self._referenced_variables[v_id] if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: vars_to_remove[v_id] = v - self._remove_variables(list(vars_to_remove.values())) + self.remove_variables(list(vars_to_remove.values())) def _check_for_new_params(self, params: List[ParamData]): new_params = {} @@ -304,7 +301,7 @@ def _check_for_new_params(self, params: List[ParamData]): pid = id(p) if pid not in self._referenced_params: new_params[pid] = p - self._add_parameters(list(new_params.values())) + self.add_parameters(list(new_params.values())) def _check_to_remove_params(self, params: List[ParamData]): params_to_remove = {} @@ -313,9 +310,9 @@ def _check_to_remove_params(self, params: List[ParamData]): ref_cons, ref_sos, ref_obj = self._referenced_params[p_id] if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: params_to_remove[p_id] = p - self._remove_parameters(list(params_to_remove.values())) + self.remove_parameters(list(params_to_remove.values())) - def _add_constraints(self, cons: List[ConstraintData]): + def add_constraints(self, cons: List[ConstraintData]): vars_to_check = [] params_to_check = [] for con in cons: @@ -343,7 +340,7 @@ def _add_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.add_constraints(cons) - def _add_sos_constraints(self, cons: List[SOSConstraintData]): + def add_sos_constraints(self, cons: List[SOSConstraintData]): vars_to_check = [] params_to_check = [] for con in cons: @@ -373,7 +370,7 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.add_sos_constraints(cons) - def _set_objective(self, obj: Optional[ObjectiveData]): + def set_objective(self, obj: Optional[ObjectiveData]): vars_to_remove_check = [] params_to_remove_check = [] if self._objective is not None: @@ -414,12 +411,12 @@ def _set_objective(self, obj: Optional[ObjectiveData]): self._check_to_remove_params(params_to_remove_check) def _add_block(self, block): - self._add_constraints( + self.add_constraints( list( block.component_data_objects(Constraint, descend_into=True, active=True) ) ) - self._add_sos_constraints( + self.add_sos_constraints( list( block.component_data_objects( SOSConstraint, descend_into=True, active=True @@ -427,9 +424,9 @@ def _add_block(self, block): ) ) obj = get_objective(block) - self._set_objective(obj) + self.set_objective(obj) - def _remove_constraints(self, cons: List[ConstraintData]): + def remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.remove_constraints(cons) vars_to_check = [] @@ -453,7 +450,7 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._check_to_remove_vars(vars_to_check) self._check_to_remove_params(params_to_check) - def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + def remove_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.remove_sos_constraints(cons) vars_to_check = [] @@ -476,7 +473,7 @@ def _remove_sos_constraints(self, cons: List[SOSConstraintData]): self._check_to_remove_vars(vars_to_check) self._check_to_remove_params(params_to_check) - def _remove_variables(self, variables: List[VarData]): + def remove_variables(self, variables: List[VarData]): for obs in self._observers: obs.remove_variables(variables) for v in variables: @@ -493,7 +490,7 @@ def _remove_variables(self, variables: List[VarData]): del self._referenced_variables[v_id] del self._vars[v_id] - def _remove_parameters(self, params: List[ParamData]): + def remove_parameters(self, params: List[ParamData]): for obs in self._observers: obs.remove_parameters(params) for p in params: @@ -510,7 +507,7 @@ def _remove_parameters(self, params: List[ParamData]): del self._referenced_params[p_id] del self._params[p_id] - def _update_variables(self, variables: List[VarData]): + def update_variables(self, variables: List[VarData]): for v in variables: self._vars[id(v)] = ( v, @@ -523,7 +520,7 @@ def _update_variables(self, variables: List[VarData]): for obs in self._observers: obs.update_variables(variables) - def _update_parameters(self, params): + def update_parameters(self, params): for p in params: self._params[id(p)] = (p, p.value) for obs in self._observers: @@ -668,28 +665,28 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.check_for_new_or_removed_constraints: timer.start('sos') new_sos, old_sos = self._check_for_new_or_removed_sos() - self._add_sos_constraints(new_sos) - self._remove_sos_constraints(old_sos) + self.add_sos_constraints(new_sos) + self.remove_sos_constraints(old_sos) added_sos.update(new_sos) timer.stop('sos') timer.start('cons') new_cons, old_cons = self._check_for_new_or_removed_constraints() - self._add_constraints(new_cons) - self._remove_constraints(old_cons) + self.add_constraints(new_cons) + self.remove_constraints(old_cons) added_cons.update(new_cons) timer.stop('cons') if config.update_constraints: timer.start('cons') cons_to_update = self._check_for_modified_constraints() - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) added_cons.update(cons_to_update) timer.stop('cons') timer.start('sos') sos_to_update = self._check_for_modified_sos() - self._remove_sos_constraints(sos_to_update) - self._add_sos_constraints(sos_to_update) + self.remove_sos_constraints(sos_to_update) + self.add_sos_constraints(sos_to_update) added_sos.update(sos_to_update) timer.stop('sos') @@ -698,10 +695,10 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.update_vars: timer.start('vars') vars_to_update, cons_to_update, update_obj = self._check_for_var_changes() - self._update_variables(vars_to_update) + self.update_variables(vars_to_update) cons_to_update = [i for i in cons_to_update if i not in added_cons] - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) added_cons.update(cons_to_update) if update_obj: need_to_set_objective = True @@ -711,8 +708,8 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): timer.start('named expressions') cons_to_update, update_obj = self._check_for_named_expression_changes() cons_to_update = [i for i in cons_to_update if i not in added_cons] - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) added_cons.update(cons_to_update) if update_obj: need_to_set_objective = True @@ -730,11 +727,11 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): need_to_set_objective = True if need_to_set_objective: - self._set_objective(new_obj) + self.set_objective(new_obj) timer.stop('objective') if config.update_parameters: timer.start('params') params_to_update = self._check_for_param_changes() - self._update_parameters(params_to_update) + self.update_parameters(params_to_update) timer.stop('params') diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 7630c614aa2..f29c4f61c4e 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -12,9 +12,8 @@ from .common.factory import SolverFactory from .solvers.ipopt import Ipopt -from .solvers.gurobi_persistent import GurobiPersistent from .solvers.gurobi.gurobi_direct import GurobiDirect -from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic +from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from .solvers.highs import Highs diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 3ae6e86526c..844502ca476 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -9,6 +9,7 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +from __future__ import annotations import io import logging import math @@ -21,6 +22,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer from pyomo.common.shutdown import python_is_shutting_down +from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler from pyomo.core.base.var import VarData @@ -67,6 +69,7 @@ ) from pyomo.contrib.solver.common.util import get_objective from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector logger = logging.getLogger(__name__) @@ -152,18 +155,33 @@ def get_duals( ) -class GurobiPersistentSolutionLoader(PersistentSolutionLoader): - def load_vars(self, vars_to_load=None, solution_number=0): +class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): + def __init__(self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) -> None: + super().__init__(solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) + self._valid = True + + def invalidate(self): + self._valid = False + + def _assert_solution_still_valid(self): + if not self._valid: + raise RuntimeError('The results in the solver are no longer valid.') + + def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> None: self._assert_solution_still_valid() - self._solver._load_vars( - vars_to_load=vars_to_load, solution_number=solution_number - ) + return super().load_vars(vars_to_load, solution_id) + + def get_primals(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_primals(vars_to_load, solution_id) - def get_primals(self, vars_to_load=None, solution_number=0): + def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() - return self._solver._get_primals( - vars_to_load=vars_to_load, solution_number=solution_number - ) + return super().get_duals(cons_to_load) + + def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_reduced_costs(vars_to_load) class _MutableLowerBound: @@ -187,46 +205,61 @@ def update(self): class _MutableLinearCoefficient: - def __init__(self): - self.expr = None - self.var = None - self.con = None - self.gurobi_model = None + def __init__(self, expr, pyomo_con, con_map, pyomo_var_id, var_map, gurobi_model): + self.expr = expr + self.pyomo_con = pyomo_con + self.pyomo_var_id = pyomo_var_id + self.con_map = con_map + self.var_map = var_map + self.gurobi_model = gurobi_model + + @property + def gurobi_var(self): + return self.var_map[self.pyomo_var_id] + + @property + def gurobi_con(self): + return self.con_map[self.pyomo_con] def update(self): - self.gurobi_model.chgCoeff(self.con, self.var, value(self.expr)) + self.gurobi_model.chgCoeff(self.gurobi_con, self.gurobi_var, value(self.expr)) class _MutableRangeConstant: - def __init__(self): - self.lhs_expr = None - self.rhs_expr = None - self.con = None - self.slack_name = None - self.gurobi_model = None + def __init__(self, lhs_expr, rhs_expr, pyomo_con, con_map, slack_name, gurobi_model): + self.lhs_expr = lhs_expr + self.rhs_expr = rhs_expr + self.pyomo_con = pyomo_con + self.con_map = con_map + self.slack_name = slack_name + self.gurobi_model = gurobi_model def update(self): rhs_val = value(self.rhs_expr) lhs_val = value(self.lhs_expr) - self.con.rhs = rhs_val + con = self.con_map[self.pyomo_con] + con.rhs = rhs_val slack = self.gurobi_model.getVarByName(self.slack_name) slack.ub = rhs_val - lhs_val class _MutableConstant: - def __init__(self): - self.expr = None - self.con = None + def __init__(self, expr, pyomo_con, con_map): + self.expr = expr + self.pyomo_con = pyomo_con + self.con_map = con_map def update(self): - self.con.rhs = value(self.expr) + con = self.con_map[self.pyomo_con] + con.rhs = value(self.expr) class _MutableQuadraticConstraint: def __init__( - self, gurobi_model, gurobi_con, constant, linear_coefs, quadratic_coefs + self, gurobi_model, pyomo_con, con_map, constant, linear_coefs, quadratic_coefs ): - self.con = gurobi_con + self.pyomo_con = pyomo_con + self.con_map = con_map self.gurobi_model = gurobi_model self.constant = constant self.last_constant_value = value(self.constant.expr) @@ -235,8 +268,12 @@ def __init__( self.quadratic_coefs = quadratic_coefs self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + @property + def gurobi_con(self): + return self.con_map[self.pyomo_con] + def get_updated_expression(self): - gurobi_expr = self.gurobi_model.getQCRow(self.con) + gurobi_expr = self.gurobi_model.getQCRow(self.gurobi_con) for ndx, coef in enumerate(self.linear_coefs): current_coef_value = value(coef.expr) incremental_coef_value = ( @@ -260,14 +297,14 @@ def get_updated_rhs(self): class _MutableObjective: def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): self.gurobi_model = gurobi_model - self.constant = constant - self.linear_coefs = linear_coefs - self.quadratic_coefs = quadratic_coefs - self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + self.constant: _MutableConstant = constant + self.linear_coefs: List[_MutableLinearCoefficient] = linear_coefs + self.quadratic_coefs: List[_MutableQuadraticCoefficient] = quadratic_coefs + self.last_quadratic_coef_values: List[float] = [value(i.expr) for i in self.quadratic_coefs] def get_updated_expression(self): for ndx, coef in enumerate(self.linear_coefs): - coef.var.obj = value(coef.expr) + coef.gurobi_var.obj = value(coef.expr) self.gurobi_model.ObjCon = value(self.constant.expr) gurobi_expr = None @@ -286,10 +323,19 @@ def get_updated_expression(self): class _MutableQuadraticCoefficient: - def __init__(self): + def __init__(self, expr, v1id, v2id, var_map): self.expr = None - self.var1 = None - self.var2 = None + self.var_map = var_map + self.v1id = v1id + self.v2id = v2id + + @property + def var1(self): + return self.var_map[self.v1id] + + @property + def var2(self): + return self.var_map[self.v2id] class GurobiDirectQuadratic(GurobiDirectBase): @@ -405,9 +451,10 @@ def _get_expr_from_pyomo_repn(self, repn): if len(repn.linear_vars) > 0: missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] self._add_variables(missing_vars) + coef_list = [value(i) for i in repn.linear_coefs] vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr( - repn.linear_coefs, + coef_list, vlist, ) else: @@ -424,9 +471,9 @@ def _get_expr_from_pyomo_repn(self, repn): for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] - new_expr += coef * gurobi_x * gurobi_y + new_expr += value(coef) * gurobi_x * gurobi_y - return new_expr, repn.constant + return new_expr def _add_constraints(self, cons: List[ConstraintData]): gurobi_expr_list = [] @@ -437,20 +484,20 @@ def _add_constraints(self, cons: List[ConstraintData]): self._quadratic_cons.add(con) else: self._linear_cons.add(con) - gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) if lb is None and ub is None: raise ValueError( "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: - gurobi_expr_list.append(gurobi_expr <= float(ub - repn_constant)) + gurobi_expr_list.append(gurobi_expr <= float(ub - repn.constant)) elif ub is None: - gurobi_expr_list.append(float(lb - repn_constant) <= gurobi_expr) + gurobi_expr_list.append(float(lb - repn.constant) <= gurobi_expr) elif lb == ub: - gurobi_expr_list.append(gurobi_expr == float(lb - repn_constant)) + gurobi_expr_list.append(gurobi_expr == float(lb - repn.constant)) else: - gurobi_expr_list.append(gurobi_expr == [float(lb-repn_constant), float(ub-repn_constant)]) + gurobi_expr_list.append(gurobi_expr == [float(lb-repn.constant), float(ub-repn.constant)]) gurobi_cons = self._solver_model.addConstrs((gurobi_expr_list[i] for i in range(len(gurobi_expr_list)))).values() self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) @@ -495,11 +542,802 @@ def _set_objective(self, obj): raise ValueError(f'Objective sense is not recognized: {obj.sense}') repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=True) - gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + repn_constant = repn.constant self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) - self._needs_updated = True -class GurobiPersistentQuadratic(GurobiDirectQuadratic): +class _GurobiObserver(Observer): + def __init__(self, opt: GurobiPersistentQuadratic) -> None: + self.opt = opt + + def add_variables(self, variables: List[VarData]): + self.opt._add_variables(variables) + + def add_parameters(self, params: List[ParamData]): + pass + + def add_constraints(self, cons: List[ConstraintData]): + self.opt._add_constraints(cons) + + def add_sos_constraints(self, cons: List[SOSConstraintData]): + self.opt._add_sos_constraints(cons) + + def set_objective(self, obj: ObjectiveData | None): + self.opt._set_objective(obj) + + def remove_constraints(self, cons: List[ConstraintData]): + self.opt._remove_constraints(cons) + + def remove_sos_constraints(self, cons: List[SOSConstraintData]): + self.opt._remove_sos_constraints(cons) + + def remove_variables(self, variables: List[VarData]): + self.opt._remove_variables(variables) + + def remove_parameters(self, params: List[ParamData]): + pass + + def update_variables(self, variables: List[VarData]): + self.opt._update_variables(variables) + + def update_parameters(self, params: List[ParamData]): + self.opt._update_parameters(params) + + +class GurobiPersistent(GurobiDirectQuadratic): _minimum_version = (7, 0, 0) + + def __init__(self, **kwds): + super().__init__(**kwds) + self._pyomo_model = None + self._objective = None + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True + self._callback_func = None + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._last_results_object: Optional[Results] = None + self._observer = _GurobiObserver(self) + self._change_detector = ModelChangeDetector(observers=[self._observer]) + self._constraint_ndx = 0 + + @property + def auto_updates(self): + return self._change_detector.config + + def _clear(self): + super()._clear() + self._pyomo_model = None + self._objective = None + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._last_results_object = None + self._constraint_ndx = 0 + + def _create_solver_model(self, pyomo_model): + if pyomo_model is self._pyomo_model: + self.update() + else: + self.set_instance(pyomo_model) + + solution_loader = GurobiPersistentSolutionLoader( + solver_model=self._solver_model, + var_id_map=self._vars, + var_map=self._pyomo_var_to_solver_var_map, + con_map=self._pyomo_con_to_solver_con_map, + linear_cons=self._linear_cons, + quadratic_cons=self._quadratic_cons, + ) + has_obj = self._objective is not None + return self._solver_model, solution_loader, has_obj + + def release_license(self): + self._clear() + self.__class__.release_license() + + def solve(self, model, **kwds) -> Results: + res = super().solve(model, **kwds) + self._needs_updated = False + return res + + def _process_domain_and_bounds(self, var): + res = super()._process_domain_and_bounds(var) + if not is_constant(var._lb): + mutable_lb = _MutableLowerBound(id(var), var.lower, self._pyomo_var_to_solver_var_map) + self._mutable_bounds[id(var), 'lb'] = (var, mutable_lb) + if not is_constant(var._ub): + mutable_ub = _MutableUpperBound(id(var), var.upper, self._pyomo_var_to_solver_var_map) + self._mutable_bounds[id(var), 'ub'] = (var, mutable_ub) + return res + + def _add_variables(self, variables: List[VarData]): + self._invalidate_last_results() + super()._add_variables(variables) + self._vars_added_since_update.update(variables) + self._needs_updated = True + + def set_instance(self, pyomo_model): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + self._clear() + self._pyomo_model = pyomo_model + self._solver_model = gurobipy.Model(env=self.env()) + timer.start('set_instance') + self._change_detector.set_instance(pyomo_model) + timer.stop('set_instance') + + def update(self): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + if self._pyomo_model is None: + raise RuntimeError('must call set_instance or solve before update') + timer.start('update') + if self._needs_updated: + self._update_gurobi_model() + self._change_detector.update(timer=timer) + timer.stop('update') + + def _add_constraints(self, cons: List[ConstraintData]): + self._invalidate_last_results() + gurobi_expr_list = [] + for ndx, con in enumerate(cons): + lb, body, ub = con.to_bounded_expression(evaluate_bounds=False) + repn = generate_standard_repn(body, quadratic=True, compute_values=False) + if len(repn.quadratic_vars) > 0: + self._quadratic_cons.add(con) + else: + self._linear_cons.add(con) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + mutable_constant = None + if lb is None and ub is None: + raise ValueError( + "Constraint does not have a lower " + f"or an upper bound: {con} \n" + ) + elif lb is None: + rhs_expr = ub - repn.constant + gurobi_expr_list.append(gurobi_expr <= float(value(rhs_expr))) + if not is_constant(rhs_expr): + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + elif ub is None: + rhs_expr = lb - repn.constant + gurobi_expr_list.append(float(value(rhs_expr)) <= gurobi_expr) + if not is_constant(rhs_expr): + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + elif con.equality: + rhs_expr = lb - repn.constant + gurobi_expr_list.append(gurobi_expr == float(value(rhs_expr))) + if not is_constant(rhs_expr): + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + else: + assert len(repn.quadratic_vars) == 0, "Quadratic range constraints are not supported" + lhs_expr = lb - repn.constant + rhs_expr = ub - repn.constant + gurobi_expr_list.append(gurobi_expr == [float(value(lhs_expr)), float(value(rhs_expr))]) + if not is_constant(lhs_expr) or not is_constant(rhs_expr): + conname = f'c{self._constraint_ndx}[{ndx}]' + mutable_constant = _MutableRangeConstant(lhs_expr, rhs_expr, con, self._pyomo_con_to_solver_con_map, 'Rg' + conname, self._solver_model) + + mlc_list = [] + for c, v in zip(repn.linear_coefs, repn.linear_vars): + if not is_constant(c): + mlc = _MutableLinearCoefficient(c, con, self._pyomo_con_to_solver_con_map, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc_list.append(mlc) + + if len(repn.quadratic_vars) == 0: + if len(mlc_list) > 0: + self._mutable_helpers[con] = mlc_list + if mutable_constant is not None: + if con not in self._mutable_helpers: + self._mutable_helpers[con] = [] + self._mutable_helpers[con].append(mutable_constant) + else: + if mutable_constant is None: + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mqc_list = [] + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + if not is_constant(coef): + mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc_list.append(mqc) + mqc = _MutableQuadraticConstraint( + self._solver_model, + con, + self._pyomo_con_to_solver_con_map, + mutable_constant, + mlc_list, + mqc_list, + ) + self._mutable_quadratic_helpers[con] = mqc + + gurobi_cons = list(self._solver_model.addConstrs( + (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))), + name=f'c{self._constraint_ndx}' + ).values()) + self._constraint_ndx += 1 + self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) + self._constraints_added_since_update.update(cons) + self._needs_updated = True + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + self._invalidate_last_results() + super()._add_sos_constraints(cons) + self._constraints_added_since_update.update(cons) + self._needs_updated = True + + def _set_objective(self, obj): + self._invalidate_last_results() + if obj is None: + sense = gurobipy.GRB.MINIMIZE + gurobi_expr = 0 + repn_constant = 0 + self._mutable_objective = None + else: + if obj.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + elif obj.sense == maximize: + sense = gurobipy.GRB.MAXIMIZE + else: + raise ValueError(f'Objective sense is not recognized: {obj.sense}') + + repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=False) + repn_constant = value(repn.constant) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + + mutable_constant = _MutableConstant(repn.constant, None, None) + + mlc_list = [] + for c, v in zip(repn.linear_coefs, repn.linear_vars): + if not is_constant(c): + mlc = _MutableLinearCoefficient(c, None, None, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc_list.append(mlc) + + mqc_list = [] + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + if not is_constant(coef): + mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc_list.append(mqc) + + self._mutable_objective = _MutableObjective(self._solver_model, mutable_constant, mlc_list, mqc_list) + + # hack + # see PR #2454 + if self._objective is not None: + self._solver_model.setObjective(0) + self._solver_model.update() + + self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) + self._objective = obj + self._needs_updated = True + + def _update_gurobi_model(self): + self._solver_model.update() + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._needs_updated = False + + def _remove_constraints(self, cons: List[ConstraintData]): + self._invalidate_last_results() + for con in cons: + if con in self._constraints_added_since_update: + self._update_gurobi_model() + solver_con = self._pyomo_con_to_solver_con_map[con] + self._solver_model.remove(solver_con) + del self._pyomo_con_to_solver_con_map[con] + self._mutable_helpers.pop(con, None) + self._mutable_quadratic_helpers.pop(con, None) + self._needs_updated = True + + def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + self._invalidate_last_results() + for con in cons: + if con in self._constraints_added_since_update: + self._update_gurobi_model() + solver_sos_con = self._pyomo_sos_to_solver_sos_map[con] + self._solver_model.remove(solver_sos_con) + del self._pyomo_sos_to_solver_sos_map[con] + self._needs_updated = True + + def _remove_variables(self, variables: List[VarData]): + self._invalidate_last_results() + for var in variables: + v_id = id(var) + if var in self._vars_added_since_update: + self._update_gurobi_model() + solver_var = self._pyomo_var_to_solver_var_map[v_id] + self._solver_model.remove(solver_var) + del self._pyomo_var_to_solver_var_map[v_id] + self._mutable_bounds.pop(v_id, None) + self._needs_updated = True + + def _update_variables(self, variables: List[VarData]): + self._invalidate_last_results() + for var in variables: + var_id = id(var) + if var_id not in self._pyomo_var_to_solver_var_map: + raise ValueError( + f'The Var provided to update_var needs to be added first: {var}' + ) + self._mutable_bounds.pop((var_id, 'lb'), None) + self._mutable_bounds.pop((var_id, 'ub'), None) + gurobipy_var = self._pyomo_var_to_solver_var_map[var_id] + lb, ub, vtype = self._process_domain_and_bounds(var) + gurobipy_var.setAttr('lb', lb) + gurobipy_var.setAttr('ub', ub) + gurobipy_var.setAttr('vtype', vtype) + self._needs_updated = True + + def _update_parameters(self, params: List[ParamData]): + self._invalidate_last_results() + for con, helpers in self._mutable_helpers.items(): + for helper in helpers: + helper.update() + for k, (v, helper) in self._mutable_bounds.items(): + helper.update() + + for con, helper in self._mutable_quadratic_helpers.items(): + if con in self._constraints_added_since_update: + self._update_gurobi_model() + gurobi_con = helper.gurobi_con + new_gurobi_expr = helper.get_updated_expression() + new_rhs = helper.get_updated_rhs() + new_sense = gurobi_con.qcsense + self._solver_model.remove(gurobi_con) + new_con = self._solver_model.addQConstr( + new_gurobi_expr, new_sense, new_rhs, + ) + self._pyomo_con_to_solver_con_map[con] = new_con + helper.pyomo_con = con + self._constraints_added_since_update.add(con) + + if self._mutable_objective is not None: + new_gurobi_expr = self._mutable_objective.get_updated_expression() + if new_gurobi_expr is not None: + if self._objective.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + else: + sense = gurobipy.GRB.MAXIMIZE + # TODO: need a test for when part of the object is linear + # and part of the objective is quadratic, but both + # parts have mutable coefficients + self._solver_model.setObjective(new_gurobi_expr, sense=sense) + + def _invalidate_last_results(self): + if self._last_results_object is not None: + self._last_results_object.solution_loader.invalidate() + + def get_model_attr(self, attr): + """ + Get the value of an attribute on the Gurobi model. + + Parameters + ---------- + attr: str + The attribute to get. See Gurobi documentation for descriptions of the attributes. + """ + if self._needs_updated: + self._update_gurobi_model() + return self._solver_model.getAttr(attr) + + def write(self, filename): + """ + Write the model to a file (e.g., and lp file). + + Parameters + ---------- + filename: str + Name of the file to which the model should be written. + """ + self._solver_model.write(filename) + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._needs_updated = False + + def set_linear_constraint_attr(self, con, attr, val): + """ + Set the value of an attribute on a gurobi linear constraint. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The pyomo constraint for which the corresponding gurobi constraint attribute + should be modified. + attr: str + The attribute to be modified. Options are: + CBasis + DStart + Lazy + val: any + See gurobi documentation for acceptable values. + """ + if attr in {'Sense', 'RHS', 'ConstrName'}: + raise ValueError( + f'Linear constraint attr {attr} cannot be set with' + ' the set_linear_constraint_attr method. Please use' + ' the remove_constraint and add_constraint methods.' + ) + self._pyomo_con_to_solver_con_map[con].setAttr(attr, val) + self._needs_updated = True + + def set_var_attr(self, var, attr, val): + """ + Set the value of an attribute on a gurobi variable. + + Parameters + ---------- + var: pyomo.core.base.var.VarData + The pyomo var for which the corresponding gurobi var attribute + should be modified. + attr: str + The attribute to be modified. Options are: + Start + VarHintVal + VarHintPri + BranchPriority + VBasis + PStart + val: any + See gurobi documentation for acceptable values. + """ + if attr in {'LB', 'UB', 'VType', 'VarName'}: + raise ValueError( + f'Var attr {attr} cannot be set with' + ' the set_var_attr method. Please use' + ' the update_var method.' + ) + if attr == 'Obj': + raise ValueError( + 'Var attr Obj cannot be set with' + ' the set_var_attr method. Please use' + ' the set_objective method.' + ) + self._pyomo_var_to_solver_var_map[id(var)].setAttr(attr, val) + self._needs_updated = True + + def get_var_attr(self, var, attr): + """ + Get the value of an attribute on a gurobi var. + + Parameters + ---------- + var: pyomo.core.base.var.VarData + The pyomo var for which the corresponding gurobi var attribute + should be retrieved. + attr: str + The attribute to get. See gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_var_to_solver_var_map[id(var)].getAttr(attr) + + def get_linear_constraint_attr(self, con, attr): + """ + Get the value of an attribute on a gurobi linear constraint. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The pyomo constraint for which the corresponding gurobi constraint attribute + should be retrieved. + attr: str + The attribute to get. See the Gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_con_to_solver_con_map[con].getAttr(attr) + + def get_sos_attr(self, con, attr): + """ + Get the value of an attribute on a gurobi sos constraint. + + Parameters + ---------- + con: pyomo.core.base.sos.SOSConstraintData + The pyomo SOS constraint for which the corresponding gurobi SOS constraint attribute + should be retrieved. + attr: str + The attribute to get. See the Gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_sos_to_solver_sos_map[con].getAttr(attr) + + def get_quadratic_constraint_attr(self, con, attr): + """ + Get the value of an attribute on a gurobi quadratic constraint. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The pyomo constraint for which the corresponding gurobi constraint attribute + should be retrieved. + attr: str + The attribute to get. See the Gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_con_to_solver_con_map[con].getAttr(attr) + + def set_gurobi_param(self, param, val): + """ + Set a gurobi parameter. + + Parameters + ---------- + param: str + The gurobi parameter to set. Options include any gurobi parameter. + Please see the Gurobi documentation for options. + val: any + The value to set the parameter to. See Gurobi documentation for possible values. + """ + self._solver_model.setParam(param, val) + + def get_gurobi_param_info(self, param): + """ + Get information about a gurobi parameter. + + Parameters + ---------- + param: str + The gurobi parameter to get info for. See Gurobi documentation for possible options. + + Returns + ------- + six-tuple containing the parameter name, type, value, minimum value, maximum value, and default value. + """ + return self._solver_model.getParamInfo(param) + + def _intermediate_callback(self): + def f(gurobi_model, where): + self._callback_func(self._pyomo_model, self, where) + + return f + + def set_callback(self, func=None): + """ + Specify a callback for gurobi to use. + + Parameters + ---------- + func: function + The function to call. The function should have three arguments. The first will be the pyomo model being + solved. The second will be the GurobiPersistent instance. The third will be an enum member of + gurobipy.GRB.Callback. This will indicate where in the branch and bound algorithm gurobi is at. For + example, suppose we want to solve + + .. math:: + + min 2*x + y + + s.t. + + y >= (x-2)**2 + + 0 <= x <= 4 + + y >= 0 + + y integer + + as an MILP using extended cutting planes in callbacks. + + >>> from gurobipy import GRB # doctest:+SKIP + >>> import pyomo.environ as pyo + >>> from pyomo.core.expr.taylor_series import taylor_series_expansion + >>> from pyomo.contrib import appsi + >>> + >>> m = pyo.ConcreteModel() + >>> m.x = pyo.Var(bounds=(0, 4)) + >>> m.y = pyo.Var(within=pyo.Integers, bounds=(0, None)) + >>> m.obj = pyo.Objective(expr=2*m.x + m.y) + >>> m.cons = pyo.ConstraintList() # for the cutting planes + >>> + >>> def _add_cut(xval): + ... # a function to generate the cut + ... m.x.value = xval + ... return m.cons.add(m.y >= taylor_series_expansion((m.x - 2)**2)) + ... + >>> _c = _add_cut(0) # start with 2 cuts at the bounds of x + >>> _c = _add_cut(4) # this is an arbitrary choice + >>> + >>> opt = appsi.solvers.Gurobi() + >>> opt.config.stream_solver = True + >>> opt.set_instance(m) # doctest:+SKIP + >>> opt.gurobi_options['PreCrush'] = 1 + >>> opt.gurobi_options['LazyConstraints'] = 1 + >>> + >>> def my_callback(cb_m, cb_opt, cb_where): + ... if cb_where == GRB.Callback.MIPSOL: + ... cb_opt.cbGetSolution(variables=[m.x, m.y]) + ... if m.y.value < (m.x.value - 2)**2 - 1e-6: + ... cb_opt.cbLazy(_add_cut(m.x.value)) + ... + >>> opt.set_callback(my_callback) + >>> res = opt.solve(m) # doctest:+SKIP + + """ + if func is not None: + self._callback_func = func + self._callback = self._intermediate_callback() + else: + self._callback = None + self._callback_func = None + + def cbCut(self, con): + """ + Add a cut within a callback. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The cut to add + """ + if not con.active: + raise ValueError('cbCut expected an active constraint.') + + if is_fixed(con.body): + raise ValueError('cbCut expected a non-trivial constraint') + + repn = generate_standard_repn(con.body, quadratic=True, compute_values=True) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + + if con.has_lb(): + if con.has_ub(): + raise ValueError('Range constraints are not supported in cbCut.') + if not is_fixed(con.lower): + raise ValueError(f'Lower bound of constraint {con} is not constant.') + if con.has_ub(): + if not is_fixed(con.upper): + raise ValueError(f'Upper bound of constraint {con} is not constant.') + + if con.equality: + self._solver_model.cbCut( + lhs=gurobi_expr, + sense=gurobipy.GRB.EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_lb() and (value(con.lower) > -float('inf')): + self._solver_model.cbCut( + lhs=gurobi_expr, + sense=gurobipy.GRB.GREATER_EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_ub() and (value(con.upper) < float('inf')): + self._solver_model.cbCut( + lhs=gurobi_expr, + sense=gurobipy.GRB.LESS_EQUAL, + rhs=value(con.upper - repn.constant), + ) + else: + raise ValueError( + f'Constraint does not have a lower or an upper bound {con} \n' + ) + + def cbGet(self, what): + return self._solver_model.cbGet(what) + + def cbGetNodeRel(self, variables): + """ + Parameters + ---------- + variables: Var or iterable of Var + """ + if not isinstance(variables, Iterable): + variables = [variables] + gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] + var_values = self._solver_model.cbGetNodeRel(gurobi_vars) + for i, v in enumerate(variables): + v.set_value(var_values[i], skip_validation=True) + + def cbGetSolution(self, variables): + """ + Parameters + ---------- + variables: iterable of vars + """ + if not isinstance(variables, Iterable): + variables = [variables] + gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] + var_values = self._solver_model.cbGetSolution(gurobi_vars) + for i, v in enumerate(variables): + v.set_value(var_values[i], skip_validation=True) + + def cbLazy(self, con): + """ + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The lazy constraint to add + """ + if not con.active: + raise ValueError('cbLazy expected an active constraint.') + + if is_fixed(con.body): + raise ValueError('cbLazy expected a non-trivial constraint') + + repn = generate_standard_repn(con.body, quadratic=True, compute_values=True) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + + if con.has_lb(): + if con.has_ub(): + raise ValueError('Range constraints are not supported in cbLazy.') + if not is_fixed(con.lower): + raise ValueError(f'Lower bound of constraint {con} is not constant.') + if con.has_ub(): + if not is_fixed(con.upper): + raise ValueError(f'Upper bound of constraint {con} is not constant.') + + if con.equality: + self._solver_model.cbLazy( + lhs=gurobi_expr, + sense=gurobipy.GRB.EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_lb() and (value(con.lower) > -float('inf')): + self._solver_model.cbLazy( + lhs=gurobi_expr, + sense=gurobipy.GRB.GREATER_EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_ub() and (value(con.upper) < float('inf')): + self._solver_model.cbLazy( + lhs=gurobi_expr, + sense=gurobipy.GRB.LESS_EQUAL, + rhs=value(con.upper - repn.constant), + ) + else: + raise ValueError( + f'Constraint does not have a lower or an upper bound {con} \n' + ) + + def cbSetSolution(self, variables, solution): + if not isinstance(variables, Iterable): + variables = [variables] + gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] + self._solver_model.cbSetSolution(gurobi_vars, solution) + + def cbUseSolution(self): + return self._solver_model.cbUseSolution() + + def reset(self): + self._solver_model.reset() + + def add_variables(self, variables): + self._change_detector.add_variables(variables) + + def add_constraints(self, cons): + self._change_detector.add_constraints(cons) + + def add_sos_constraints(self, cons): + self._change_detector.add_sos_constraints(cons) + + def set_objective(self, obj): + self._change_detector.set_objective(obj) + + def remove_constrains(self, cons): + self._change_detector.remove_constraints(cons) + + def remove_sos_constraints(self, cons): + self._change_detector.remove_sos_constraints(cons) + + def remove_variables(self, variables): + self._change_detector.remove_variables(variables) + + def update_variables(self, variables): + self._change_detector.update_variables(variables) + + def update_parameters(self, params): + self._change_detector.update_parameters(params) \ No newline at end of file diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 748e0127151..a0d87835e13 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -30,9 +30,8 @@ from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.solvers.ipopt import Ipopt -# from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect -from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -48,14 +47,14 @@ raise unittest.SkipTest('Parameterized is not available.') all_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('highs', Highs), ] mip_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), @@ -64,13 +63,13 @@ ('ipopt', Ipopt), ] qcp_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ] qp_solvers = qcp_solvers + [("highs", Highs)] miqcqp_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ] nl_solvers = [('ipopt', Ipopt)] From 862c387a8e6478d9b9c3f176b0059046d02f1198 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:18:04 -0600 Subject: [PATCH 07/37] bugs --- pyomo/contrib/solver/plugins.py | 4 ++-- pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index f29c4f61c4e..fed739232ad 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -35,7 +35,7 @@ def load(): name='gurobi_direct_quadratic', legacy_name='gurobi_direct_quadratic_v2', doc='Direct interface to Gurobi', - )(GurobiDirect) + )(GurobiDirectQuadratic) SolverFactory.register( - name='highs', legacy_name='highs', doc='Persistent interface to HiGHS' + name='highs', legacy_name='highs_v2', doc='Persistent interface to HiGHS' )(Highs) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 844502ca476..b8a8f46d1f6 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -279,7 +279,7 @@ def get_updated_expression(self): incremental_coef_value = ( current_coef_value - self.last_linear_coef_values[ndx] ) - gurobi_expr += incremental_coef_value * coef.var + gurobi_expr += incremental_coef_value * coef.gurobi_var self.last_linear_coef_values[ndx] = current_coef_value for ndx, coef in enumerate(self.quadratic_coefs): current_coef_value = value(coef.expr) @@ -324,7 +324,7 @@ def get_updated_expression(self): class _MutableQuadraticCoefficient: def __init__(self, expr, v1id, v2id, var_map): - self.expr = None + self.expr = expr self.var_map = var_map self.v1id = v1id self.v2id = v2id @@ -860,6 +860,7 @@ def _remove_variables(self, variables: List[VarData]): solver_var = self._pyomo_var_to_solver_var_map[v_id] self._solver_model.remove(solver_var) del self._pyomo_var_to_solver_var_map[v_id] + del self._vars[v_id] self._mutable_bounds.pop(v_id, None) self._needs_updated = True From 8f7a61ed3bf8eafc8eee12544755f61097a2b7be Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:35:33 -0600 Subject: [PATCH 08/37] refactoring gurobi interfaces --- pyomo/contrib/observer/model_observer.py | 7 +- .../solvers/gurobi/gurobi_persistent.py | 6 +- pyomo/contrib/solver/solvers/gurobi_direct.py | 470 ------ .../solver/solvers/gurobi_persistent.py | 1409 ----------------- .../tests/solvers/test_gurobi_persistent.py | 49 +- 5 files changed, 25 insertions(+), 1916 deletions(-) delete mode 100644 pyomo/contrib/solver/solvers/gurobi_direct.py delete mode 100644 pyomo/contrib/solver/solvers/gurobi_persistent.py diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 8f7238c2ee9..bd905e1c61d 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -361,12 +361,15 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): self._named_expressions[con] = [] self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) + for con in cons: + variables = self._vars_referenced_by_con[con] + params = self._params_referenced_by_con[con] for v in variables: self._referenced_variables[id(v)][1][con] = None for p in params: self._referenced_params[id(p)][1][con] = None - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) for obs in self._observers: obs.add_sos_constraints(cons) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index b8a8f46d1f6..e91381f41a3 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -39,10 +39,6 @@ SolutionStatus, ) from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig -from pyomo.contrib.solver.solvers.gurobi_direct import ( - GurobiConfigMixin, - GurobiSolverMixin, -) from pyomo.contrib.solver.common.util import ( NoFeasibleSolutionError, NoOptimalSolutionError, @@ -1328,7 +1324,7 @@ def add_sos_constraints(self, cons): def set_objective(self, obj): self._change_detector.set_objective(obj) - def remove_constrains(self, cons): + def remove_constraints(self, cons): self._change_detector.remove_constraints(cons) def remove_sos_constraints(self, cons): diff --git a/pyomo/contrib/solver/solvers/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi_direct.py deleted file mode 100644 index 45ea9dcc873..00000000000 --- a/pyomo/contrib/solver/solvers/gurobi_direct.py +++ /dev/null @@ -1,470 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import datetime -import io -import math -import operator -import os - -from pyomo.common.collections import ComponentMap, ComponentSet -from pyomo.common.config import ConfigValue -from pyomo.common.dependencies import attempt_import -from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import MouseTrap, ApplicationError -from pyomo.common.shutdown import python_is_shutting_down -from pyomo.common.tee import capture_output, TeeStream -from pyomo.common.timing import HierarchicalTimer -from pyomo.core.staleflag import StaleFlagManager -from pyomo.repn.plugins.standard_form import LinearStandardFormCompiler - -from pyomo.contrib.solver.common.base import SolverBase, Availability -from pyomo.contrib.solver.common.config import BranchAndBoundConfig -from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, - NoDualsError, - NoReducedCostsError, - NoSolutionError, - IncompatibleModelError, -) -from pyomo.contrib.solver.common.results import ( - Results, - SolutionStatus, - TerminationCondition, -) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase - - -gurobipy, gurobipy_available = attempt_import('gurobipy') - - -class GurobiConfigMixin: - """ - Mixin class for Gurobi-specific configurations - """ - - def __init__(self): - self.use_mipstart: bool = self.declare( - 'use_mipstart', - ConfigValue( - default=False, - domain=bool, - description="If True, the current values of the integer variables " - "will be passed to Gurobi.", - ), - ) - - -class GurobiConfig(BranchAndBoundConfig, GurobiConfigMixin): - def __init__( - self, - description=None, - doc=None, - implicit=False, - implicit_domain=None, - visibility=0, - ): - BranchAndBoundConfig.__init__( - self, - description=description, - doc=doc, - implicit=implicit, - implicit_domain=implicit_domain, - visibility=visibility, - ) - GurobiConfigMixin.__init__(self) - - -class GurobiDirectSolutionLoader(SolutionLoaderBase): - def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars, pyo_obj): - self._grb_model = grb_model - self._grb_cons = grb_cons - self._grb_vars = grb_vars - self._pyo_cons = pyo_cons - self._pyo_vars = pyo_vars - self._pyo_obj = pyo_obj - GurobiDirect._register_env_client() - - def __del__(self): - if python_is_shutting_down(): - return - # Free the associated model - if self._grb_model is not None: - self._grb_cons = None - self._grb_vars = None - self._pyo_cons = None - self._pyo_vars = None - self._pyo_obj = None - # explicitly release the model - self._grb_model.dispose() - self._grb_model = None - # Release the gurobi license if this is the last reference to - # the environment (either through a results object or solver - # interface) - GurobiDirect._release_env_client() - - def load_vars(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 - if self._grb_model.SolCount == 0: - raise NoSolutionError() - - iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) - for p_var, g_var in iterator: - p_var.set_value(g_var, skip_validation=True) - StaleFlagManager.mark_all_as_stale(delayed=True) - - def get_primals(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 - if self._grb_model.SolCount == 0: - raise NoSolutionError() - - iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) - return ComponentMap(iterator) - - def get_duals(self, cons_to_load=None): - if self._grb_model.Status != gurobipy.GRB.OPTIMAL: - raise NoDualsError() - - def dedup(_iter): - last = None - for con_info_dual in _iter: - if not con_info_dual[1] and con_info_dual[0][0] is last: - continue - last = con_info_dual[0][0] - yield con_info_dual - - iterator = dedup(zip(self._pyo_cons, self._grb_cons.getAttr('Pi').tolist())) - if cons_to_load: - cons_to_load = set(cons_to_load) - iterator = filter( - lambda con_info_dual: con_info_dual[0][0] in cons_to_load, iterator - ) - return {con_info[0]: dual for con_info, dual in iterator} - - def get_reduced_costs(self, vars_to_load=None): - if self._grb_model.Status != gurobipy.GRB.OPTIMAL: - raise NoReducedCostsError() - - iterator = zip(self._pyo_vars, self._grb_vars.getAttr('Rc').tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) - return ComponentMap(iterator) - - -class GurobiSolverMixin: - """ - gurobi_direct and gurobi_persistent check availability and set versions - in the same way. This moves the logic to a central location to reduce - duplicate code. - """ - - _num_gurobipy_env_clients = 0 - _gurobipy_env = None - _available = None - _gurobipy_available = gurobipy_available - - def available(self): - if self._available is None: - # this triggers the deferred import, and for the persistent - # interface, may update the _available flag - # - # Note that we set the _available flag on the *most derived - # class* and not on the instance, or on the base class. That - # allows different derived interfaces to have different - # availability (e.g., persistent has a minimum version - # requirement that the direct interface doesn't) - if not self._gurobipy_available: - if self._available is None: - self.__class__._available = Availability.NotFound - else: - self.__class__._available = self._check_license() - return self._available - - @staticmethod - def release_license(): - if GurobiSolverMixin._gurobipy_env is None: - return - if GurobiSolverMixin._num_gurobipy_env_clients: - logger.warning( - "Call to GurobiSolverMixin.release_license() with %s remaining " - "environment clients." % (GurobiSolverMixin._num_gurobipy_env_clients,) - ) - GurobiSolverMixin._gurobipy_env.close() - GurobiSolverMixin._gurobipy_env = None - - @staticmethod - def env(): - if GurobiSolverMixin._gurobipy_env is None: - with capture_output(capture_fd=True): - GurobiSolverMixin._gurobipy_env = gurobipy.Env() - return GurobiSolverMixin._gurobipy_env - - @staticmethod - def _register_env_client(): - GurobiSolverMixin._num_gurobipy_env_clients += 1 - - @staticmethod - def _release_env_client(): - GurobiSolverMixin._num_gurobipy_env_clients -= 1 - if GurobiSolverMixin._num_gurobipy_env_clients <= 0: - # Note that _num_gurobipy_env_clients should never be <0, - # but if it is, release_license will issue a warning (that - # we want to know about) - GurobiSolverMixin.release_license() - - def _check_license(self): - try: - model = gurobipy.Model(env=self.env()) - except gurobipy.GurobiError: - return Availability.BadLicense - - model.setParam('OutputFlag', 0) - try: - model.addVars(range(2001)) - model.optimize() - return Availability.FullLicense - except gurobipy.GurobiError: - return Availability.LimitedLicense - finally: - model.dispose() - - def version(self): - version = ( - gurobipy.GRB.VERSION_MAJOR, - gurobipy.GRB.VERSION_MINOR, - gurobipy.GRB.VERSION_TECHNICAL, - ) - return version - - -class GurobiDirect(GurobiSolverMixin, SolverBase): - """ - Interface to Gurobi using gurobipy - """ - - CONFIG = GurobiConfig() - - _tc_map = None - - def __init__(self, **kwds): - super().__init__(**kwds) - self._register_env_client() - - def __del__(self): - if not python_is_shutting_down(): - self._release_env_client() - - def solve(self, model, **kwds) -> Results: - start_timestamp = datetime.datetime.now(datetime.timezone.utc) - config = self.config(value=kwds, preserve_implicit=True) - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - if config.timer is None: - config.timer = HierarchicalTimer() - timer = config.timer - - StaleFlagManager.mark_all_as_stale() - - timer.start('compile_model') - repn = LinearStandardFormCompiler().write( - model, mixed_form=True, set_sense=None - ) - timer.stop('compile_model') - - if len(repn.objectives) > 1: - raise IncompatibleModelError( - f"The {self.__class__.__name__} solver only supports models " - f"with zero or one objectives (received {len(repn.objectives)})." - ) - - timer.start('prepare_matrices') - inf = float('inf') - ninf = -inf - bounds = list(map(operator.attrgetter('bounds'), repn.columns)) - lb = [ninf if _b is None else _b for _b in map(operator.itemgetter(0), bounds)] - ub = [inf if _b is None else _b for _b in map(operator.itemgetter(1), bounds)] - CON = gurobipy.GRB.CONTINUOUS - BIN = gurobipy.GRB.BINARY - INT = gurobipy.GRB.INTEGER - vtype = [ - ( - CON - if v.is_continuous() - else BIN if v.is_binary() else INT if v.is_integer() else '?' - ) - for v in repn.columns - ] - sense_type = list('=<>') # Note: ordering matches 0, 1, -1 - sense = [sense_type[r[1]] for r in repn.rows] - timer.stop('prepare_matrices') - - ostreams = [io.StringIO()] + config.tee - res = Results() - - orig_cwd = os.getcwd() - try: - if config.working_dir: - os.chdir(config.working_dir) - with capture_output(TeeStream(*ostreams), capture_fd=False): - gurobi_model = gurobipy.Model(env=self.env()) - - timer.start('transfer_model') - x = gurobi_model.addMVar( - len(repn.columns), - lb=lb, - ub=ub, - obj=repn.c.todense()[0] if repn.c.shape[0] else 0, - vtype=vtype, - ) - A = gurobi_model.addMConstr(repn.A, x, sense, repn.rhs) - if repn.c.shape[0]: - gurobi_model.setAttr('ObjCon', repn.c_offset[0]) - gurobi_model.setAttr('ModelSense', int(repn.objectives[0].sense)) - # Note: calling gurobi_model.update() here is not - # necessary (it will happen as part of optimize()): - # gurobi_model.update() - timer.stop('transfer_model') - - options = config.solver_options - - gurobi_model.setParam('LogToConsole', 1) - - if config.threads is not None: - gurobi_model.setParam('Threads', config.threads) - if config.time_limit is not None: - gurobi_model.setParam('TimeLimit', config.time_limit) - if config.rel_gap is not None: - gurobi_model.setParam('MIPGap', config.rel_gap) - if config.abs_gap is not None: - gurobi_model.setParam('MIPGapAbs', config.abs_gap) - - if config.use_mipstart: - raise MouseTrap("MIPSTART not yet supported") - - for key, option in options.items(): - gurobi_model.setParam(key, option) - - timer.start('optimize') - gurobi_model.optimize() - timer.stop('optimize') - finally: - os.chdir(orig_cwd) - - res = self._postsolve( - timer, - config, - GurobiDirectSolutionLoader( - gurobi_model, A, x, repn.rows, repn.columns, repn.objectives - ), - ) - - res.solver_config = config - res.solver_name = 'Gurobi' - res.solver_version = self.version() - res.solver_log = ostreams[0].getvalue() - - end_timestamp = datetime.datetime.now(datetime.timezone.utc) - res.timing_info.start_timestamp = start_timestamp - res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() - res.timing_info.timer = timer - return res - - def _postsolve(self, timer: HierarchicalTimer, config, loader): - grb_model = loader._grb_model - status = grb_model.Status - - results = Results() - results.solution_loader = loader - results.timing_info.gurobi_time = grb_model.Runtime - - if grb_model.SolCount > 0: - if status == gurobipy.GRB.OPTIMAL: - results.solution_status = SolutionStatus.optimal - else: - results.solution_status = SolutionStatus.feasible - else: - results.solution_status = SolutionStatus.noSolution - - results.termination_condition = self._get_tc_map().get( - status, TerminationCondition.unknown - ) - - if ( - results.termination_condition - != TerminationCondition.convergenceCriteriaSatisfied - and config.raise_exception_on_nonoptimal_result - ): - raise NoOptimalSolutionError() - - if loader._pyo_obj: - try: - if math.isfinite(grb_model.ObjVal): - results.incumbent_objective = grb_model.ObjVal - else: - results.incumbent_objective = None - except (gurobipy.GurobiError, AttributeError): - results.incumbent_objective = None - try: - results.objective_bound = grb_model.ObjBound - except (gurobipy.GurobiError, AttributeError): - if grb_model.ModelSense == ObjectiveSense.minimize: - results.objective_bound = -math.inf - else: - results.objective_bound = math.inf - else: - results.incumbent_objective = None - results.objective_bound = None - - results.iteration_count = grb_model.getAttr('IterCount') - - timer.start('load solution') - if config.load_solutions: - if grb_model.SolCount > 0: - results.solution_loader.load_vars() - else: - raise NoFeasibleSolutionError() - timer.stop('load solution') - - return results - - def _get_tc_map(self): - if GurobiDirect._tc_map is None: - grb = gurobipy.GRB - tc = TerminationCondition - GurobiDirect._tc_map = { - grb.LOADED: tc.unknown, # problem is loaded, but no solution - grb.OPTIMAL: tc.convergenceCriteriaSatisfied, - grb.INFEASIBLE: tc.provenInfeasible, - grb.INF_OR_UNBD: tc.infeasibleOrUnbounded, - grb.UNBOUNDED: tc.unbounded, - grb.CUTOFF: tc.objectiveLimit, - grb.ITERATION_LIMIT: tc.iterationLimit, - grb.NODE_LIMIT: tc.iterationLimit, - grb.TIME_LIMIT: tc.maxTimeLimit, - grb.SOLUTION_LIMIT: tc.unknown, - grb.INTERRUPTED: tc.interrupted, - grb.NUMERIC: tc.unknown, - grb.SUBOPTIMAL: tc.unknown, - grb.USER_OBJ_LIMIT: tc.objectiveLimit, - } - return GurobiDirect._tc_map diff --git a/pyomo/contrib/solver/solvers/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi_persistent.py deleted file mode 100644 index ea3693c1c70..00000000000 --- a/pyomo/contrib/solver/solvers/gurobi_persistent.py +++ /dev/null @@ -1,1409 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import io -import logging -import math -from typing import List, Optional -from collections.abc import Iterable - -from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet -from pyomo.common.dependencies import attempt_import -from pyomo.common.errors import ApplicationError -from pyomo.common.tee import capture_output, TeeStream -from pyomo.common.timing import HierarchicalTimer -from pyomo.common.shutdown import python_is_shutting_down -from pyomo.core.kernel.objective import minimize, maximize -from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler -from pyomo.core.base.var import VarData -from pyomo.core.base.constraint import ConstraintData -from pyomo.core.base.sos import SOSConstraintData -from pyomo.core.base.param import ParamData -from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types -from pyomo.repn import generate_standard_repn -from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability -from pyomo.contrib.solver.common.results import ( - Results, - TerminationCondition, - SolutionStatus, -) -from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig -from pyomo.contrib.solver.solvers.gurobi_direct import ( - GurobiConfigMixin, - GurobiSolverMixin, -) -from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, - NoDualsError, - NoReducedCostsError, - NoSolutionError, - IncompatibleModelError, -) -from pyomo.contrib.solver.common.persistent import ( - PersistentSolverUtils, - PersistentSolverMixin, -) -from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader -from pyomo.core.staleflag import StaleFlagManager - - -logger = logging.getLogger(__name__) - - -def _import_gurobipy(): - try: - import gurobipy - except ImportError: - GurobiPersistent._available = Availability.NotFound - raise - if gurobipy.GRB.VERSION_MAJOR < 7: - GurobiPersistent._available = Availability.BadVersion - raise ImportError('The Persistent Gurobi interface requires gurobipy>=7.0.0') - return gurobipy - - -gurobipy, gurobipy_available = attempt_import('gurobipy', importer=_import_gurobipy) - - -class GurobiConfig(PersistentBranchAndBoundConfig, GurobiConfigMixin): - def __init__( - self, - description=None, - doc=None, - implicit=False, - implicit_domain=None, - visibility=0, - ): - PersistentBranchAndBoundConfig.__init__( - self, - description=description, - doc=doc, - implicit=implicit, - implicit_domain=implicit_domain, - visibility=visibility, - ) - GurobiConfigMixin.__init__(self) - - -class GurobiSolutionLoader(PersistentSolutionLoader): - def load_vars(self, vars_to_load=None, solution_number=0): - self._assert_solution_still_valid() - self._solver._load_vars( - vars_to_load=vars_to_load, solution_number=solution_number - ) - - def get_primals(self, vars_to_load=None, solution_number=0): - self._assert_solution_still_valid() - return self._solver._get_primals( - vars_to_load=vars_to_load, solution_number=solution_number - ) - - -class _MutableLowerBound: - def __init__(self, expr): - self.var = None - self.expr = expr - - def update(self): - self.var.setAttr('lb', value(self.expr)) - - -class _MutableUpperBound: - def __init__(self, expr): - self.var = None - self.expr = expr - - def update(self): - self.var.setAttr('ub', value(self.expr)) - - -class _MutableLinearCoefficient: - def __init__(self): - self.expr = None - self.var = None - self.con = None - self.gurobi_model = None - - def update(self): - self.gurobi_model.chgCoeff(self.con, self.var, value(self.expr)) - - -class _MutableRangeConstant: - def __init__(self): - self.lhs_expr = None - self.rhs_expr = None - self.con = None - self.slack_name = None - self.gurobi_model = None - - def update(self): - rhs_val = value(self.rhs_expr) - lhs_val = value(self.lhs_expr) - self.con.rhs = rhs_val - slack = self.gurobi_model.getVarByName(self.slack_name) - slack.ub = rhs_val - lhs_val - - -class _MutableConstant: - def __init__(self): - self.expr = None - self.con = None - - def update(self): - self.con.rhs = value(self.expr) - - -class _MutableQuadraticConstraint: - def __init__( - self, gurobi_model, gurobi_con, constant, linear_coefs, quadratic_coefs - ): - self.con = gurobi_con - self.gurobi_model = gurobi_model - self.constant = constant - self.last_constant_value = value(self.constant.expr) - self.linear_coefs = linear_coefs - self.last_linear_coef_values = [value(i.expr) for i in self.linear_coefs] - self.quadratic_coefs = quadratic_coefs - self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] - - def get_updated_expression(self): - gurobi_expr = self.gurobi_model.getQCRow(self.con) - for ndx, coef in enumerate(self.linear_coefs): - current_coef_value = value(coef.expr) - incremental_coef_value = ( - current_coef_value - self.last_linear_coef_values[ndx] - ) - gurobi_expr += incremental_coef_value * coef.var - self.last_linear_coef_values[ndx] = current_coef_value - for ndx, coef in enumerate(self.quadratic_coefs): - current_coef_value = value(coef.expr) - incremental_coef_value = ( - current_coef_value - self.last_quadratic_coef_values[ndx] - ) - gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 - self.last_quadratic_coef_values[ndx] = current_coef_value - return gurobi_expr - - def get_updated_rhs(self): - return value(self.constant.expr) - - -class _MutableObjective: - def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): - self.gurobi_model = gurobi_model - self.constant = constant - self.linear_coefs = linear_coefs - self.quadratic_coefs = quadratic_coefs - self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] - - def get_updated_expression(self): - for ndx, coef in enumerate(self.linear_coefs): - coef.var.obj = value(coef.expr) - self.gurobi_model.ObjCon = value(self.constant.expr) - - gurobi_expr = None - for ndx, coef in enumerate(self.quadratic_coefs): - if value(coef.expr) != self.last_quadratic_coef_values[ndx]: - if gurobi_expr is None: - self.gurobi_model.update() - gurobi_expr = self.gurobi_model.getObjective() - current_coef_value = value(coef.expr) - incremental_coef_value = ( - current_coef_value - self.last_quadratic_coef_values[ndx] - ) - gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 - self.last_quadratic_coef_values[ndx] = current_coef_value - return gurobi_expr - - -class _MutableQuadraticCoefficient: - def __init__(self): - self.expr = None - self.var1 = None - self.var2 = None - - -class GurobiPersistent( - GurobiSolverMixin, - PersistentSolverMixin, - PersistentSolverUtils, - PersistentSolverBase, -): - """ - Interface to Gurobi persistent - """ - - CONFIG = GurobiConfig() - _gurobipy_available = gurobipy_available - - def __init__(self, **kwds): - treat_fixed_vars_as_params = kwds.pop('treat_fixed_vars_as_params', True) - PersistentSolverBase.__init__(self, **kwds) - PersistentSolverUtils.__init__( - self, treat_fixed_vars_as_params=treat_fixed_vars_as_params - ) - self._register_env_client() - self._solver_model = None - self._symbol_map = SymbolMap() - self._labeler = None - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._solver_con_to_pyomo_con_map = {} - self._pyomo_sos_to_solver_sos_map = {} - self._range_constraints = OrderedSet() - self._mutable_helpers = {} - self._mutable_bounds = {} - self._mutable_quadratic_helpers = {} - self._mutable_objective = None - self._needs_updated = True - self._callback = None - self._callback_func = None - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._last_results_object: Optional[Results] = None - - def release_license(self): - self._reinit() - self.__class__.release_license() - - def __del__(self): - if not python_is_shutting_down(): - self._release_env_client() - - @property - def symbol_map(self): - return self._symbol_map - - def _solve(self): - config = self._active_config - timer = config.timer - ostreams = [io.StringIO()] + config.tee - - with capture_output(TeeStream(*ostreams), capture_fd=False): - options = config.solver_options - - self._solver_model.setParam('LogToConsole', 1) - - if config.threads is not None: - self._solver_model.setParam('Threads', config.threads) - if config.time_limit is not None: - self._solver_model.setParam('TimeLimit', config.time_limit) - if config.rel_gap is not None: - self._solver_model.setParam('MIPGap', config.rel_gap) - if config.abs_gap is not None: - self._solver_model.setParam('MIPGapAbs', config.abs_gap) - - if config.use_mipstart: - for ( - pyomo_var_id, - gurobi_var, - ) in self._pyomo_var_to_solver_var_map.items(): - pyomo_var = self._vars[pyomo_var_id][0] - if pyomo_var.is_integer() and pyomo_var.value is not None: - self.set_var_attr(pyomo_var, 'Start', pyomo_var.value) - - for key, option in options.items(): - self._solver_model.setParam(key, option) - - timer.start('optimize') - self._solver_model.optimize(self._callback) - timer.stop('optimize') - - self._needs_updated = False - res = self._postsolve(timer) - res.solver_config = config - res.solver_name = 'Gurobi' - res.solver_version = self.version() - res.solver_log = ostreams[0].getvalue() - return res - - def _process_domain_and_bounds( - self, var, var_id, mutable_lbs, mutable_ubs, ndx, gurobipy_var - ): - _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[id(var)] - lb, ub, step = _domain_interval - if lb is None: - lb = -gurobipy.GRB.INFINITY - if ub is None: - ub = gurobipy.GRB.INFINITY - if step == 0: - vtype = gurobipy.GRB.CONTINUOUS - elif step == 1: - if lb == 0 and ub == 1: - vtype = gurobipy.GRB.BINARY - else: - vtype = gurobipy.GRB.INTEGER - else: - raise ValueError( - f'Unrecognized domain step: {step} (should be either 0 or 1)' - ) - if _fixed: - lb = _value - ub = _value - else: - if _lb is not None: - if not is_constant(_lb): - mutable_bound = _MutableLowerBound(NPV_MaxExpression((_lb, lb))) - if gurobipy_var is None: - mutable_lbs[ndx] = mutable_bound - else: - mutable_bound.var = gurobipy_var - self._mutable_bounds[var_id, 'lb'] = (var, mutable_bound) - lb = max(value(_lb), lb) - if _ub is not None: - if not is_constant(_ub): - mutable_bound = _MutableUpperBound(NPV_MinExpression((_ub, ub))) - if gurobipy_var is None: - mutable_ubs[ndx] = mutable_bound - else: - mutable_bound.var = gurobipy_var - self._mutable_bounds[var_id, 'ub'] = (var, mutable_bound) - ub = min(value(_ub), ub) - - return lb, ub, vtype - - def _add_variables(self, variables: List[VarData]): - var_names = [] - vtypes = [] - lbs = [] - ubs = [] - mutable_lbs = {} - mutable_ubs = {} - for ndx, var in enumerate(variables): - varname = self._symbol_map.getSymbol(var, self._labeler) - lb, ub, vtype = self._process_domain_and_bounds( - var, id(var), mutable_lbs, mutable_ubs, ndx, None - ) - var_names.append(varname) - vtypes.append(vtype) - lbs.append(lb) - ubs.append(ub) - - gurobi_vars = self._solver_model.addVars( - len(variables), lb=lbs, ub=ubs, vtype=vtypes, name=var_names - ) - - for ndx, pyomo_var in enumerate(variables): - gurobi_var = gurobi_vars[ndx] - self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var - for ndx, mutable_bound in mutable_lbs.items(): - mutable_bound.var = gurobi_vars[ndx] - for ndx, mutable_bound in mutable_ubs.items(): - mutable_bound.var = gurobi_vars[ndx] - self._vars_added_since_update.update(variables) - self._needs_updated = True - - def _add_parameters(self, params: List[ParamData]): - pass - - def _reinit(self): - saved_config = self.config - saved_tmp_config = self._active_config - self.__init__(treat_fixed_vars_as_params=self._treat_fixed_vars_as_params) - # Note that __init__ registers a new env client, so we need to - # release it here: - self._release_env_client() - self.config = saved_config - self._active_config = saved_tmp_config - - def set_instance(self, model): - if self._last_results_object is not None: - self._last_results_object.solution_loader.invalidate() - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - self._reinit() - self._model = model - - if self.config.symbolic_solver_labels: - self._labeler = TextLabeler() - else: - self._labeler = NumericLabeler('x') - - self._solver_model = gurobipy.Model(name=model.name or '', env=self.env()) - - self.add_block(model) - if self._objective is None: - self.set_objective(None) - - def _get_expr_from_pyomo_expr(self, expr): - mutable_linear_coefficients = [] - mutable_quadratic_coefficients = [] - repn = generate_standard_repn(expr, quadratic=True, compute_values=False) - - degree = repn.polynomial_degree() - if (degree is None) or (degree > 2): - raise IncompatibleModelError( - f'GurobiAuto does not support expressions of degree {degree}.' - ) - - if len(repn.linear_vars) > 0: - linear_coef_vals = [] - for ndx, coef in enumerate(repn.linear_coefs): - if not is_constant(coef): - mutable_linear_coefficient = _MutableLinearCoefficient() - mutable_linear_coefficient.expr = coef - mutable_linear_coefficient.var = self._pyomo_var_to_solver_var_map[ - id(repn.linear_vars[ndx]) - ] - mutable_linear_coefficients.append(mutable_linear_coefficient) - linear_coef_vals.append(value(coef)) - new_expr = gurobipy.LinExpr( - linear_coef_vals, - [self._pyomo_var_to_solver_var_map[id(i)] for i in repn.linear_vars], - ) - else: - new_expr = 0.0 - - for ndx, v in enumerate(repn.quadratic_vars): - x, y = v - gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] - gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] - coef = repn.quadratic_coefs[ndx] - if not is_constant(coef): - mutable_quadratic_coefficient = _MutableQuadraticCoefficient() - mutable_quadratic_coefficient.expr = coef - mutable_quadratic_coefficient.var1 = gurobi_x - mutable_quadratic_coefficient.var2 = gurobi_y - mutable_quadratic_coefficients.append(mutable_quadratic_coefficient) - coef_val = value(coef) - new_expr += coef_val * gurobi_x * gurobi_y - - return ( - new_expr, - repn.constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) - - def _add_constraints(self, cons: List[ConstraintData]): - for con in cons: - conname = self._symbol_map.getSymbol(con, self._labeler) - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(con.body) - - if ( - gurobi_expr.__class__ in {gurobipy.LinExpr, gurobipy.Var} - or gurobi_expr.__class__ in native_numeric_types - ): - if con.equality: - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addLConstr( - gurobi_expr, gurobipy.GRB.EQUAL, rhs_val, name=conname - ) - if not is_constant(rhs_expr): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_constant.con = gurobipy_con - self._mutable_helpers[con] = [mutable_constant] - elif con.has_lb() and con.has_ub(): - lhs_expr = con.lower - repn_constant - rhs_expr = con.upper - repn_constant - lhs_val = value(lhs_expr) - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addRange( - gurobi_expr, lhs_val, rhs_val, name=conname - ) - self._range_constraints.add(con) - if not is_constant(lhs_expr) or not is_constant(rhs_expr): - mutable_range_constant = _MutableRangeConstant() - mutable_range_constant.lhs_expr = lhs_expr - mutable_range_constant.rhs_expr = rhs_expr - mutable_range_constant.con = gurobipy_con - mutable_range_constant.slack_name = 'Rg' + conname - mutable_range_constant.gurobi_model = self._solver_model - self._mutable_helpers[con] = [mutable_range_constant] - elif con.has_lb(): - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addLConstr( - gurobi_expr, gurobipy.GRB.GREATER_EQUAL, rhs_val, name=conname - ) - if not is_constant(rhs_expr): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_constant.con = gurobipy_con - self._mutable_helpers[con] = [mutable_constant] - elif con.has_ub(): - rhs_expr = con.upper - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addLConstr( - gurobi_expr, gurobipy.GRB.LESS_EQUAL, rhs_val, name=conname - ) - if not is_constant(rhs_expr): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_constant.con = gurobipy_con - self._mutable_helpers[con] = [mutable_constant] - else: - raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" - ) - for tmp in mutable_linear_coefficients: - tmp.con = gurobipy_con - tmp.gurobi_model = self._solver_model - if len(mutable_linear_coefficients) > 0: - if con not in self._mutable_helpers: - self._mutable_helpers[con] = mutable_linear_coefficients - else: - self._mutable_helpers[con].extend(mutable_linear_coefficients) - elif gurobi_expr.__class__ is gurobipy.QuadExpr: - if con.equality: - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addQConstr( - gurobi_expr, gurobipy.GRB.EQUAL, rhs_val, name=conname - ) - elif con.has_lb() and con.has_ub(): - raise NotImplementedError( - 'Quadratic range constraints are not supported' - ) - elif con.has_lb(): - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addQConstr( - gurobi_expr, gurobipy.GRB.GREATER_EQUAL, rhs_val, name=conname - ) - elif con.has_ub(): - rhs_expr = con.upper - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addQConstr( - gurobi_expr, gurobipy.GRB.LESS_EQUAL, rhs_val, name=conname - ) - else: - raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" - ) - if ( - len(mutable_linear_coefficients) > 0 - or len(mutable_quadratic_coefficients) > 0 - or not is_constant(repn_constant) - ): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_quadratic_constraint = _MutableQuadraticConstraint( - self._solver_model, - gurobipy_con, - mutable_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) - self._mutable_quadratic_helpers[con] = mutable_quadratic_constraint - else: - raise ValueError( - f'Unrecognized Gurobi expression type: {str(gurobi_expr.__class__)}' - ) - - self._pyomo_con_to_solver_con_map[con] = gurobipy_con - self._solver_con_to_pyomo_con_map[id(gurobipy_con)] = con - self._constraints_added_since_update.update(cons) - self._needs_updated = True - - def _add_sos_constraints(self, cons: List[SOSConstraintData]): - for con in cons: - conname = self._symbol_map.getSymbol(con, self._labeler) - level = con.level - if level == 1: - sos_type = gurobipy.GRB.SOS_TYPE1 - elif level == 2: - sos_type = gurobipy.GRB.SOS_TYPE2 - else: - raise ValueError( - f"Solver does not support SOS level {level} constraints" - ) - - gurobi_vars = [] - weights = [] - - for v, w in con.get_items(): - v_id = id(v) - gurobi_vars.append(self._pyomo_var_to_solver_var_map[v_id]) - weights.append(w) - - gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) - self._pyomo_sos_to_solver_sos_map[con] = gurobipy_con - self._constraints_added_since_update.update(cons) - self._needs_updated = True - - def _remove_constraints(self, cons: List[ConstraintData]): - for con in cons: - if con in self._constraints_added_since_update: - self._update_gurobi_model() - solver_con = self._pyomo_con_to_solver_con_map[con] - self._solver_model.remove(solver_con) - self._symbol_map.removeSymbol(con) - del self._pyomo_con_to_solver_con_map[con] - del self._solver_con_to_pyomo_con_map[id(solver_con)] - self._range_constraints.discard(con) - self._mutable_helpers.pop(con, None) - self._mutable_quadratic_helpers.pop(con, None) - self._needs_updated = True - - def _remove_sos_constraints(self, cons: List[SOSConstraintData]): - for con in cons: - if con in self._constraints_added_since_update: - self._update_gurobi_model() - solver_sos_con = self._pyomo_sos_to_solver_sos_map[con] - self._solver_model.remove(solver_sos_con) - self._symbol_map.removeSymbol(con) - del self._pyomo_sos_to_solver_sos_map[con] - self._needs_updated = True - - def _remove_variables(self, variables: List[VarData]): - for var in variables: - v_id = id(var) - if var in self._vars_added_since_update: - self._update_gurobi_model() - solver_var = self._pyomo_var_to_solver_var_map[v_id] - self._solver_model.remove(solver_var) - self._symbol_map.removeSymbol(var) - del self._pyomo_var_to_solver_var_map[v_id] - self._mutable_bounds.pop(v_id, None) - self._needs_updated = True - - def _remove_parameters(self, params: List[ParamData]): - pass - - def _update_variables(self, variables: List[VarData]): - for var in variables: - var_id = id(var) - if var_id not in self._pyomo_var_to_solver_var_map: - raise ValueError( - f'The Var provided to update_var needs to be added first: {var}' - ) - self._mutable_bounds.pop((var_id, 'lb'), None) - self._mutable_bounds.pop((var_id, 'ub'), None) - gurobipy_var = self._pyomo_var_to_solver_var_map[var_id] - lb, ub, vtype = self._process_domain_and_bounds( - var, var_id, None, None, None, gurobipy_var - ) - gurobipy_var.setAttr('lb', lb) - gurobipy_var.setAttr('ub', ub) - gurobipy_var.setAttr('vtype', vtype) - self._needs_updated = True - - def update_parameters(self): - for con, helpers in self._mutable_helpers.items(): - for helper in helpers: - helper.update() - for k, (v, helper) in self._mutable_bounds.items(): - helper.update() - - for con, helper in self._mutable_quadratic_helpers.items(): - if con in self._constraints_added_since_update: - self._update_gurobi_model() - gurobi_con = helper.con - new_gurobi_expr = helper.get_updated_expression() - new_rhs = helper.get_updated_rhs() - new_sense = gurobi_con.qcsense - pyomo_con = self._solver_con_to_pyomo_con_map[id(gurobi_con)] - name = self._symbol_map.getSymbol(pyomo_con, self._labeler) - self._solver_model.remove(gurobi_con) - new_con = self._solver_model.addQConstr( - new_gurobi_expr, new_sense, new_rhs, name=name - ) - self._pyomo_con_to_solver_con_map[id(pyomo_con)] = new_con - del self._solver_con_to_pyomo_con_map[id(gurobi_con)] - self._solver_con_to_pyomo_con_map[id(new_con)] = pyomo_con - helper.con = new_con - self._constraints_added_since_update.add(con) - - helper = self._mutable_objective - pyomo_obj = self._objective - new_gurobi_expr = helper.get_updated_expression() - if new_gurobi_expr is not None: - if pyomo_obj.sense == minimize: - sense = gurobipy.GRB.MINIMIZE - else: - sense = gurobipy.GRB.MAXIMIZE - self._solver_model.setObjective(new_gurobi_expr, sense=sense) - - def _set_objective(self, obj): - if obj is None: - sense = gurobipy.GRB.MINIMIZE - gurobi_expr = 0 - repn_constant = 0 - mutable_linear_coefficients = [] - mutable_quadratic_coefficients = [] - else: - if obj.sense == minimize: - sense = gurobipy.GRB.MINIMIZE - elif obj.sense == maximize: - sense = gurobipy.GRB.MAXIMIZE - else: - raise ValueError(f'Objective sense is not recognized: {obj.sense}') - - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(obj.expr) - - mutable_constant = _MutableConstant() - mutable_constant.expr = repn_constant - mutable_objective = _MutableObjective( - self._solver_model, - mutable_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) - self._mutable_objective = mutable_objective - - # These two lines are needed as a workaround - # see PR #2454 - self._solver_model.setObjective(0) - self._solver_model.update() - - self._solver_model.setObjective(gurobi_expr + value(repn_constant), sense=sense) - self._needs_updated = True - - def _postsolve(self, timer: HierarchicalTimer): - config = self._active_config - - gprob = self._solver_model - grb = gurobipy.GRB - status = gprob.Status - - results = Results() - results.solution_loader = GurobiSolutionLoader(self) - results.timing_info.gurobi_time = gprob.Runtime - - if gprob.SolCount > 0: - if status == grb.OPTIMAL: - results.solution_status = SolutionStatus.optimal - else: - results.solution_status = SolutionStatus.feasible - else: - results.solution_status = SolutionStatus.noSolution - - if status == grb.LOADED: # problem is loaded, but no solution - results.termination_condition = TerminationCondition.unknown - elif status == grb.OPTIMAL: # optimal - results.termination_condition = ( - TerminationCondition.convergenceCriteriaSatisfied - ) - elif status == grb.INFEASIBLE: - results.termination_condition = TerminationCondition.provenInfeasible - elif status == grb.INF_OR_UNBD: - results.termination_condition = TerminationCondition.infeasibleOrUnbounded - elif status == grb.UNBOUNDED: - results.termination_condition = TerminationCondition.unbounded - elif status == grb.CUTOFF: - results.termination_condition = TerminationCondition.objectiveLimit - elif status == grb.ITERATION_LIMIT: - results.termination_condition = TerminationCondition.iterationLimit - elif status == grb.NODE_LIMIT: - results.termination_condition = TerminationCondition.iterationLimit - elif status == grb.TIME_LIMIT: - results.termination_condition = TerminationCondition.maxTimeLimit - elif status == grb.SOLUTION_LIMIT: - results.termination_condition = TerminationCondition.unknown - elif status == grb.INTERRUPTED: - results.termination_condition = TerminationCondition.interrupted - elif status == grb.NUMERIC: - results.termination_condition = TerminationCondition.unknown - elif status == grb.SUBOPTIMAL: - results.termination_condition = TerminationCondition.unknown - elif status == grb.USER_OBJ_LIMIT: - results.termination_condition = TerminationCondition.objectiveLimit - else: - results.termination_condition = TerminationCondition.unknown - - if ( - results.termination_condition - != TerminationCondition.convergenceCriteriaSatisfied - and config.raise_exception_on_nonoptimal_result - ): - raise NoOptimalSolutionError() - - results.incumbent_objective = None - results.objective_bound = None - if self._objective is not None: - try: - results.incumbent_objective = gprob.ObjVal - except (gurobipy.GurobiError, AttributeError): - results.incumbent_objective = None - try: - results.objective_bound = gprob.ObjBound - except (gurobipy.GurobiError, AttributeError): - if self._objective.sense == minimize: - results.objective_bound = -math.inf - else: - results.objective_bound = math.inf - - if results.incumbent_objective is not None and not math.isfinite( - results.incumbent_objective - ): - results.incumbent_objective = None - - results.iteration_count = gprob.getAttr('IterCount') - - timer.start('load solution') - if config.load_solutions: - if gprob.SolCount > 0: - self._load_vars() - else: - raise NoFeasibleSolutionError() - timer.stop('load solution') - - return results - - def _load_suboptimal_mip_solution(self, vars_to_load, solution_number): - if ( - self.get_model_attr('NumIntVars') == 0 - and self.get_model_attr('NumBinVars') == 0 - ): - raise ValueError( - 'Cannot obtain suboptimal solutions for a continuous model' - ) - var_map = self._pyomo_var_to_solver_var_map - ref_vars = self._referenced_variables - original_solution_number = self.get_gurobi_param_info('SolutionNumber')[2] - self.set_gurobi_param('SolutionNumber', solution_number) - gurobi_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] - vals = self._solver_model.getAttr("Xn", gurobi_vars_to_load) - res = ComponentMap() - for var_id, val in zip(vars_to_load, vals): - using_cons, using_sos, using_obj = ref_vars[var_id] - if using_cons or using_sos or (using_obj is not None): - res[self._vars[var_id][0]] = val - self.set_gurobi_param('SolutionNumber', original_solution_number) - return res - - def _load_vars(self, vars_to_load=None, solution_number=0): - for v, val in self._get_primals( - vars_to_load=vars_to_load, solution_number=solution_number - ).items(): - v.set_value(val, skip_validation=True) - StaleFlagManager.mark_all_as_stale(delayed=True) - - def _get_primals(self, vars_to_load=None, solution_number=0): - if self._needs_updated: - self._update_gurobi_model() # this is needed to ensure that solutions cannot be loaded after the model has been changed - - if self._solver_model.SolCount == 0: - raise NoSolutionError() - - var_map = self._pyomo_var_to_solver_var_map - ref_vars = self._referenced_variables - if vars_to_load is None: - vars_to_load = self._pyomo_var_to_solver_var_map.keys() - else: - vars_to_load = [id(v) for v in vars_to_load] - - if solution_number != 0: - return self._load_suboptimal_mip_solution( - vars_to_load=vars_to_load, solution_number=solution_number - ) - - gurobi_vars_to_load = [var_map[pyomo_var_id] for pyomo_var_id in vars_to_load] - vals = self._solver_model.getAttr("X", gurobi_vars_to_load) - - res = ComponentMap() - for var_id, val in zip(vars_to_load, vals): - using_cons, using_sos, using_obj = ref_vars[var_id] - if using_cons or using_sos or (using_obj is not None): - res[self._vars[var_id][0]] = val - return res - - def _get_reduced_costs(self, vars_to_load=None): - if self._needs_updated: - self._update_gurobi_model() - - if self._solver_model.Status != gurobipy.GRB.OPTIMAL: - raise NoReducedCostsError() - - var_map = self._pyomo_var_to_solver_var_map - ref_vars = self._referenced_variables - res = ComponentMap() - if vars_to_load is None: - vars_to_load = self._pyomo_var_to_solver_var_map.keys() - else: - vars_to_load = [id(v) for v in vars_to_load] - - gurobi_vars_to_load = [var_map[pyomo_var_id] for pyomo_var_id in vars_to_load] - vals = self._solver_model.getAttr("Rc", gurobi_vars_to_load) - - for var_id, val in zip(vars_to_load, vals): - using_cons, using_sos, using_obj = ref_vars[var_id] - if using_cons or using_sos or (using_obj is not None): - res[self._vars[var_id][0]] = val - - return res - - def _get_duals(self, cons_to_load=None): - if self._needs_updated: - self._update_gurobi_model() - - if self._solver_model.Status != gurobipy.GRB.OPTIMAL: - raise NoDualsError() - - con_map = self._pyomo_con_to_solver_con_map - reverse_con_map = self._solver_con_to_pyomo_con_map - dual = {} - - if cons_to_load is None: - linear_cons_to_load = self._solver_model.getConstrs() - quadratic_cons_to_load = self._solver_model.getQConstrs() - else: - gurobi_cons_to_load = OrderedSet( - [con_map[pyomo_con] for pyomo_con in cons_to_load] - ) - linear_cons_to_load = list( - gurobi_cons_to_load.intersection( - OrderedSet(self._solver_model.getConstrs()) - ) - ) - quadratic_cons_to_load = list( - gurobi_cons_to_load.intersection( - OrderedSet(self._solver_model.getQConstrs()) - ) - ) - linear_vals = self._solver_model.getAttr("Pi", linear_cons_to_load) - quadratic_vals = self._solver_model.getAttr("QCPi", quadratic_cons_to_load) - - for gurobi_con, val in zip(linear_cons_to_load, linear_vals): - pyomo_con = reverse_con_map[id(gurobi_con)] - dual[pyomo_con] = val - for gurobi_con, val in zip(quadratic_cons_to_load, quadratic_vals): - pyomo_con = reverse_con_map[id(gurobi_con)] - dual[pyomo_con] = val - - return dual - - def update(self, timer: HierarchicalTimer = None): - if self._needs_updated: - self._update_gurobi_model() - super().update(timer=timer) - self._update_gurobi_model() - - def _update_gurobi_model(self): - self._solver_model.update() - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._needs_updated = False - - def get_model_attr(self, attr): - """ - Get the value of an attribute on the Gurobi model. - - Parameters - ---------- - attr: str - The attribute to get. See Gurobi documentation for descriptions of the attributes. - """ - if self._needs_updated: - self._update_gurobi_model() - return self._solver_model.getAttr(attr) - - def write(self, filename): - """ - Write the model to a file (e.g., and lp file). - - Parameters - ---------- - filename: str - Name of the file to which the model should be written. - """ - self._solver_model.write(filename) - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._needs_updated = False - - def set_linear_constraint_attr(self, con, attr, val): - """ - Set the value of an attribute on a gurobi linear constraint. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The pyomo constraint for which the corresponding gurobi constraint attribute - should be modified. - attr: str - The attribute to be modified. Options are: - CBasis - DStart - Lazy - val: any - See gurobi documentation for acceptable values. - """ - if attr in {'Sense', 'RHS', 'ConstrName'}: - raise ValueError( - f'Linear constraint attr {attr} cannot be set with' - ' the set_linear_constraint_attr method. Please use' - ' the remove_constraint and add_constraint methods.' - ) - self._pyomo_con_to_solver_con_map[con].setAttr(attr, val) - self._needs_updated = True - - def set_var_attr(self, var, attr, val): - """ - Set the value of an attribute on a gurobi variable. - - Parameters - ---------- - var: pyomo.core.base.var.VarData - The pyomo var for which the corresponding gurobi var attribute - should be modified. - attr: str - The attribute to be modified. Options are: - Start - VarHintVal - VarHintPri - BranchPriority - VBasis - PStart - val: any - See gurobi documentation for acceptable values. - """ - if attr in {'LB', 'UB', 'VType', 'VarName'}: - raise ValueError( - f'Var attr {attr} cannot be set with' - ' the set_var_attr method. Please use' - ' the update_var method.' - ) - if attr == 'Obj': - raise ValueError( - 'Var attr Obj cannot be set with' - ' the set_var_attr method. Please use' - ' the set_objective method.' - ) - self._pyomo_var_to_solver_var_map[id(var)].setAttr(attr, val) - self._needs_updated = True - - def get_var_attr(self, var, attr): - """ - Get the value of an attribute on a gurobi var. - - Parameters - ---------- - var: pyomo.core.base.var.VarData - The pyomo var for which the corresponding gurobi var attribute - should be retrieved. - attr: str - The attribute to get. See gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_var_to_solver_var_map[id(var)].getAttr(attr) - - def get_linear_constraint_attr(self, con, attr): - """ - Get the value of an attribute on a gurobi linear constraint. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The pyomo constraint for which the corresponding gurobi constraint attribute - should be retrieved. - attr: str - The attribute to get. See the Gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_con_to_solver_con_map[con].getAttr(attr) - - def get_sos_attr(self, con, attr): - """ - Get the value of an attribute on a gurobi sos constraint. - - Parameters - ---------- - con: pyomo.core.base.sos.SOSConstraintData - The pyomo SOS constraint for which the corresponding gurobi SOS constraint attribute - should be retrieved. - attr: str - The attribute to get. See the Gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_sos_to_solver_sos_map[con].getAttr(attr) - - def get_quadratic_constraint_attr(self, con, attr): - """ - Get the value of an attribute on a gurobi quadratic constraint. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The pyomo constraint for which the corresponding gurobi constraint attribute - should be retrieved. - attr: str - The attribute to get. See the Gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_con_to_solver_con_map[con].getAttr(attr) - - def set_gurobi_param(self, param, val): - """ - Set a gurobi parameter. - - Parameters - ---------- - param: str - The gurobi parameter to set. Options include any gurobi parameter. - Please see the Gurobi documentation for options. - val: any - The value to set the parameter to. See Gurobi documentation for possible values. - """ - self._solver_model.setParam(param, val) - - def get_gurobi_param_info(self, param): - """ - Get information about a gurobi parameter. - - Parameters - ---------- - param: str - The gurobi parameter to get info for. See Gurobi documentation for possible options. - - Returns - ------- - six-tuple containing the parameter name, type, value, minimum value, maximum value, and default value. - """ - return self._solver_model.getParamInfo(param) - - def _intermediate_callback(self): - def f(gurobi_model, where): - self._callback_func(self._model, self, where) - - return f - - def set_callback(self, func=None): - """ - Specify a callback for gurobi to use. - - Parameters - ---------- - func: function - The function to call. The function should have three arguments. The first will be the pyomo model being - solved. The second will be the GurobiPersistent instance. The third will be an enum member of - gurobipy.GRB.Callback. This will indicate where in the branch and bound algorithm gurobi is at. For - example, suppose we want to solve - - .. math:: - - min 2*x + y - - s.t. - - y >= (x-2)**2 - - 0 <= x <= 4 - - y >= 0 - - y integer - - as an MILP using extended cutting planes in callbacks. - - >>> from gurobipy import GRB # doctest:+SKIP - >>> import pyomo.environ as pyo - >>> from pyomo.core.expr.taylor_series import taylor_series_expansion - >>> from pyomo.contrib import appsi - >>> - >>> m = pyo.ConcreteModel() - >>> m.x = pyo.Var(bounds=(0, 4)) - >>> m.y = pyo.Var(within=pyo.Integers, bounds=(0, None)) - >>> m.obj = pyo.Objective(expr=2*m.x + m.y) - >>> m.cons = pyo.ConstraintList() # for the cutting planes - >>> - >>> def _add_cut(xval): - ... # a function to generate the cut - ... m.x.value = xval - ... return m.cons.add(m.y >= taylor_series_expansion((m.x - 2)**2)) - ... - >>> _c = _add_cut(0) # start with 2 cuts at the bounds of x - >>> _c = _add_cut(4) # this is an arbitrary choice - >>> - >>> opt = appsi.solvers.Gurobi() - >>> opt.config.stream_solver = True - >>> opt.set_instance(m) # doctest:+SKIP - >>> opt.gurobi_options['PreCrush'] = 1 - >>> opt.gurobi_options['LazyConstraints'] = 1 - >>> - >>> def my_callback(cb_m, cb_opt, cb_where): - ... if cb_where == GRB.Callback.MIPSOL: - ... cb_opt.cbGetSolution(variables=[m.x, m.y]) - ... if m.y.value < (m.x.value - 2)**2 - 1e-6: - ... cb_opt.cbLazy(_add_cut(m.x.value)) - ... - >>> opt.set_callback(my_callback) - >>> res = opt.solve(m) # doctest:+SKIP - - """ - if func is not None: - self._callback_func = func - self._callback = self._intermediate_callback() - else: - self._callback = None - self._callback_func = None - - def cbCut(self, con): - """ - Add a cut within a callback. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The cut to add - """ - if not con.active: - raise ValueError('cbCut expected an active constraint.') - - if is_fixed(con.body): - raise ValueError('cbCut expected a non-trivial constraint') - - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(con.body) - - if con.has_lb(): - if con.has_ub(): - raise ValueError('Range constraints are not supported in cbCut.') - if not is_fixed(con.lower): - raise ValueError(f'Lower bound of constraint {con} is not constant.') - if con.has_ub(): - if not is_fixed(con.upper): - raise ValueError(f'Upper bound of constraint {con} is not constant.') - - if con.equality: - self._solver_model.cbCut( - lhs=gurobi_expr, - sense=gurobipy.GRB.EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_lb() and (value(con.lower) > -float('inf')): - self._solver_model.cbCut( - lhs=gurobi_expr, - sense=gurobipy.GRB.GREATER_EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_ub() and (value(con.upper) < float('inf')): - self._solver_model.cbCut( - lhs=gurobi_expr, - sense=gurobipy.GRB.LESS_EQUAL, - rhs=value(con.upper - repn_constant), - ) - else: - raise ValueError( - f'Constraint does not have a lower or an upper bound {con} \n' - ) - - def cbGet(self, what): - return self._solver_model.cbGet(what) - - def cbGetNodeRel(self, variables): - """ - Parameters - ---------- - variables: Var or iterable of Var - """ - if not isinstance(variables, Iterable): - variables = [variables] - gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] - var_values = self._solver_model.cbGetNodeRel(gurobi_vars) - for i, v in enumerate(variables): - v.set_value(var_values[i], skip_validation=True) - - def cbGetSolution(self, variables): - """ - Parameters - ---------- - variables: iterable of vars - """ - if not isinstance(variables, Iterable): - variables = [variables] - gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] - var_values = self._solver_model.cbGetSolution(gurobi_vars) - for i, v in enumerate(variables): - v.set_value(var_values[i], skip_validation=True) - - def cbLazy(self, con): - """ - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The lazy constraint to add - """ - if not con.active: - raise ValueError('cbLazy expected an active constraint.') - - if is_fixed(con.body): - raise ValueError('cbLazy expected a non-trivial constraint') - - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(con.body) - - if con.has_lb(): - if con.has_ub(): - raise ValueError('Range constraints are not supported in cbLazy.') - if not is_fixed(con.lower): - raise ValueError(f'Lower bound of constraint {con} is not constant.') - if con.has_ub(): - if not is_fixed(con.upper): - raise ValueError(f'Upper bound of constraint {con} is not constant.') - - if con.equality: - self._solver_model.cbLazy( - lhs=gurobi_expr, - sense=gurobipy.GRB.EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_lb() and (value(con.lower) > -float('inf')): - self._solver_model.cbLazy( - lhs=gurobi_expr, - sense=gurobipy.GRB.GREATER_EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_ub() and (value(con.upper) < float('inf')): - self._solver_model.cbLazy( - lhs=gurobi_expr, - sense=gurobipy.GRB.LESS_EQUAL, - rhs=value(con.upper - repn_constant), - ) - else: - raise ValueError( - f'Constraint does not have a lower or an upper bound {con} \n' - ) - - def cbSetSolution(self, variables, solution): - if not isinstance(variables, Iterable): - variables = [variables] - gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] - self._solver_model.cbSetSolution(gurobi_vars, solution) - - def cbUseSolution(self): - return self._solver_model.cbUseSolution() - - def reset(self): - self._solver_model.reset() diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py index 8703ae9edff..96cd1498956 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py @@ -11,7 +11,7 @@ import pyomo.common.unittest as unittest import pyomo.environ as pyo -from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiPersistent from pyomo.contrib.solver.common.results import SolutionStatus from pyomo.core.expr.taylor_series import taylor_series_expansion @@ -471,11 +471,11 @@ def test_solution_number(self): res = opt.solve(m) num_solutions = opt.get_model_attr('SolCount') self.assertEqual(num_solutions, 3) - res.solution_loader.load_vars(solution_number=0) + res.solution_loader.load_vars(solution_id=0) self.assertAlmostEqual(pyo.value(m.obj.expr), 6.431184939357673) - res.solution_loader.load_vars(solution_number=1) + res.solution_loader.load_vars(solution_id=1) self.assertAlmostEqual(pyo.value(m.obj.expr), 6.584793218502477) - res.solution_loader.load_vars(solution_number=2) + res.solution_loader.load_vars(solution_id=2) self.assertAlmostEqual(pyo.value(m.obj.expr), 6.592304628123309) def test_zero_time_limit(self): @@ -496,16 +496,14 @@ def test_zero_time_limit(self): self.assertIsNone(res.incumbent_objective) -class TestManualModel(unittest.TestCase): +class TestManualMode(unittest.TestCase): def setUp(self): opt = GurobiPersistent() - opt.config.auto_updates.check_for_new_or_removed_params = False - opt.config.auto_updates.check_for_new_or_removed_vars = False - opt.config.auto_updates.check_for_new_or_removed_constraints = False - opt.config.auto_updates.update_parameters = False - opt.config.auto_updates.update_vars = False - opt.config.auto_updates.update_constraints = False - opt.config.auto_updates.update_named_expressions = False + opt.auto_updates.check_for_new_or_removed_constraints = False + opt.auto_updates.update_parameters = False + opt.auto_updates.update_vars = False + opt.auto_updates.update_constraints = False + opt.auto_updates.update_named_expressions = False self.opt = opt def test_basics(self): @@ -603,16 +601,13 @@ def test_update1(self): opt = self.opt opt.set_instance(m) - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumQConstrs'), 1) opt.remove_constraints([m.c1]) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 0) + self.assertEqual(opt.get_model_attr('NumQConstrs'), 0) opt.add_constraints([m.c1]) - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 0) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumQConstrs'), 1) def test_update2(self): m = pyo.ConcreteModel() @@ -625,16 +620,13 @@ def test_update2(self): opt = self.opt opt.config.symbolic_solver_labels = True opt.set_instance(m) - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumConstrs'), 1) opt.remove_constraints([m.c2]) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 0) + self.assertEqual(opt.get_model_attr('NumConstrs'), 0) opt.add_constraints([m.c2]) - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 0) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumConstrs'), 1) def test_update3(self): m = pyo.ConcreteModel() @@ -684,16 +676,13 @@ def test_update5(self): opt = self.opt opt.set_instance(m) - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 1) + self.assertEqual(opt.get_model_attr('NumSOS'), 1) opt.remove_sos_constraints([m.c1]) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 0) + self.assertEqual(opt.get_model_attr('NumSOS'), 0) opt.add_sos_constraints([m.c1]) - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 0) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 1) + self.assertEqual(opt.get_model_attr('NumSOS'), 1) def test_update6(self): m = pyo.ConcreteModel() From 92fa4f5c72a4d26c5568a40e7ce7726ddd12e991 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:49:29 -0600 Subject: [PATCH 09/37] remove unused imports --- .../solver/solvers/gurobi/gurobi_direct.py | 20 ---------- .../solvers/gurobi/gurobi_direct_base.py | 7 +--- .../solvers/gurobi/gurobi_persistent.py | 38 +++---------------- 3 files changed, 7 insertions(+), 58 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index f4a33e2cc54..16c633c7d7c 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -9,40 +9,20 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -import datetime -import io -import math import operator -import os from pyomo.common.collections import ComponentMap, ComponentSet -from pyomo.common.config import ConfigValue -from pyomo.common.dependencies import attempt_import -from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import MouseTrap, ApplicationError from pyomo.common.shutdown import python_is_shutting_down -from pyomo.common.tee import capture_output, TeeStream -from pyomo.common.timing import HierarchicalTimer from pyomo.core.staleflag import StaleFlagManager from pyomo.repn.plugins.standard_form import LinearStandardFormCompiler -from pyomo.contrib.solver.common.base import SolverBase, Availability -from pyomo.contrib.solver.common.config import BranchAndBoundConfig from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, NoDualsError, NoReducedCostsError, NoSolutionError, IncompatibleModelError, ) -from pyomo.contrib.solver.common.results import ( - Results, - SolutionStatus, - TerminationCondition, -) from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase -import logging from .gurobi_direct_base import GurobiDirectBase, gurobipy diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index d26dbf54c83..41bdb244743 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -12,14 +12,13 @@ import datetime import io import math -import operator import os -from pyomo.common.collections import ComponentMap, ComponentSet +from pyomo.common.collections import ComponentMap from pyomo.common.config import ConfigValue from pyomo.common.dependencies import attempt_import from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import MouseTrap, ApplicationError +from pyomo.common.errors import ApplicationError from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer @@ -33,14 +32,12 @@ NoDualsError, NoReducedCostsError, NoSolutionError, - IncompatibleModelError, ) from pyomo.contrib.solver.common.results import ( Results, SolutionStatus, TerminationCondition, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase import logging diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index e91381f41a3..7b0463d2cf1 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -10,61 +10,33 @@ # ___________________________________________________________________________ from __future__ import annotations -import io import logging -import math -from typing import Dict, List, NoReturn, Optional, Sequence, Mapping +from typing import Dict, List, Optional, Sequence, Mapping from collections.abc import Iterable -from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet -from pyomo.common.dependencies import attempt_import -from pyomo.common.errors import ApplicationError -from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.collections import ComponentSet, OrderedSet from pyomo.common.timing import HierarchicalTimer -from pyomo.common.shutdown import python_is_shutting_down from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize -from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler from pyomo.core.base.var import VarData from pyomo.core.base.constraint import ConstraintData, Constraint from pyomo.core.base.sos import SOSConstraintData, SOSConstraint from pyomo.core.base.param import ParamData from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types from pyomo.repn import generate_standard_repn -from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability -from pyomo.contrib.solver.common.results import ( - Results, - TerminationCondition, - SolutionStatus, -) -from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig -from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, - NoDualsError, - NoReducedCostsError, - NoSolutionError, - IncompatibleModelError, -) -from pyomo.contrib.solver.common.persistent import ( - PersistentSolverUtils, - PersistentSolverMixin, -) -from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader, SolutionLoaderBase +from pyomo.contrib.solver.common.results import Results +from pyomo.contrib.solver.common.util import IncompatibleModelError +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( - GurobiConfig, GurobiDirectBase, gurobipy, - _load_suboptimal_mip_solution, _load_vars, _get_primals, _get_duals, _get_reduced_costs, ) from pyomo.contrib.solver.common.util import get_objective -from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector From 8a9fc46b802dcc181a2b00a131ab089dfc8c59a4 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:50:15 -0600 Subject: [PATCH 10/37] run black --- .../solvers/gurobi/gurobi_direct_base.py | 29 +-- .../solvers/gurobi/gurobi_persistent.py | 213 +++++++++++------- .../solver/tests/solvers/test_solvers.py | 11 +- 3 files changed, 153 insertions(+), 100 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 41bdb244743..df6bb8b5327 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -87,9 +87,7 @@ def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_ solver_model.getAttr('NumIntVars') == 0 and solver_model.getAttr('NumBinVars') == 0 ): - raise ValueError( - 'Cannot obtain suboptimal solutions for a continuous model' - ) + raise ValueError('Cannot obtain suboptimal solutions for a continuous model') original_solution_number = solver_model.getParamInfo('SolutionNumber')[2] solver_model.setParam('SolutionNumber', solution_number) gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] @@ -112,7 +110,7 @@ def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): for v, val in _get_primals( solver_model=solver_model, var_map=var_map, - vars_to_load=vars_to_load, + vars_to_load=vars_to_load, solution_number=solution_number, ).items(): v.set_value(val, skip_validation=True) @@ -177,7 +175,7 @@ def _get_duals(solver_model, con_map, linear_cons_to_load, quadratic_cons_to_loa """ if solver_model.Status != gurobipy.GRB.OPTIMAL: raise NoDualsError() - + linear_gurobi_cons = [con_map[c] for c in linear_cons_to_load] quadratic_gurobi_cons = [con_map[c] for c in quadratic_cons_to_load] linear_vals = solver_model.getAttr("Pi", linear_gurobi_cons) @@ -293,7 +291,7 @@ def _create_solver_model(self, pyomo_model): def _pyomo_gurobi_var_iter(self): # generator of tuples (pyomo_var, gurobi_var) raise NotImplementedError('should be implemented by derived classes') - + def _mipstart(self): for pyomo_var, gurobi_var in self._pyomo_gurobi_var_iter(): if pyomo_var.is_integer() and pyomo_var.value is not None: @@ -304,11 +302,8 @@ def solve(self, model, **kwds) -> Results: orig_config = self.config orig_cwd = os.getcwd() try: - config = self.config( - value=kwds, - preserve_implicit=True, - ) - + config = self.config(value=kwds, preserve_implicit=True) + # hack to work around legacy solver wrapper __setattr__ # otherwise, this would just be self.config = config object.__setattr__(self, 'config', config) @@ -329,7 +324,9 @@ def solve(self, model, **kwds) -> Results: if config.working_dir: os.chdir(config.working_dir) with capture_output(TeeStream(*ostreams), capture_fd=False): - gurobi_model, solution_loader, has_obj = self._create_solver_model(model) + gurobi_model, solution_loader, has_obj = self._create_solver_model( + model + ) options = config.solver_options gurobi_model.setParam('LogToConsole', 1) @@ -354,9 +351,7 @@ def solve(self, model, **kwds) -> Results: timer.stop('optimize') res = self._postsolve( - grb_model=gurobi_model, - solution_loader=solution_loader, - has_obj=has_obj, + grb_model=gurobi_model, solution_loader=solution_loader, has_obj=has_obj ) finally: os.chdir(orig_cwd) @@ -450,9 +445,9 @@ def _postsolve(self, grb_model, solution_loader, has_obj): raise NoFeasibleSolutionError() self.config.timer.stop('load solution') - # self.config gets copied a the beginning of + # self.config gets copied a the beginning of # solve and restored at the end, so modifying - # results.solver_config will not actually + # results.solver_config will not actually # modify self.config results.solver_config = self.config results.solver_name = self.name diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 7b0463d2cf1..6628f001421 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -29,8 +29,8 @@ from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( - GurobiDirectBase, - gurobipy, + GurobiDirectBase, + gurobipy, _load_vars, _get_primals, _get_duals, @@ -45,13 +45,7 @@ class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): def __init__( - self, - solver_model, - var_id_map, - var_map, - con_map, - linear_cons, - quadratic_cons, + self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons ) -> None: super().__init__() self._solver_model = solver_model @@ -62,9 +56,7 @@ def __init__( self._quadratic_cons = quadratic_cons def load_vars( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=0, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> None: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -76,9 +68,7 @@ def load_vars( ) def get_primals( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=0, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -88,10 +78,9 @@ def get_primals( vars_to_load=vars_to_load, solution_number=solution_id, ) - + def get_reduced_costs( - self, - vars_to_load: Optional[Sequence[VarData]] = None, + self, vars_to_load: Optional[Sequence[VarData]] = None ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -100,10 +89,9 @@ def get_reduced_costs( var_map=self._var_map, vars_to_load=vars_to_load, ) - + def get_duals( - self, - cons_to_load: Optional[Sequence[ConstraintData]] = None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None ) -> Dict[ConstraintData, float]: if cons_to_load is None: cons_to_load = list(self._con_map.keys()) @@ -124,8 +112,12 @@ def get_duals( class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): - def __init__(self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) -> None: - super().__init__(solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) + def __init__( + self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + ) -> None: + super().__init__( + solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + ) self._valid = True def invalidate(self): @@ -135,19 +127,27 @@ def _assert_solution_still_valid(self): if not self._valid: raise RuntimeError('The results in the solver are no longer valid.') - def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> None: + def load_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + ) -> None: self._assert_solution_still_valid() return super().load_vars(vars_to_load, solution_id) - - def get_primals(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> Mapping[VarData, float]: + + def get_primals( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_primals(vars_to_load, solution_id) - def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None) -> Dict[ConstraintData, float]: + def get_duals( + self, cons_to_load: Sequence[ConstraintData] | None = None + ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return super().get_duals(cons_to_load) - - def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None) -> Mapping[VarData, float]: + + def get_reduced_costs( + self, vars_to_load: Sequence[VarData] | None = None + ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_reduced_costs(vars_to_load) @@ -194,7 +194,9 @@ def update(self): class _MutableRangeConstant: - def __init__(self, lhs_expr, rhs_expr, pyomo_con, con_map, slack_name, gurobi_model): + def __init__( + self, lhs_expr, rhs_expr, pyomo_con, con_map, slack_name, gurobi_model + ): self.lhs_expr = lhs_expr self.rhs_expr = rhs_expr self.pyomo_con = pyomo_con @@ -268,7 +270,9 @@ def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): self.constant: _MutableConstant = constant self.linear_coefs: List[_MutableLinearCoefficient] = linear_coefs self.quadratic_coefs: List[_MutableQuadraticCoefficient] = quadratic_coefs - self.last_quadratic_coef_values: List[float] = [value(i.expr) for i in self.quadratic_coefs] + self.last_quadratic_coef_values: List[float] = [ + value(i.expr) for i in self.quadratic_coefs + ] def get_updated_expression(self): for ndx, coef in enumerate(self.linear_coefs): @@ -300,7 +304,7 @@ def __init__(self, expr, v1id, v2id, var_map): @property def var1(self): return self.var_map[self.v1id] - + @property def var2(self): return self.var_map[self.v2id] @@ -325,13 +329,21 @@ def _create_solver_model(self, pyomo_model): self._clear() self._solver_model = gurobipy.Model(env=self.env()) timer.start('collect constraints') - cons = list(pyomo_model.component_data_objects(Constraint, descend_into=True, active=True)) + cons = list( + pyomo_model.component_data_objects( + Constraint, descend_into=True, active=True + ) + ) timer.stop('collect constraints') timer.start('translate constraints') self._add_constraints(cons) timer.stop('translate constraints') timer.start('sos') - sos = list(pyomo_model.component_data_objects(SOSConstraint, descend_into=True, active=True)) + sos = list( + pyomo_model.component_data_objects( + SOSConstraint, descend_into=True, active=True + ) + ) self._add_sos_constraints(sos) timer.stop('sos') timer.start('get objective') @@ -351,7 +363,7 @@ def _create_solver_model(self, pyomo_model): ) timer.stop('create gurobipy model') return self._solver_model, solution_loader, has_obj - + def _clear(self): self._solver_model = None self._vars = {} @@ -379,9 +391,7 @@ def _process_domain_and_bounds(self, var): else: vtype = gurobipy.GRB.INTEGER else: - raise ValueError( - f'Unrecognized domain: {var.domain}' - ) + raise ValueError(f'Unrecognized domain: {var.domain}') if var.fixed: lb = var.value ub = lb @@ -415,16 +425,13 @@ def _get_expr_from_pyomo_repn(self, repn): raise IncompatibleModelError( f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' ) - + if len(repn.linear_vars) > 0: missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] self._add_variables(missing_vars) coef_list = [value(i) for i in repn.linear_coefs] vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] - new_expr = gurobipy.LinExpr( - coef_list, - vlist, - ) + new_expr = gurobipy.LinExpr(coef_list, vlist) else: new_expr = 0.0 @@ -455,8 +462,7 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobi_expr = self._get_expr_from_pyomo_repn(repn) if lb is None and ub is None: raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" + "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: gurobi_expr_list.append(gurobi_expr <= float(ub - repn.constant)) @@ -465,9 +471,14 @@ def _add_constraints(self, cons: List[ConstraintData]): elif lb == ub: gurobi_expr_list.append(gurobi_expr == float(lb - repn.constant)) else: - gurobi_expr_list.append(gurobi_expr == [float(lb-repn.constant), float(ub-repn.constant)]) + gurobi_expr_list.append( + gurobi_expr + == [float(lb - repn.constant), float(ub - repn.constant)] + ) - gurobi_cons = self._solver_model.addConstrs((gurobi_expr_list[i] for i in range(len(gurobi_expr_list)))).values() + gurobi_cons = self._solver_model.addConstrs( + (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))) + ).values() self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) def _add_sos_constraints(self, cons: List[SOSConstraintData]): @@ -485,7 +496,9 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): gurobi_vars = [] weights = [] - missing_vars = {id(v): v for v, w in con.get_items() if id(v) not in self._vars} + missing_vars = { + id(v): v for v, w in con.get_items() if id(v) not in self._vars + } self._add_variables(list(missing_vars.values())) for v, w in con.get_items(): @@ -608,7 +621,7 @@ def _create_solver_model(self, pyomo_model): ) has_obj = self._objective is not None return self._solver_model, solution_loader, has_obj - + def release_license(self): self._clear() self.__class__.release_license() @@ -617,17 +630,21 @@ def solve(self, model, **kwds) -> Results: res = super().solve(model, **kwds) self._needs_updated = False return res - + def _process_domain_and_bounds(self, var): res = super()._process_domain_and_bounds(var) if not is_constant(var._lb): - mutable_lb = _MutableLowerBound(id(var), var.lower, self._pyomo_var_to_solver_var_map) + mutable_lb = _MutableLowerBound( + id(var), var.lower, self._pyomo_var_to_solver_var_map + ) self._mutable_bounds[id(var), 'lb'] = (var, mutable_lb) if not is_constant(var._ub): - mutable_ub = _MutableUpperBound(id(var), var.upper, self._pyomo_var_to_solver_var_map) + mutable_ub = _MutableUpperBound( + id(var), var.upper, self._pyomo_var_to_solver_var_map + ) self._mutable_bounds[id(var), 'ub'] = (var, mutable_ub) return res - + def _add_variables(self, variables: List[VarData]): self._invalidate_last_results() super()._add_variables(variables) @@ -673,37 +690,60 @@ def _add_constraints(self, cons: List[ConstraintData]): mutable_constant = None if lb is None and ub is None: raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" + "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: rhs_expr = ub - repn.constant gurobi_expr_list.append(gurobi_expr <= float(value(rhs_expr))) if not is_constant(rhs_expr): - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) elif ub is None: rhs_expr = lb - repn.constant gurobi_expr_list.append(float(value(rhs_expr)) <= gurobi_expr) if not is_constant(rhs_expr): - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) elif con.equality: rhs_expr = lb - repn.constant gurobi_expr_list.append(gurobi_expr == float(value(rhs_expr))) if not is_constant(rhs_expr): - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) else: - assert len(repn.quadratic_vars) == 0, "Quadratic range constraints are not supported" + assert ( + len(repn.quadratic_vars) == 0 + ), "Quadratic range constraints are not supported" lhs_expr = lb - repn.constant rhs_expr = ub - repn.constant - gurobi_expr_list.append(gurobi_expr == [float(value(lhs_expr)), float(value(rhs_expr))]) + gurobi_expr_list.append( + gurobi_expr == [float(value(lhs_expr)), float(value(rhs_expr))] + ) if not is_constant(lhs_expr) or not is_constant(rhs_expr): conname = f'c{self._constraint_ndx}[{ndx}]' - mutable_constant = _MutableRangeConstant(lhs_expr, rhs_expr, con, self._pyomo_con_to_solver_con_map, 'Rg' + conname, self._solver_model) + mutable_constant = _MutableRangeConstant( + lhs_expr, + rhs_expr, + con, + self._pyomo_con_to_solver_con_map, + 'Rg' + conname, + self._solver_model, + ) mlc_list = [] for c, v in zip(repn.linear_coefs, repn.linear_vars): if not is_constant(c): - mlc = _MutableLinearCoefficient(c, con, self._pyomo_con_to_solver_con_map, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc = _MutableLinearCoefficient( + c, + con, + self._pyomo_con_to_solver_con_map, + id(v), + self._pyomo_var_to_solver_var_map, + self._solver_model, + ) mlc_list.append(mlc) if len(repn.quadratic_vars) == 0: @@ -715,15 +755,19 @@ def _add_constraints(self, cons: List[ConstraintData]): self._mutable_helpers[con].append(mutable_constant) else: if mutable_constant is None: - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) mqc_list = [] for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): if not is_constant(coef): - mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc = _MutableQuadraticCoefficient( + coef, id(x), id(y), self._pyomo_var_to_solver_var_map + ) mqc_list.append(mqc) mqc = _MutableQuadraticConstraint( self._solver_model, - con, + con, self._pyomo_con_to_solver_con_map, mutable_constant, mlc_list, @@ -731,10 +775,12 @@ def _add_constraints(self, cons: List[ConstraintData]): ) self._mutable_quadratic_helpers[con] = mqc - gurobi_cons = list(self._solver_model.addConstrs( - (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))), - name=f'c{self._constraint_ndx}' - ).values()) + gurobi_cons = list( + self._solver_model.addConstrs( + (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))), + name=f'c{self._constraint_ndx}', + ).values() + ) self._constraint_ndx += 1 self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) self._constraints_added_since_update.update(cons) @@ -761,7 +807,9 @@ def _set_objective(self, obj): else: raise ValueError(f'Objective sense is not recognized: {obj.sense}') - repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=False) + repn = generate_standard_repn( + obj.expr, quadratic=True, compute_values=False + ) repn_constant = value(repn.constant) gurobi_expr = self._get_expr_from_pyomo_repn(repn) @@ -770,16 +818,27 @@ def _set_objective(self, obj): mlc_list = [] for c, v in zip(repn.linear_coefs, repn.linear_vars): if not is_constant(c): - mlc = _MutableLinearCoefficient(c, None, None, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc = _MutableLinearCoefficient( + c, + None, + None, + id(v), + self._pyomo_var_to_solver_var_map, + self._solver_model, + ) mlc_list.append(mlc) mqc_list = [] for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): if not is_constant(coef): - mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc = _MutableQuadraticCoefficient( + coef, id(x), id(y), self._pyomo_var_to_solver_var_map + ) mqc_list.append(mqc) - self._mutable_objective = _MutableObjective(self._solver_model, mutable_constant, mlc_list, mqc_list) + self._mutable_objective = _MutableObjective( + self._solver_model, mutable_constant, mlc_list, mqc_list + ) # hack # see PR #2454 @@ -865,9 +924,7 @@ def _update_parameters(self, params: List[ParamData]): new_rhs = helper.get_updated_rhs() new_sense = gurobi_con.qcsense self._solver_model.remove(gurobi_con) - new_con = self._solver_model.addQConstr( - new_gurobi_expr, new_sense, new_rhs, - ) + new_con = self._solver_model.addQConstr(new_gurobi_expr, new_sense, new_rhs) self._pyomo_con_to_solver_con_map[con] = new_con helper.pyomo_con = con self._constraints_added_since_update.add(con) @@ -880,7 +937,7 @@ def _update_parameters(self, params: List[ParamData]): else: sense = gurobipy.GRB.MAXIMIZE # TODO: need a test for when part of the object is linear - # and part of the objective is quadratic, but both + # and part of the objective is quadratic, but both # parts have mutable coefficients self._solver_model.setObjective(new_gurobi_expr, sense=sense) @@ -1309,4 +1366,4 @@ def update_variables(self, variables): self._change_detector.update_variables(variables) def update_parameters(self, params): - self._change_detector.update_parameters(params) \ No newline at end of file + self._change_detector.update_parameters(params) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index a0d87835e13..96e2e7b2c38 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -31,7 +31,10 @@ from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.solvers.ipopt import Ipopt from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect -from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import ( + GurobiDirectQuadratic, + GurobiPersistent, +) from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -59,11 +62,9 @@ ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), ] -nlp_solvers = [ - ('ipopt', Ipopt), -] +nlp_solvers = [('ipopt', Ipopt)] qcp_solvers = [ - ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ] From 7249b1941c6b3aad5cd3f7a3bd34f7ab28ef3566 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:57:56 -0600 Subject: [PATCH 11/37] update solution loader --- .../solvers/gurobi/gurobi_persistent.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 6628f001421..05acfef2b4f 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -15,6 +15,7 @@ from collections.abc import Iterable from pyomo.common.collections import ComponentSet, OrderedSet +from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize @@ -54,6 +55,25 @@ def __init__( self._con_map = con_map self._linear_cons = linear_cons self._quadratic_cons = quadratic_cons + GurobiDirectBase._register_env_client() + + def __del__(self): + if python_is_shutting_down(): + return + # Free the associated model + if self._solver_model is not None: + self._vars = None + self._var_map = None + self._con_map = None + self._linear_cons = None + self._quadratic_cons = None + # explicitly release the model + self._solver_model.dispose() + self._solver_model = None + # Release the gurobi license if this is the last reference to + # the environment (either through a results object or solver + # interface) + GurobiDirectBase._release_env_client() def load_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 From 275d848d2c5eb09db7ac6b519794e1a0065b5869 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 17:43:11 -0600 Subject: [PATCH 12/37] run black --- pyomo/contrib/observer/component_collector.py | 5 +++- pyomo/contrib/observer/model_observer.py | 24 ++++++++-------- .../observer/tests/test_change_detector.py | 28 ++++++++++--------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index 5cbbdaf31bd..d52ec46086c 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -10,7 +10,10 @@ # ___________________________________________________________________________ from pyomo.core.expr.visitor import StreamBasedExpressionVisitor -from pyomo.core.expr.numeric_expr import ExternalFunctionExpression, NPV_ExternalFunctionExpression +from pyomo.core.expr.numeric_expr import ( + ExternalFunctionExpression, + NPV_ExternalFunctionExpression, +) from pyomo.core.base.var import VarData, ScalarVar from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.expression import ExpressionData, ScalarExpression diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index bd905e1c61d..4ab52100376 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -208,10 +208,7 @@ def update_parameters(self, params: List[ParamData]): class ModelChangeDetector: - def __init__( - self, observers: Sequence[Observer], - **kwds, - ): + def __init__(self, observers: Sequence[Observer], **kwds): """ Parameters ---------- @@ -237,13 +234,15 @@ def __init__( ) # var_id: [dict[constraints, None], dict[sos constraints, None], None or objective] self._referenced_params = ( {} - ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] + ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] self._vars_referenced_by_con = {} self._vars_referenced_by_obj = [] self._params_referenced_by_con = {} self._params_referenced_by_obj = [] self._expr_types = None - self.config: AutoUpdateConfig = AutoUpdateConfig()(value=kwds, preserve_implicit=True) + self.config: AutoUpdateConfig = AutoUpdateConfig()( + value=kwds, preserve_implicit=True + ) def set_instance(self, model): saved_config = self.config @@ -347,7 +346,10 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') sos_items = list(con.get_items()) - self._active_sos[con] = ([i[0] for i in sos_items], [i[1] for i in sos_items]) + self._active_sos[con] = ( + [i[0] for i in sos_items], + [i[1] for i in sos_items], + ) variables = [] params = [] for v, p in sos_items: @@ -616,14 +618,14 @@ def _check_for_var_changes(self): vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) return vars_to_update, cons_to_update, update_obj - + def _check_for_param_changes(self): params_to_update = [] for pid, (p, val) in self._params.items(): if p.value != val: params_to_update.append(p) return params_to_update - + def _check_for_named_expression_changes(self): cons_to_update = [] for con, ne_list in self._named_expressions.items(): @@ -644,7 +646,7 @@ def _check_for_new_objective(self): new_obj = get_objective(self._model) if new_obj is not self._objective: update_obj = True - return new_obj, update_obj + return new_obj, update_obj def _check_for_objective_changes(self): update_obj = False @@ -717,7 +719,7 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if update_obj: need_to_set_objective = True timer.stop('named expressions') - + timer.start('objective') new_obj = self._objective if config.check_for_new_objective: diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index efda8a181d9..29e0de01eb9 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -6,7 +6,11 @@ import pyomo.environ as pe from pyomo.common import unittest from typing import List -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.contrib.observer.model_observer import ( + Observer, + ModelChangeDetector, + AutoUpdateConfig, +) from pyomo.common.collections import ComponentMap import logging @@ -31,11 +35,9 @@ def __init__(self): def check(self, expected): unittest.assertStructuredAlmostEqual( - first=expected, - second=self.counts, - places=7, + first=expected, second=self.counts, places=7 ) - + def _process(self, comps, key): for c in comps: if c not in self.counts: @@ -120,7 +122,7 @@ def test_objective(self): detector.set_instance(m) obs.check(expected) - m.obj = pe.Objective(expr=m.x**2 + m.p*m.y**2) + m.obj = pe.Objective(expr=m.x**2 + m.p * m.y**2) detector.update() expected[m.obj] = make_count_dict() expected[m.obj]['set'] += 1 @@ -131,7 +133,7 @@ def test_objective(self): expected[m.p] = make_count_dict() expected[m.p]['add'] += 1 obs.check(expected) - + m.y.setlb(0) detector.update() expected[m.y]['update'] += 1 @@ -161,7 +163,7 @@ def test_objective(self): obs.check(expected) del m.obj - m.obj = pe.Objective(expr=m.p*m.x) + m.obj = pe.Objective(expr=m.p * m.x) detector.update() expected[m.p]['add'] += 1 expected[m.y]['remove'] += 1 @@ -186,7 +188,7 @@ def test_constraints(self): obs.check(expected) m.obj = pe.Objective(expr=m.y) - m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p)**2) + m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p) ** 2) detector.update() expected[m.x] = make_count_dict() expected[m.y] = make_count_dict() @@ -208,9 +210,9 @@ def test_constraints(self): obs.pprint() expected[m.c1]['remove'] += 1 expected[m.c1]['add'] += 1 - # because x and p are only used in the - # one constraint, they get removed when - # the constraint is removed and then + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then # added again when the constraint is added expected[m.x]['update'] += 1 expected[m.x]['remove'] += 1 @@ -220,4 +222,4 @@ def test_constraints(self): obs.check(expected) def test_vars_and_params_elsewhere(self): - pass \ No newline at end of file + pass From 1788ff371d52448a16b539c868aa40142c416b73 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 07:23:17 -0600 Subject: [PATCH 13/37] dont free gurobi models twice --- .../solver/solvers/gurobi/gurobi_persistent.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 05acfef2b4f..847ec958bdd 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -58,18 +58,6 @@ def __init__( GurobiDirectBase._register_env_client() def __del__(self): - if python_is_shutting_down(): - return - # Free the associated model - if self._solver_model is not None: - self._vars = None - self._var_map = None - self._con_map = None - self._linear_cons = None - self._quadratic_cons = None - # explicitly release the model - self._solver_model.dispose() - self._solver_model = None # Release the gurobi license if this is the last reference to # the environment (either through a results object or solver # interface) From a43a38bacc66aafa6af6c7305448e3f55a3fe263 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 16 Aug 2025 09:30:56 -0600 Subject: [PATCH 14/37] forgot to inherit from PersistentSolverBase --- .../solvers/gurobi/gurobi_persistent.py | 3 +- .../solver/tests/solvers/test_solvers.py | 139 ++++++++---------- 2 files changed, 64 insertions(+), 78 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 847ec958bdd..8d16bb9082e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -28,6 +28,7 @@ from pyomo.contrib.solver.common.results import Results from pyomo.contrib.solver.common.util import IncompatibleModelError from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +from pyomo.contrib.solver.common.base import PersistentSolverBase from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( GurobiDirectBase, @@ -575,7 +576,7 @@ def update_parameters(self, params: List[ParamData]): self.opt._update_parameters(params) -class GurobiPersistent(GurobiDirectQuadratic): +class GurobiPersistent(GurobiDirectQuadratic, PersistentSolverBase): _minimum_version = (7, 0, 0) def __init__(self, **kwds): diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 96e2e7b2c38..6965147d167 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -1221,55 +1221,50 @@ def test_mutable_quadratic_objective_qp( def test_fixed_vars( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): - for treat_fixed_vars_as_params in [True, False]: - opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=treat_fixed_vars_as_params) - if not opt.available(): - raise unittest.SkipTest(f'Solver {opt.name} not available.') - if any(name.startswith(i) for i in nl_solvers_set): - if use_presolve: - opt.config.writer_config.linear_presolve = True - else: - opt.config.writer_config.linear_presolve = False - m = pyo.ConcreteModel() - m.x = pyo.Var() - m.x.fix(0) - m.y = pyo.Var() - a1 = 1 - a2 = -1 - b1 = 1 - b2 = 2 - m.obj = pyo.Objective(expr=m.y) - m.c1 = pyo.Constraint(expr=m.y >= a1 * m.x + b1) - m.c2 = pyo.Constraint(expr=m.y >= a2 * m.x + b2) - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 2) - m.x.unfix() - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, (b2 - b1) / (a1 - a2)) - self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) - m.x.fix(0) - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 2) - m.x.value = 2 - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 2) - self.assertAlmostEqual(m.y.value, 3) - m.x.value = 0 - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 2) + opt: SolverBase = opt_class() + if not opt.available(): + raise unittest.SkipTest(f'Solver {opt.name} not available.') + if any(name.startswith(i) for i in nl_solvers_set): + if use_presolve: + opt.config.writer_config.linear_presolve = True + else: + opt.config.writer_config.linear_presolve = False + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.x.fix(0) + m.y = pyo.Var() + a1 = 1 + a2 = -1 + b1 = 1 + b2 = 2 + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.Constraint(expr=m.y >= a1 * m.x + b1) + m.c2 = pyo.Constraint(expr=m.y >= a2 * m.x + b2) + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 2) + m.x.unfix() + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, (b2 - b1) / (a1 - a2)) + self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) + m.x.fix(0) + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 2) + m.x.value = 2 + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 2) + self.assertAlmostEqual(m.y.value, 3) + m.x.value = 0 + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 2) @parameterized.expand(input=_load_tests(all_solvers)) def test_fixed_vars_2( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=True) if not opt.available(): raise unittest.SkipTest(f'Solver {opt.name} not available.') if any(name.startswith(i) for i in nl_solvers_set): @@ -1313,8 +1308,6 @@ def test_fixed_vars_3( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=True) if not opt.available(): raise unittest.SkipTest(f'Solver {opt.name} not available.') if any(name.startswith(i) for i in nl_solvers_set): @@ -1337,8 +1330,6 @@ def test_fixed_vars_4( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=True) if not opt.available(): raise unittest.SkipTest(f'Solver {opt.name} not available.') if any(name.startswith(i) for i in nl_solvers_set): @@ -1892,10 +1883,7 @@ def test_fixed_binaries( res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 1) - if opt.is_persistent(): - opt: SolverBase = opt_class(treat_fixed_vars_as_params=False) - else: - opt = opt_class() + opt = opt_class() m.x.fix(0) res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 0) @@ -2049,33 +2037,30 @@ def test_bug_2(self, name: str, opt_class: Type[SolverBase], use_presolve: bool) This test is for a bug where an objective containing a fixed variable does not get updated properly when the variable is unfixed. """ - for fixed_var_option in [True, False]: - opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=fixed_var_option) - if not opt.available(): - raise unittest.SkipTest(f'Solver {opt.name} not available.') - if any(name.startswith(i) for i in nl_solvers_set): - if use_presolve: - opt.config.writer_config.linear_presolve = True - else: - opt.config.writer_config.linear_presolve = False - - m = pyo.ConcreteModel() - m.x = pyo.Var(bounds=(-10, 10)) - m.y = pyo.Var() - m.obj = pyo.Objective(expr=3 * m.y - m.x) - m.c = pyo.Constraint(expr=m.y >= m.x) - - m.x.fix(1) - res = opt.solve(m) - self.assertAlmostEqual(res.incumbent_objective, 2, 5) + opt: SolverBase = opt_class() + if not opt.available(): + raise unittest.SkipTest(f'Solver {opt.name} not available.') + if any(name.startswith(i) for i in nl_solvers_set): + if use_presolve: + opt.config.writer_config.linear_presolve = True + else: + opt.config.writer_config.linear_presolve = False - m.x.unfix() - m.x.setlb(-9) - m.x.setub(9) - res = opt.solve(m) - self.assertAlmostEqual(res.incumbent_objective, -18, 5) + m = pyo.ConcreteModel() + m.x = pyo.Var(bounds=(-10, 10)) + m.y = pyo.Var() + m.obj = pyo.Objective(expr=3 * m.y - m.x) + m.c = pyo.Constraint(expr=m.y >= m.x) + + m.x.fix(1) + res = opt.solve(m) + self.assertAlmostEqual(res.incumbent_objective, 2, 5) + + m.x.unfix() + m.x.setlb(-9) + m.x.setub(9) + res = opt.solve(m) + self.assertAlmostEqual(res.incumbent_objective, -18, 5) @parameterized.expand(input=_load_tests(nl_solvers)) def test_presolve_with_zero_coef( From e76baae7b130aa3d845e053c802a93ca5e76de26 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 16 Aug 2025 12:25:39 -0600 Subject: [PATCH 15/37] bug --- pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 8d16bb9082e..8145777ffb9 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -595,6 +595,7 @@ def __init__(self, **kwds): self._observer = _GurobiObserver(self) self._change_detector = ModelChangeDetector(observers=[self._observer]) self._constraint_ndx = 0 + self._should_update_parameters = False @property def auto_updates(self): @@ -683,6 +684,8 @@ def update(self): if self._needs_updated: self._update_gurobi_model() self._change_detector.update(timer=timer) + if self._should_update_parameters: + self._update_parameters([]) timer.stop('update') def _add_constraints(self, cons: List[ConstraintData]): @@ -915,6 +918,8 @@ def _update_variables(self, variables: List[VarData]): gurobipy_var.setAttr('lb', lb) gurobipy_var.setAttr('ub', ub) gurobipy_var.setAttr('vtype', vtype) + if var.fixed: + self._should_update_parameters = True self._needs_updated = True def _update_parameters(self, params: List[ParamData]): @@ -950,6 +955,8 @@ def _update_parameters(self, params: List[ParamData]): # parts have mutable coefficients self._solver_model.setObjective(new_gurobi_expr, sense=sense) + self._should_update_parameters = False + def _invalidate_last_results(self): if self._last_results_object is not None: self._last_results_object.solution_loader.invalidate() From c2a01777d43d47d7ea072c5b93ddc020e8b787ee Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 18 Aug 2025 09:13:12 -0600 Subject: [PATCH 16/37] bug --- pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 8145777ffb9..603e8e21800 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -432,7 +432,7 @@ def _add_variables(self, variables: List[VarData]): def _get_expr_from_pyomo_repn(self, repn): if repn.nonlinear_expr is not None: raise IncompatibleModelError( - f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' + f'GurobiDirectQuadratic only supports linear and quadratic expressions: {repn}.' ) if len(repn.linear_vars) > 0: From 576a21768c7c8acbdfb63d863c69fe603017a6c3 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 2 Oct 2025 07:22:54 -0600 Subject: [PATCH 17/37] run black --- pyomo/contrib/solver/tests/solvers/test_solvers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 192f1536800..74d3b7ccdbc 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -22,7 +22,10 @@ from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.config import SolverConfig from pyomo.contrib.solver.common.factory import SolverFactory -from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent, GurobiDirectQuadratic +from pyomo.contrib.solver.solvers.gurobi_persistent import ( + GurobiPersistent, + GurobiDirectQuadratic, +) from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect from pyomo.contrib.solver.solvers.highs import Highs from pyomo.contrib.solver.solvers.ipopt import Ipopt From ce99fb2597611449874d78d02678d35fb21eaf47 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 4 Oct 2025 15:56:11 -0600 Subject: [PATCH 18/37] observer improvements --- pyomo/contrib/observer/component_collector.py | 7 +- pyomo/contrib/observer/model_observer.py | 166 +++++++++++------- .../solvers/gurobi/gurobi_persistent.py | 166 ++++++++++++------ .../tests/solvers/test_gurobi_persistent.py | 10 +- .../solver/tests/solvers/test_solvers.py | 10 +- 5 files changed, 229 insertions(+), 130 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index d30bb128758..22e66aa6c80 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -33,6 +33,7 @@ from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.expression import ExpressionData, ScalarExpression from pyomo.repn.util import ExitNodeDispatcher +from pyomo.common.numeric_types import native_numeric_types def handle_var(node, collector): @@ -79,8 +80,6 @@ def handle_skip(node, collector): collector_handlers[RangedExpression] = handle_skip collector_handlers[InequalityExpression] = handle_skip collector_handlers[EqualityExpression] = handle_skip -collector_handlers[int] = handle_skip -collector_handlers[float] = handle_skip class _ComponentFromExprCollector(StreamBasedExpressionVisitor): @@ -92,6 +91,10 @@ def __init__(self, **kwds): super().__init__(**kwds) def exitNode(self, node, data): + if type(node) in native_numeric_types: + # we need this here to handle numpy + # (we can't put numpy in the dispatcher?) + return None return collector_handlers[node.__class__](node, self) def beforeChild(self, node, child, child_idx): diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index b0a5b07fdfd..71c1a7c5460 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -16,16 +16,18 @@ from pyomo.core.base.constraint import ConstraintData, Constraint from pyomo.core.base.sos import SOSConstraintData, SOSConstraint from pyomo.core.base.var import VarData -from pyomo.core.base.param import ParamData +from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.objective import ObjectiveData, Objective -from pyomo.core.base.block import BlockData +from pyomo.core.base.block import BlockData, Block from pyomo.core.base.component import ActiveComponent +from pyomo.core.base.suffix import Suffix from pyomo.common.collections import ComponentMap from pyomo.common.timing import HierarchicalTimer from pyomo.contrib.solver.common.util import get_objective from pyomo.contrib.observer.component_collector import collect_components_from_expr from pyomo.common.numeric_types import native_numeric_types import gc +import warnings """ @@ -49,6 +51,9 @@ """ +_param_types = {ParamData, ScalarParam} + + @document_configdict() class AutoUpdateConfig(ConfigDict): """ @@ -492,7 +497,7 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): observers: Sequence[Observer] The objects to notify when changes are made to the model """ - self._known_active_ctypes = {Constraint, SOSConstraint, Objective} + self._known_active_ctypes = {Constraint, SOSConstraint, Objective, Block} self._observers: List[Observer] = list(observers) self._active_constraints = {} # maps constraint to expression self._active_sos = {} @@ -520,12 +525,14 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): # dict[constraints, None], # dict[sos constraints, None], # dict[objectives, None], + # dict[var_id, None], # ) self._referenced_params = {} self._vars_referenced_by_con = {} self._vars_referenced_by_obj = {} self._params_referenced_by_con = {} + self._params_referenced_by_var = {} # for when parameters show up in variable bounds self._params_referenced_by_obj = {} self.config: AutoUpdateConfig = AutoUpdateConfig()( @@ -536,11 +543,13 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): self._set_instance() def add_variables(self, variables: List[VarData]): + params_to_check = {} for v in variables: - if id(v) in self._referenced_variables: + vid = id(v) + if vid in self._referenced_variables: raise ValueError(f'Variable {v.name} has already been added') - self._referenced_variables[id(v)] = ({}, {}, {}) - self._vars[id(v)] = ( + self._referenced_variables[vid] = ({}, {}, {}) + self._vars[vid] = ( v, v._lb, v._ub, @@ -548,6 +557,31 @@ def add_variables(self, variables: List[VarData]): v.domain.get_interval(), v.value, ) + ref_params = set() + for bnd in (v._lb, v._ub): + if bnd is None or type(bnd) in native_numeric_types: + continue + (named_exprs, _vars, parameters, external_functions) = ( + collect_components_from_expr(bnd) + ) + if _vars: + raise NotImplementedError('ModelChangeDetector does not support variables in the bounds of other variables') + if named_exprs: + raise NotImplementedError('ModelChangeDetector does not support Expressions in the bounds of other variables') + if external_functions: + raise NotImplementedError('ModelChangeDetector does not support external functions in the bounds of other variables') + params_to_check.update((id(p), p) for p in parameters) + if vid not in self._params_referenced_by_var: + self._params_referenced_by_var[vid] = [] + self._params_referenced_by_var[vid].extend(p for p in parameters if id(p) not in ref_params) + ref_params.update(id(p) for p in parameters) + self._check_for_new_params(list(params_to_check.values())) + for v in variables: + if id(v) not in self._params_referenced_by_var: + continue + parameters = self._params_referenced_by_var[id(v)] + for p in parameters: + self._referenced_params[id(p)][3][id(v)] = None for obs in self._observers: obs.add_variables(variables) @@ -556,46 +590,46 @@ def add_parameters(self, params: List[ParamData]): pid = id(p) if pid in self._referenced_params: raise ValueError(f'Parameter {p.name} has already been added') - self._referenced_params[pid] = ({}, {}, {}) + self._referenced_params[pid] = ({}, {}, {}, {}) self._params[id(p)] = (p, p.value) for obs in self._observers: obs.add_parameters(params) def _check_for_new_vars(self, variables: List[VarData]): - new_vars = [] + new_vars = {} for v in variables: if id(v) not in self._referenced_variables: - new_vars.append(v) - self.add_variables(new_vars) + new_vars[id(v)] = v + self.add_variables(list(new_vars.values())) def _check_to_remove_vars(self, variables: List[VarData]): - vars_to_remove = [] + vars_to_remove = {} for v in variables: v_id = id(v) ref_cons, ref_sos, ref_obj = self._referenced_variables[v_id] if not ref_cons and not ref_sos and not ref_obj: - vars_to_remove.append(v) - self._remove_variables(vars_to_remove) + vars_to_remove[v_id] = v + self.remove_variables(list(vars_to_remove.values())) def _check_for_new_params(self, params: List[ParamData]): - new_params = [] + new_params = {} for p in params: if id(p) not in self._referenced_params: - new_params.append(p) - self.add_parameters(new_params) + new_params[id(p)] = p + self.add_parameters(list(new_params.values())) def _check_to_remove_params(self, params: List[ParamData]): - params_to_remove = [] + params_to_remove = {} for p in params: p_id = id(p) - ref_cons, ref_sos, ref_obj = self._referenced_params[p_id] - if not ref_cons and not ref_sos and not ref_obj: - params_to_remove.append(p) - self._remove_parameters(params_to_remove) + ref_cons, ref_sos, ref_obj, ref_vars = self._referenced_params[p_id] + if not ref_cons and not ref_sos and not ref_obj and not ref_vars: + params_to_remove[p_id] = p + self.remove_parameters(list(params_to_remove.values())) def add_constraints(self, cons: List[ConstraintData]): - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for con in cons: if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') @@ -603,16 +637,16 @@ def add_constraints(self, cons: List[ConstraintData]): (named_exprs, variables, parameters, external_functions) = ( collect_components_from_expr(con.expr) ) - vars_to_check.extend(variables) - params_to_check.extend(parameters) + vars_to_check.update((id(v), v) for v in variables) + params_to_check.update((id(p), p) for p in parameters) if named_exprs: self._named_expressions[con] = [(e, e.expr) for e in named_exprs] if external_functions: self._external_functions[con] = external_functions self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = parameters - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) + self._check_for_new_vars(list(vars_to_check.values())) + self._check_for_new_params(list(params_to_check.values())) for con in cons: variables = self._vars_referenced_by_con[con] parameters = self._params_referenced_by_con[con] @@ -624,8 +658,8 @@ def add_constraints(self, cons: List[ConstraintData]): obs.add_constraints(cons) def add_sos_constraints(self, cons: List[SOSConstraintData]): - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for con in cons: if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') @@ -642,12 +676,12 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): continue if p.is_parameter_type(): params.append(p) - vars_to_check.extend(variables) - params_to_check.extend(params) + vars_to_check.update((id(v), v) for v in variables) + params_to_check.update((id(p), p) for p in params) self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) + self._check_for_new_vars(list(vars_to_check.values())) + self._check_for_new_params(list(params_to_check.values())) for con in cons: variables = self._vars_referenced_by_con[con] params = self._params_referenced_by_con[con] @@ -659,24 +693,24 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): obs.add_sos_constraints(cons) def add_objectives(self, objs: List[ObjectiveData]): - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for obj in objs: obj_id = id(obj) self._objectives[obj_id] = (obj, obj.expr, obj.sense) (named_exprs, variables, parameters, external_functions) = ( collect_components_from_expr(obj.expr) ) - vars_to_check.extend(variables) - params_to_check.extend(parameters) + vars_to_check.update((id(v), v) for v in variables) + params_to_check.update((id(p), p) for p in parameters) if named_exprs: self._obj_named_expressions[obj_id] = [(e, e.expr) for e in named_exprs] if external_functions: self._external_functions[obj] = external_functions self._vars_referenced_by_obj[obj_id] = variables self._params_referenced_by_obj[obj_id] = parameters - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) + self._check_for_new_vars(list(vars_to_check.values())) + self._check_for_new_params(list(params_to_check.values())) for obj in objs: obj_id = id(obj) variables = self._vars_referenced_by_obj[obj_id] @@ -692,8 +726,8 @@ def remove_objectives(self, objs: List[ObjectiveData]): for obs in self._observers: obs.remove_objectives(objs) - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for obj in objs: obj_id = id(obj) if obj_id not in self._objectives: @@ -704,15 +738,15 @@ def remove_objectives(self, objs: List[ObjectiveData]): self._referenced_variables[id(v)][2].pop(obj_id) for p in self._params_referenced_by_obj[obj_id]: self._referenced_params[id(p)][2].pop(obj_id) - vars_to_check.extend(self._vars_referenced_by_obj[obj_id]) - params_to_check.extend(self._params_referenced_by_obj[obj_id]) + vars_to_check.update((id(v), v) for v in self._vars_referenced_by_obj[obj_id]) + params_to_check.update((id(p), p) for p in self._params_referenced_by_obj[obj_id]) del self._objectives[obj_id] self._obj_named_expressions.pop(obj_id, None) self._external_functions.pop(obj, None) del self._vars_referenced_by_obj[obj_id] del self._params_referenced_by_obj[obj_id] - self._check_to_remove_vars(vars_to_check) - self._check_to_remove_params(params_to_check) + self._check_to_remove_vars(list(vars_to_check.values())) + self._check_to_remove_params(list(params_to_check.values())) def _check_for_unknown_active_components(self): for ctype in self._model.collect_ctypes(): @@ -723,9 +757,12 @@ def _check_for_unknown_active_components(self): for comp in self._model.component_data_objects( ctype, active=True, descend_into=True ): + if isinstance(comp, Suffix): + warnings.warn('ModelChangeDetector does not detect changes to suffixes') + continue raise NotImplementedError( - f'ModelChangeDetector does not know how to ' - 'handle components with ctype {ctype}' + 'ModelChangeDetector does not know how to ' + f'handle components with ctype {ctype}' ) def _set_instance(self): @@ -764,8 +801,8 @@ def _set_instance(self): def remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.remove_constraints(cons) - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for con in cons: if con not in self._active_constraints: raise ValueError( @@ -775,21 +812,21 @@ def remove_constraints(self, cons: List[ConstraintData]): self._referenced_variables[id(v)][0].pop(con) for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][0].pop(con) - vars_to_check.extend(self._vars_referenced_by_con[con]) - params_to_check.extend(self._params_referenced_by_con[con]) + vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) + params_to_check.update((id(p), p) for p in self._params_referenced_by_con[con]) del self._active_constraints[con] self._named_expressions.pop(con, None) self._external_functions.pop(con, None) del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] - self._check_to_remove_vars(vars_to_check) - self._check_to_remove_params(params_to_check) + self._check_to_remove_vars(list(vars_to_check.values())) + self._check_to_remove_params(list(params_to_check.values())) def remove_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.remove_sos_constraints(cons) - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for con in cons: if con not in self._active_sos: raise ValueError( @@ -799,23 +836,29 @@ def remove_sos_constraints(self, cons: List[SOSConstraintData]): self._referenced_variables[id(v)][1].pop(con) for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][1].pop(con) - vars_to_check.extend(self._vars_referenced_by_con[con]) - params_to_check.extend(self._params_referenced_by_con[con]) + vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) + params_to_check.update((id(p), p) for p in self._params_referenced_by_con[con]) del self._active_sos[con] del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] - self._check_to_remove_vars(vars_to_check) - self._check_to_remove_params(params_to_check) + self._check_to_remove_vars(list(vars_to_check.values())) + self._check_to_remove_params(list(params_to_check.values())) def remove_variables(self, variables: List[VarData]): for obs in self._observers: obs.remove_variables(variables) + params_to_check = {} for v in variables: v_id = id(v) if v_id not in self._referenced_variables: raise ValueError( f'Cannot remove variable {v.name} - it has not been added' ) + if v_id in self._params_referenced_by_var: + for p in self._params_referenced_by_var[v_id]: + self._referenced_params[id(p)][3].pop(v_id) + params_to_check.update((id(p), p) for p in self._params_referenced_by_var[v_id]) + self._params_referenced_by_var.pop(v_id) cons_using, sos_using, obj_using = self._referenced_variables[v_id] if cons_using or sos_using or obj_using: raise ValueError( @@ -823,6 +866,7 @@ def remove_variables(self, variables: List[VarData]): ) del self._referenced_variables[v_id] del self._vars[v_id] + self._check_to_remove_params(list(params_to_check.values())) def remove_parameters(self, params: List[ParamData]): for obs in self._observers: @@ -833,8 +877,8 @@ def remove_parameters(self, params: List[ParamData]): raise ValueError( f'Cannot remove parameter {p.name} - it has not been added' ) - cons_using, sos_using, obj_using = self._referenced_params[p_id] - if cons_using or sos_using or obj_using: + cons_using, sos_using, obj_using, vars_using = self._referenced_params[p_id] + if cons_using or sos_using or obj_using or vars_using: raise ValueError( f'Cannot remove parameter {p.name} - it is still being used by constraints/objectives' ) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 603e8e21800..74952af2b49 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -15,6 +15,7 @@ from collections.abc import Iterable from pyomo.common.collections import ComponentSet, OrderedSet +from pyomo.common.errors import PyomoException from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer from pyomo.core.base.objective import ObjectiveData @@ -33,13 +34,14 @@ from .gurobi_direct_base import ( GurobiDirectBase, gurobipy, + GurobiConfig, _load_vars, _get_primals, _get_duals, _get_reduced_costs, ) from pyomo.contrib.solver.common.util import get_objective -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig logger = logging.getLogger(__name__) @@ -539,7 +541,7 @@ def _set_objective(self, obj): class _GurobiObserver(Observer): - def __init__(self, opt: GurobiPersistentQuadratic) -> None: + def __init__(self, opt: GurobiPersistent) -> None: self.opt = opt def add_variables(self, variables: List[VarData]): @@ -554,8 +556,11 @@ def add_constraints(self, cons: List[ConstraintData]): def add_sos_constraints(self, cons: List[SOSConstraintData]): self.opt._add_sos_constraints(cons) - def set_objective(self, obj: ObjectiveData | None): - self.opt._set_objective(obj) + def add_objectives(self, objs: List[ObjectiveData]): + self.opt._add_objectives(objs) + + def remove_objectives(self, objs: List[ObjectiveData]): + self.opt._remove_objectives(objs) def remove_constraints(self, cons: List[ConstraintData]): self.opt._remove_constraints(cons) @@ -576,8 +581,31 @@ def update_parameters(self, params: List[ParamData]): self.opt._update_parameters(params) +class GurobiPersistentConfig(GurobiConfig): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + GurobiConfig.__init__( + self, + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + self.auto_updates: bool = self.declare( + 'auto_updates', AutoUpdateConfig() + ) + + class GurobiPersistent(GurobiDirectQuadratic, PersistentSolverBase): _minimum_version = (7, 0, 0) + CONFIG = GurobiPersistentConfig() def __init__(self, **kwds): super().__init__(**kwds) @@ -592,15 +620,11 @@ def __init__(self, **kwds): self._constraints_added_since_update = OrderedSet() self._vars_added_since_update = ComponentSet() self._last_results_object: Optional[Results] = None - self._observer = _GurobiObserver(self) - self._change_detector = ModelChangeDetector(observers=[self._observer]) + self._observer = None + self._change_detector = None self._constraint_ndx = 0 self._should_update_parameters = False - @property - def auto_updates(self): - return self._change_detector.config - def _clear(self): super()._clear() self._pyomo_model = None @@ -669,8 +693,10 @@ def set_instance(self, pyomo_model): self._clear() self._pyomo_model = pyomo_model self._solver_model = gurobipy.Model(env=self.env()) + self._observer = _GurobiObserver(self) timer.start('set_instance') - self._change_detector.set_instance(pyomo_model) + self._change_detector = ModelChangeDetector(model=self._pyomo_model, observers=[self._observer], **dict(self.config.auto_updates)) + self._change_detector.config = self.config.auto_updates timer.stop('set_instance') def update(self): @@ -804,59 +830,83 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): self._constraints_added_since_update.update(cons) self._needs_updated = True - def _set_objective(self, obj): - self._invalidate_last_results() - if obj is None: - sense = gurobipy.GRB.MINIMIZE - gurobi_expr = 0 - repn_constant = 0 - self._mutable_objective = None - else: - if obj.sense == minimize: - sense = gurobipy.GRB.MINIMIZE - elif obj.sense == maximize: - sense = gurobipy.GRB.MAXIMIZE + def _remove_objectives(self, objs: List[ObjectiveData]): + for obj in objs: + if obj is not self._objective: + raise RuntimeError( + 'tried to remove an objective that has not been added: ' \ + f'{str(obj)}' + ) else: - raise ValueError(f'Objective sense is not recognized: {obj.sense}') + self._invalidate_last_results() + self._solver_model.setObjective(0, sense=gurobipy.GRB.MINIMIZE) + # see PR #2454 + self._solver_model.update() + self._objective = None + self._needs_updated = False + + def _add_objectives(self, objs: List[ObjectiveData]): + if len(objs) > 1: + raise NotImplementedError( + 'the persistent interface to gurobi currently ' \ + f'only supports single-objective problems; got {len(objs)}: ' + f'{[str(i) for i in objs]}' + ) + + if len(objs) == 0: + return + + obj = objs[0] - repn = generate_standard_repn( - obj.expr, quadratic=True, compute_values=False + if self._objective is not None: + raise NotImplementedError( + 'the persistent interface to gurobi currently ' \ + 'only supports single-objective problems; tried to add ' \ + f'an objective ({str(obj)}), but there is already an ' \ + f'active objective ({str(self._objective)})' ) - repn_constant = value(repn.constant) - gurobi_expr = self._get_expr_from_pyomo_repn(repn) - mutable_constant = _MutableConstant(repn.constant, None, None) + self._invalidate_last_results() - mlc_list = [] - for c, v in zip(repn.linear_coefs, repn.linear_vars): - if not is_constant(c): - mlc = _MutableLinearCoefficient( - c, - None, - None, - id(v), - self._pyomo_var_to_solver_var_map, - self._solver_model, - ) - mlc_list.append(mlc) + if obj.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + elif obj.sense == maximize: + sense = gurobipy.GRB.MAXIMIZE + else: + raise ValueError(f'Objective sense is not recognized: {obj.sense}') - mqc_list = [] - for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): - if not is_constant(coef): - mqc = _MutableQuadraticCoefficient( - coef, id(x), id(y), self._pyomo_var_to_solver_var_map - ) - mqc_list.append(mqc) + repn = generate_standard_repn( + obj.expr, quadratic=True, compute_values=False + ) + repn_constant = value(repn.constant) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) - self._mutable_objective = _MutableObjective( - self._solver_model, mutable_constant, mlc_list, mqc_list - ) + mutable_constant = _MutableConstant(repn.constant, None, None) + + mlc_list = [] + for c, v in zip(repn.linear_coefs, repn.linear_vars): + if not is_constant(c): + mlc = _MutableLinearCoefficient( + c, + None, + None, + id(v), + self._pyomo_var_to_solver_var_map, + self._solver_model, + ) + mlc_list.append(mlc) - # hack - # see PR #2454 - if self._objective is not None: - self._solver_model.setObjective(0) - self._solver_model.update() + mqc_list = [] + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + if not is_constant(coef): + mqc = _MutableQuadraticCoefficient( + coef, id(x), id(y), self._pyomo_var_to_solver_var_map + ) + mqc_list.append(mqc) + + self._mutable_objective = _MutableObjective( + self._solver_model, mutable_constant, mlc_list, mqc_list + ) self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) self._objective = obj @@ -1366,8 +1416,8 @@ def add_constraints(self, cons): def add_sos_constraints(self, cons): self._change_detector.add_sos_constraints(cons) - def set_objective(self, obj): - self._change_detector.set_objective(obj) + def set_objective(self, obj: ObjectiveData): + self._change_detector.add_objectives([obj]) def remove_constraints(self, cons): self._change_detector.remove_constraints(cons) diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py index 96cd1498956..24b53a19f2b 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py @@ -499,11 +499,11 @@ def test_zero_time_limit(self): class TestManualMode(unittest.TestCase): def setUp(self): opt = GurobiPersistent() - opt.auto_updates.check_for_new_or_removed_constraints = False - opt.auto_updates.update_parameters = False - opt.auto_updates.update_vars = False - opt.auto_updates.update_constraints = False - opt.auto_updates.update_named_expressions = False + opt.config.auto_updates.check_for_new_or_removed_constraints = False + opt.config.auto_updates.update_parameters = False + opt.config.auto_updates.update_vars = False + opt.config.auto_updates.update_constraints = False + opt.config.auto_updates.update_named_expressions = False self.opt = opt def test_basics(self): diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 74d3b7ccdbc..e7ff00f7f41 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -22,7 +22,7 @@ from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.config import SolverConfig from pyomo.contrib.solver.common.factory import SolverFactory -from pyomo.contrib.solver.solvers.gurobi_persistent import ( +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import ( GurobiPersistent, GurobiDirectQuadratic, ) @@ -1537,9 +1537,7 @@ def test_add_and_remove_vars( opt.config.auto_updates.update_vars = False opt.config.auto_updates.update_constraints = False opt.config.auto_updates.update_named_expressions = False - opt.config.auto_updates.check_for_new_or_removed_params = False opt.config.auto_updates.check_for_new_or_removed_constraints = False - opt.config.auto_updates.check_for_new_or_removed_vars = False opt.config.load_solutions = False res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) @@ -1864,7 +1862,11 @@ def test_objective_changes( res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 3) if opt.is_persistent(): - opt.config.auto_updates.check_for_new_objective = False + # hack until we get everything ported to the observer + try: + opt.config.auto_updates.check_for_new_or_removed_objectives = False + except: + opt.config.auto_updates.check_for_new_objective = False m.e.expr = 4 res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 4) From 066e4fdf80f401194d63355495ba43f10da073e8 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 4 Oct 2025 15:56:58 -0600 Subject: [PATCH 19/37] run black --- pyomo/contrib/observer/model_observer.py | 44 ++++++++++++++----- .../solvers/gurobi/gurobi_persistent.py | 30 +++++++------ 2 files changed, 50 insertions(+), 24 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 71c1a7c5460..2da340aab4f 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -532,7 +532,9 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): self._vars_referenced_by_con = {} self._vars_referenced_by_obj = {} self._params_referenced_by_con = {} - self._params_referenced_by_var = {} # for when parameters show up in variable bounds + self._params_referenced_by_var = ( + {} + ) # for when parameters show up in variable bounds self._params_referenced_by_obj = {} self.config: AutoUpdateConfig = AutoUpdateConfig()( @@ -565,15 +567,23 @@ def add_variables(self, variables: List[VarData]): collect_components_from_expr(bnd) ) if _vars: - raise NotImplementedError('ModelChangeDetector does not support variables in the bounds of other variables') + raise NotImplementedError( + 'ModelChangeDetector does not support variables in the bounds of other variables' + ) if named_exprs: - raise NotImplementedError('ModelChangeDetector does not support Expressions in the bounds of other variables') + raise NotImplementedError( + 'ModelChangeDetector does not support Expressions in the bounds of other variables' + ) if external_functions: - raise NotImplementedError('ModelChangeDetector does not support external functions in the bounds of other variables') + raise NotImplementedError( + 'ModelChangeDetector does not support external functions in the bounds of other variables' + ) params_to_check.update((id(p), p) for p in parameters) if vid not in self._params_referenced_by_var: self._params_referenced_by_var[vid] = [] - self._params_referenced_by_var[vid].extend(p for p in parameters if id(p) not in ref_params) + self._params_referenced_by_var[vid].extend( + p for p in parameters if id(p) not in ref_params + ) ref_params.update(id(p) for p in parameters) self._check_for_new_params(list(params_to_check.values())) for v in variables: @@ -738,8 +748,12 @@ def remove_objectives(self, objs: List[ObjectiveData]): self._referenced_variables[id(v)][2].pop(obj_id) for p in self._params_referenced_by_obj[obj_id]: self._referenced_params[id(p)][2].pop(obj_id) - vars_to_check.update((id(v), v) for v in self._vars_referenced_by_obj[obj_id]) - params_to_check.update((id(p), p) for p in self._params_referenced_by_obj[obj_id]) + vars_to_check.update( + (id(v), v) for v in self._vars_referenced_by_obj[obj_id] + ) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_obj[obj_id] + ) del self._objectives[obj_id] self._obj_named_expressions.pop(obj_id, None) self._external_functions.pop(obj, None) @@ -758,7 +772,9 @@ def _check_for_unknown_active_components(self): ctype, active=True, descend_into=True ): if isinstance(comp, Suffix): - warnings.warn('ModelChangeDetector does not detect changes to suffixes') + warnings.warn( + 'ModelChangeDetector does not detect changes to suffixes' + ) continue raise NotImplementedError( 'ModelChangeDetector does not know how to ' @@ -813,7 +829,9 @@ def remove_constraints(self, cons: List[ConstraintData]): for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][0].pop(con) vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) - params_to_check.update((id(p), p) for p in self._params_referenced_by_con[con]) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_con[con] + ) del self._active_constraints[con] self._named_expressions.pop(con, None) self._external_functions.pop(con, None) @@ -837,7 +855,9 @@ def remove_sos_constraints(self, cons: List[SOSConstraintData]): for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][1].pop(con) vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) - params_to_check.update((id(p), p) for p in self._params_referenced_by_con[con]) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_con[con] + ) del self._active_sos[con] del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] @@ -857,7 +877,9 @@ def remove_variables(self, variables: List[VarData]): if v_id in self._params_referenced_by_var: for p in self._params_referenced_by_var[v_id]: self._referenced_params[id(p)][3].pop(v_id) - params_to_check.update((id(p), p) for p in self._params_referenced_by_var[v_id]) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_var[v_id] + ) self._params_referenced_by_var.pop(v_id) cons_using, sos_using, obj_using = self._referenced_variables[v_id] if cons_using or sos_using or obj_using: diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 74952af2b49..315d8c6dc4a 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -41,7 +41,11 @@ _get_reduced_costs, ) from pyomo.contrib.solver.common.util import get_objective -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.contrib.observer.model_observer import ( + Observer, + ModelChangeDetector, + AutoUpdateConfig, +) logger = logging.getLogger(__name__) @@ -598,9 +602,7 @@ def __init__( implicit_domain=implicit_domain, visibility=visibility, ) - self.auto_updates: bool = self.declare( - 'auto_updates', AutoUpdateConfig() - ) + self.auto_updates: bool = self.declare('auto_updates', AutoUpdateConfig()) class GurobiPersistent(GurobiDirectQuadratic, PersistentSolverBase): @@ -695,7 +697,11 @@ def set_instance(self, pyomo_model): self._solver_model = gurobipy.Model(env=self.env()) self._observer = _GurobiObserver(self) timer.start('set_instance') - self._change_detector = ModelChangeDetector(model=self._pyomo_model, observers=[self._observer], **dict(self.config.auto_updates)) + self._change_detector = ModelChangeDetector( + model=self._pyomo_model, + observers=[self._observer], + **dict(self.config.auto_updates), + ) self._change_detector.config = self.config.auto_updates timer.stop('set_instance') @@ -834,7 +840,7 @@ def _remove_objectives(self, objs: List[ObjectiveData]): for obj in objs: if obj is not self._objective: raise RuntimeError( - 'tried to remove an objective that has not been added: ' \ + 'tried to remove an objective that has not been added: ' f'{str(obj)}' ) else: @@ -848,7 +854,7 @@ def _remove_objectives(self, objs: List[ObjectiveData]): def _add_objectives(self, objs: List[ObjectiveData]): if len(objs) > 1: raise NotImplementedError( - 'the persistent interface to gurobi currently ' \ + 'the persistent interface to gurobi currently ' f'only supports single-objective problems; got {len(objs)}: ' f'{[str(i) for i in objs]}' ) @@ -860,9 +866,9 @@ def _add_objectives(self, objs: List[ObjectiveData]): if self._objective is not None: raise NotImplementedError( - 'the persistent interface to gurobi currently ' \ - 'only supports single-objective problems; tried to add ' \ - f'an objective ({str(obj)}), but there is already an ' \ + 'the persistent interface to gurobi currently ' + 'only supports single-objective problems; tried to add ' + f'an objective ({str(obj)}), but there is already an ' f'active objective ({str(self._objective)})' ) @@ -875,9 +881,7 @@ def _add_objectives(self, objs: List[ObjectiveData]): else: raise ValueError(f'Objective sense is not recognized: {obj.sense}') - repn = generate_standard_repn( - obj.expr, quadratic=True, compute_values=False - ) + repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=False) repn_constant = value(repn.constant) gurobi_expr = self._get_expr_from_pyomo_repn(repn) From cf000a1f663cae02c4c0171179fd8de51fe5dfda Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 1 Nov 2025 08:28:10 -0600 Subject: [PATCH 20/37] directory for all gurobi interfaces --- pyomo/contrib/solver/solvers/{ => gurobi}/gurobi_direct_minlp.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pyomo/contrib/solver/solvers/{ => gurobi}/gurobi_direct_minlp.py (100%) diff --git a/pyomo/contrib/solver/solvers/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py similarity index 100% rename from pyomo/contrib/solver/solvers/gurobi_direct_minlp.py rename to pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py From 7b20095c4a39a0ca45a837a1be6aac4714ef50ad Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 1 Nov 2025 11:29:49 -0600 Subject: [PATCH 21/37] clean up gurobi interfaces --- pyomo/contrib/solver/plugins.py | 7 +- .../contrib/solver/solvers/gurobi/__init__.py | 3 +- .../solver/solvers/gurobi/gurobi_direct.py | 93 +--- .../solvers/gurobi/gurobi_direct_base.py | 135 ++++-- .../solvers/gurobi/gurobi_direct_minlp.py | 84 +--- .../solvers/gurobi/gurobi_persistent.py | 425 +++++------------- .../solver/tests/solvers/test_gurobi_minlp.py | 2 +- .../tests/solvers/test_gurobi_minlp_walker.py | 2 +- .../tests/solvers/test_gurobi_minlp_writer.py | 2 +- .../solver/tests/solvers/test_solvers.py | 5 - 10 files changed, 250 insertions(+), 508 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 99c6f4fb612..430207a736c 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -13,7 +13,7 @@ from .common.factory import SolverFactory from .solvers.ipopt import Ipopt, LegacyIpoptSolver from .solvers.gurobi.gurobi_direct import GurobiDirect -from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent +from .solvers.gurobi.gurobi_persistent import GurobiPersistent from .solvers.gurobi.gurobi_direct_minlp import GurobiDirectMINLP from .solvers.highs import Highs from .solvers.knitro.direct import KnitroDirectSolver @@ -33,11 +33,6 @@ def load(): legacy_name="gurobi_direct_v2", doc="Direct (scipy-based) interface to Gurobi", )(GurobiDirect) - SolverFactory.register( - name='gurobi_direct_quadratic', - legacy_name='gurobi_direct_quadratic_v2', - doc='Direct interface to Gurobi', - )(GurobiDirectQuadratic) SolverFactory.register( name='gurobi_direct_minlp', legacy_name='gurobi_direct_minlp', diff --git a/pyomo/contrib/solver/solvers/gurobi/__init__.py b/pyomo/contrib/solver/solvers/gurobi/__init__.py index 0ef0c8c9908..0809846ebc3 100644 --- a/pyomo/contrib/solver/solvers/gurobi/__init__.py +++ b/pyomo/contrib/solver/solvers/gurobi/__init__.py @@ -1,2 +1,3 @@ from .gurobi_direct import GurobiDirect -from .gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent +from .gurobi_persistent import GurobiPersistent +from .gurobi_direct_minlp import GurobiDirectMINLP diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 16c633c7d7c..5ee8ad54f0e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -23,88 +23,25 @@ IncompatibleModelError, ) from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase -from .gurobi_direct_base import GurobiDirectBase, gurobipy +from .gurobi_direct_base import GurobiDirectBase, gurobipy, GurobiDirectSolutionLoaderBase +import logging -class GurobiDirectSolutionLoader(SolutionLoaderBase): - def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars): - self._grb_model = grb_model - self._grb_cons = grb_cons - self._grb_vars = grb_vars - self._pyo_cons = pyo_cons - self._pyo_vars = pyo_vars - GurobiDirectBase._register_env_client() +logger = logging.getLogger(__name__) + +class GurobiDirectSolutionLoader(GurobiDirectSolutionLoaderBase): def __del__(self): + super().__del__() if python_is_shutting_down(): return # Free the associated model - if self._grb_model is not None: - self._grb_cons = None - self._grb_vars = None - self._pyo_cons = None - self._pyo_vars = None + if self._solver_model is not None: + self._var_map = None + self._con_map = None # explicitly release the model - self._grb_model.dispose() - self._grb_model = None - # Release the gurobi license if this is the last reference to - # the environment (either through a results object or solver - # interface) - GurobiDirectBase._release_env_client() - - def load_vars(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 - if self._grb_model.SolCount == 0: - raise NoSolutionError() - - iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) - for p_var, g_var in iterator: - p_var.set_value(g_var, skip_validation=True) - StaleFlagManager.mark_all_as_stale(delayed=True) - - def get_primals(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 - if self._grb_model.SolCount == 0: - raise NoSolutionError() - - iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) - return ComponentMap(iterator) - - def get_duals(self, cons_to_load=None): - if self._grb_model.Status != gurobipy.GRB.OPTIMAL: - raise NoDualsError() - - def dedup(_iter): - last = None - for con_info_dual in _iter: - if not con_info_dual[1] and con_info_dual[0][0] is last: - continue - last = con_info_dual[0][0] - yield con_info_dual - - iterator = dedup(zip(self._pyo_cons, self._grb_cons.getAttr('Pi').tolist())) - if cons_to_load: - cons_to_load = set(cons_to_load) - iterator = filter( - lambda con_info_dual: con_info_dual[0][0] in cons_to_load, iterator - ) - return {con_info[0]: dual for con_info, dual in iterator} - - def get_reduced_costs(self, vars_to_load=None): - if self._grb_model.Status != gurobipy.GRB.OPTIMAL: - raise NoReducedCostsError() - - iterator = zip(self._pyo_vars, self._grb_vars.getAttr('Rc').tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) - return ComponentMap(iterator) + self._solver_model.dispose() + self._solver_model = None class GurobiDirect(GurobiDirectBase): @@ -116,7 +53,7 @@ def __init__(self, **kwds): self._pyomo_vars = None def _pyomo_gurobi_var_iter(self): - return zip(self._pyomo_vars, self._gurobi_vars.tolist()) + return zip(self._pyomo_vars, self._gurobi_vars) def _create_solver_model(self, pyomo_model): timer = self.config.timer @@ -174,10 +111,12 @@ def _create_solver_model(self, pyomo_model): timer.stop('transfer_model') self._pyomo_vars = repn.columns - self._gurobi_vars = x + self._gurobi_vars = x.tolist() + var_map = ComponentMap(zip(repn.columns, self._gurobi_vars)) + con_map = dict(zip([i.constraint for i in repn.rows], A.tolist())) solution_loader = GurobiDirectSolutionLoader( - gurobi_model, A, x, repn.rows, repn.columns + solver_model=gurobi_model, var_map=var_map, con_map=con_map, ) has_obj = len(repn.objectives) > 0 diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 489f0b5fe71..506279ee37e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -14,6 +14,7 @@ import math import os import logging +from typing import Mapping, Optional, Sequence, Dict from pyomo.common.collections import ComponentMap from pyomo.common.config import ConfigValue @@ -24,6 +25,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer from pyomo.core.staleflag import StaleFlagManager +from pyomo.core.base import VarData, ConstraintData from pyomo.contrib.solver.common.base import SolverBase, Availability from pyomo.contrib.solver.common.config import BranchAndBoundConfig @@ -39,10 +41,8 @@ SolutionStatus, TerminationCondition, ) -import logging - +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase -logger = logging.getLogger(__name__) logger = logging.getLogger(__name__) @@ -80,8 +80,8 @@ def __init__( def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_number): """ solver_model: gurobipy.Model - var_map: Dict[int, gurobipy.Var] - Maps the id of the pyomo variable to the gurobipy variable + var_map: Mapping[VarData, gurobipy.Var] + Maps the pyomo variable to the gurobipy variable vars_to_load: List[VarData] solution_number: int """ @@ -92,11 +92,9 @@ def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_ raise ValueError('Cannot obtain suboptimal solutions for a continuous model') original_solution_number = solver_model.getParamInfo('SolutionNumber')[2] solver_model.setParam('SolutionNumber', solution_number) - gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + gurobi_vars_to_load = [var_map[v] for v in vars_to_load] vals = solver_model.getAttr("Xn", gurobi_vars_to_load) - res = ComponentMap() - for var, val in zip(vars_to_load, vals): - res[var] = val + res = ComponentMap(zip(vars_to_load, vals)) solver_model.setParam('SolutionNumber', original_solution_number) return res @@ -104,8 +102,8 @@ def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_ def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): """ solver_model: gurobipy.Model - var_map: Dict[int, gurobipy.Var] - Maps the id of the pyomo variable to the gurobipy variable + var_map: Mapping[VarData, gurobipy.Var] + Maps the pyomo variable to the gurobipy variable vars_to_load: List[VarData] solution_number: int """ @@ -122,8 +120,8 @@ def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): """ solver_model: gurobipy.Model - var_map: Dict[int, gurobipy.Var] - Maps the id of the pyomo variable to the gurobipy variable + var_map: Mapping[Vardata, gurobipy.Var] + Maps the pyomo variable to the gurobipy variable vars_to_load: List[VarData] solution_number: int """ @@ -138,59 +136,122 @@ def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): solution_number=solution_number, ) - gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + gurobi_vars_to_load = [var_map[v] for v in vars_to_load] vals = solver_model.getAttr("X", gurobi_vars_to_load) - res = ComponentMap() - for var, val in zip(vars_to_load, vals): - res[var] = val + res = ComponentMap(zip(vars_to_load, vals)) return res def _get_reduced_costs(solver_model, var_map, vars_to_load): """ solver_model: gurobipy.Model - var_map: Dict[int, gurobipy.Var] - Maps the id of the pyomo variable to the gurobipy variable + var_map: Mapping[VarData, gurobipy.Var] + Maps the pyomo variable to the gurobipy variable vars_to_load: List[VarData] """ if solver_model.Status != gurobipy.GRB.OPTIMAL: raise NoReducedCostsError() + if solver_model.IsMIP: + # this will also return True for continuous, nonconvex models + raise NoDualsError() - res = ComponentMap() - gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + gurobi_vars_to_load = [var_map[v] for v in vars_to_load] vals = solver_model.getAttr("Rc", gurobi_vars_to_load) - for var, val in zip(vars_to_load, vals): - res[var] = val - + res = ComponentMap(zip(vars_to_load, vals)) return res -def _get_duals(solver_model, con_map, linear_cons_to_load, quadratic_cons_to_load): +def _get_duals(solver_model, con_map, cons_to_load): """ solver_model: gurobipy.Model con_map: Dict[ConstraintData, gurobipy.Constr] Maps the pyomo constraint to the gurobipy constraint - linear_cons_to_load: List[ConstraintData] - quadratic_cons_to_load: List[ConstraintData] + cons_to_load: List[ConstraintData] """ if solver_model.Status != gurobipy.GRB.OPTIMAL: raise NoDualsError() - - linear_gurobi_cons = [con_map[c] for c in linear_cons_to_load] - quadratic_gurobi_cons = [con_map[c] for c in quadratic_cons_to_load] - linear_vals = solver_model.getAttr("Pi", linear_gurobi_cons) - quadratic_vals = solver_model.getAttr("QCPi", quadratic_gurobi_cons) - + if solver_model.IsMIP: + # this will also return True for continuous, nonconvex models + raise NoDualsError() + + qcons = set(solver_model.getQConstrs()) + duals = {} - for c, val in zip(linear_cons_to_load, linear_vals): - duals[c] = val - for c, val in zip(quadratic_cons_to_load, quadratic_vals): - duals[c] = val + for c in cons_to_load: + gurobi_con = con_map[c] + if gurobi_con in qcons: + duals[c] = gurobi_con.QCPi + else: + duals[c] = gurobi_con.Pi + return duals +class GurobiDirectSolutionLoaderBase(SolutionLoaderBase): + def __init__( + self, solver_model, var_map, con_map, + ) -> None: + super().__init__() + self._solver_model = solver_model + self._var_map = var_map + self._con_map = con_map + GurobiDirectBase._register_env_client() + + def __del__(self): + # Release the gurobi license if this is the last reference to + # the environment (either through a results object or solver + # interface) + GurobiDirectBase._release_env_client() + + def load_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 + ) -> None: + if vars_to_load is None: + vars_to_load = self._var_map + _load_vars( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + solution_number=solution_id, + ) + + def get_primals( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = self._var_map + return _get_primals( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + solution_number=solution_id, + ) + + def get_reduced_costs( + self, vars_to_load: Optional[Sequence[VarData]] = None + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = self._var_map + return _get_reduced_costs( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + ) + + def get_duals( + self, cons_to_load: Optional[Sequence[ConstraintData]] = None + ) -> Dict[ConstraintData, float]: + if cons_to_load is None: + cons_to_load = self._con_map + return _get_duals( + solver_model=self._solver_model, + con_map=self._con_map, + cons_to_load=cons_to_load, + ) + + class GurobiDirectBase(SolverBase): _num_gurobipy_env_clients = 0 diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py index bc7b8362aea..1fdbf27c018 100755 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py @@ -24,10 +24,8 @@ from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase from pyomo.contrib.solver.common.util import NoSolutionError -from pyomo.contrib.solver.solvers.gurobi_direct import ( - GurobiDirect, - GurobiDirectSolutionLoader, -) +from .gurobi_direct_base import GurobiDirectBase +from .gurobi_direct import GurobiDirectSolutionLoader from pyomo.core.base import ( Binary, @@ -584,27 +582,18 @@ def write(self, model, **options): doc='Direct interface to Gurobi version 12 and up ' 'supporting general nonlinear expressions', ) -class GurobiDirectMINLP(GurobiDirect): - def solve(self, model, **kwds): - """Solve the model. +class GurobiDirectMINLP(GurobiDirectBase): + _minimum_version = (12, 0, 0) - Args: - model (Block): a Pyomo model or Block to be solved - """ - start_timestamp = datetime.datetime.now(datetime.timezone.utc) - config = self.config(value=kwds, preserve_implicit=True) - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - if config.timer is None: - config.timer = HierarchicalTimer() - timer = config.timer + def __init__(self, **kwds): + super().__init__(**kwds) + self._var_map = None - StaleFlagManager.mark_all_as_stale() + def _pyomo_gurobi_var_iter(self): + return self._var_map.items() + def _create_solver_model(self, pyomo_model): + timer = self.config.timer timer.start('compile_model') writer = GurobiMINLPWriter() @@ -614,50 +603,11 @@ def solve(self, model, **kwds): timer.stop('compile_model') - ostreams = [io.StringIO()] + config.tee - - # set options - options = config.solver_options - - grb_model.setParam('LogToConsole', 1) - - if config.threads is not None: - grb_model.setParam('Threads', config.threads) - if config.time_limit is not None: - grb_model.setParam('TimeLimit', config.time_limit) - if config.rel_gap is not None: - grb_model.setParam('MIPGap', config.rel_gap) - if config.abs_gap is not None: - grb_model.setParam('MIPGapAbs', config.abs_gap) - - if config.use_mipstart: - raise MouseTrap("MIPSTART not yet supported") - - for key, option in options.items(): - grb_model.setParam(key, option) - - grbsol = grb_model.optimize() - - res = self._postsolve( - timer, - config, - GurobiDirectSolutionLoader( - grb_model, - grb_cons=grb_cons, - grb_vars=var_map.values(), - pyo_cons=pyo_cons, - pyo_vars=var_map.keys(), - pyo_obj=pyo_obj, - ), - ) + self._var_map = var_map + con_map = dict(zip(pyo_cons, grb_cons)) - res.solver_config = config - res.solver_name = 'Gurobi' - res.solver_version = self.version() - res.solver_log = ostreams[0].getvalue() + solution_loader = GurobiDirectSolutionLoader( + solver_model=grb_model, var_map=var_map, con_map=con_map, + ) - end_timestamp = datetime.datetime.now(datetime.timezone.utc) - res.timing_info.start_timestamp = start_timestamp - res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() - res.timing_info.timer = timer - return res + return grb_model, solution_loader, bool(pyo_obj) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 315d8c6dc4a..3462f50b437 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -14,7 +14,7 @@ from typing import Dict, List, Optional, Sequence, Mapping from collections.abc import Iterable -from pyomo.common.collections import ComponentSet, OrderedSet +from pyomo.common.collections import ComponentSet, OrderedSet, ComponentMap from pyomo.common.errors import PyomoException from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer @@ -28,18 +28,15 @@ from pyomo.repn import generate_standard_repn from pyomo.contrib.solver.common.results import Results from pyomo.contrib.solver.common.util import IncompatibleModelError -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase from pyomo.contrib.solver.common.base import PersistentSolverBase from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( GurobiDirectBase, gurobipy, GurobiConfig, - _load_vars, - _get_primals, - _get_duals, - _get_reduced_costs, + GurobiDirectSolutionLoaderBase, ) +from .gurobi_direct import GurobiDirectSolutionLoader from pyomo.contrib.solver.common.util import get_objective from pyomo.contrib.observer.model_observer import ( Observer, @@ -51,87 +48,12 @@ logger = logging.getLogger(__name__) -class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): +class GurobiPersistentSolutionLoader(GurobiDirectSolutionLoaderBase): def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons - ) -> None: - super().__init__() - self._solver_model = solver_model - self._vars = var_id_map - self._var_map = var_map - self._con_map = con_map - self._linear_cons = linear_cons - self._quadratic_cons = quadratic_cons - GurobiDirectBase._register_env_client() - - def __del__(self): - # Release the gurobi license if this is the last reference to - # the environment (either through a results object or solver - # interface) - GurobiDirectBase._release_env_client() - - def load_vars( - self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 - ) -> None: - if vars_to_load is None: - vars_to_load = list(self._vars.values()) - _load_vars( - solver_model=self._solver_model, - var_map=self._var_map, - vars_to_load=vars_to_load, - solution_number=solution_id, - ) - - def get_primals( - self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 - ) -> Mapping[VarData, float]: - if vars_to_load is None: - vars_to_load = list(self._vars.values()) - return _get_primals( - solver_model=self._solver_model, - var_map=self._var_map, - vars_to_load=vars_to_load, - solution_number=solution_id, - ) - - def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None - ) -> Mapping[VarData, float]: - if vars_to_load is None: - vars_to_load = list(self._vars.values()) - return _get_reduced_costs( - solver_model=self._solver_model, - var_map=self._var_map, - vars_to_load=vars_to_load, - ) - - def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None - ) -> Dict[ConstraintData, float]: - if cons_to_load is None: - cons_to_load = list(self._con_map.keys()) - linear_cons_to_load = [] - quadratic_cons_to_load = [] - for c in cons_to_load: - if c in self._linear_cons: - linear_cons_to_load.append(c) - else: - assert c in self._quadratic_cons - quadratic_cons_to_load.append(c) - return _get_duals( - solver_model=self._solver_model, - con_map=self._con_map, - linear_cons_to_load=linear_cons_to_load, - quadratic_cons_to_load=quadratic_cons_to_load, - ) - - -class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): - def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + self, solver_model, var_map, con_map, ) -> None: super().__init__( - solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + solver_model, var_map, con_map, ) self._valid = True @@ -325,225 +247,6 @@ def var2(self): return self.var_map[self.v2id] -class GurobiDirectQuadratic(GurobiDirectBase): - _minimum_version = (7, 0, 0) - - def __init__(self, **kwds): - super().__init__(**kwds) - self._solver_model = None - self._vars = {} # from id(v) to v - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._linear_cons = set() - self._quadratic_cons = set() - self._pyomo_sos_to_solver_sos_map = {} - - def _create_solver_model(self, pyomo_model): - timer = self.config.timer - timer.start('create gurobipy model') - self._clear() - self._solver_model = gurobipy.Model(env=self.env()) - timer.start('collect constraints') - cons = list( - pyomo_model.component_data_objects( - Constraint, descend_into=True, active=True - ) - ) - timer.stop('collect constraints') - timer.start('translate constraints') - self._add_constraints(cons) - timer.stop('translate constraints') - timer.start('sos') - sos = list( - pyomo_model.component_data_objects( - SOSConstraint, descend_into=True, active=True - ) - ) - self._add_sos_constraints(sos) - timer.stop('sos') - timer.start('get objective') - obj = get_objective(pyomo_model) - timer.stop('get objective') - timer.start('translate objective') - self._set_objective(obj) - timer.stop('translate objective') - has_obj = obj is not None - solution_loader = GurobiDirectQuadraticSolutionLoader( - solver_model=self._solver_model, - var_id_map=self._vars, - var_map=self._pyomo_var_to_solver_var_map, - con_map=self._pyomo_con_to_solver_con_map, - linear_cons=self._linear_cons, - quadratic_cons=self._quadratic_cons, - ) - timer.stop('create gurobipy model') - return self._solver_model, solution_loader, has_obj - - def _clear(self): - self._solver_model = None - self._vars = {} - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._linear_cons = set() - self._quadratic_cons = set() - self._pyomo_sos_to_solver_sos_map = {} - - def _pyomo_gurobi_var_iter(self): - for vid, v in self._vars.items(): - yield v, self._pyomo_var_to_solver_var_map[vid] - - def _process_domain_and_bounds(self, var): - lb, ub, step = var.domain.get_interval() - if lb is None: - lb = -gurobipy.GRB.INFINITY - if ub is None: - ub = gurobipy.GRB.INFINITY - if step == 0: - vtype = gurobipy.GRB.CONTINUOUS - elif step == 1: - if lb == 0 and ub == 1: - vtype = gurobipy.GRB.BINARY - else: - vtype = gurobipy.GRB.INTEGER - else: - raise ValueError(f'Unrecognized domain: {var.domain}') - if var.fixed: - lb = var.value - ub = lb - else: - if var._lb is not None: - lb = max(lb, value(var._lb)) - if var._ub is not None: - ub = min(ub, value(var._ub)) - return lb, ub, vtype - - def _add_variables(self, variables: List[VarData]): - vtypes = [] - lbs = [] - ubs = [] - for ndx, var in enumerate(variables): - self._vars[id(var)] = var - lb, ub, vtype = self._process_domain_and_bounds(var) - vtypes.append(vtype) - lbs.append(lb) - ubs.append(ub) - - gurobi_vars = self._solver_model.addVars( - len(variables), lb=lbs, ub=ubs, vtype=vtypes - ).values() - - for pyomo_var, gurobi_var in zip(variables, gurobi_vars): - self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var - - def _get_expr_from_pyomo_repn(self, repn): - if repn.nonlinear_expr is not None: - raise IncompatibleModelError( - f'GurobiDirectQuadratic only supports linear and quadratic expressions: {repn}.' - ) - - if len(repn.linear_vars) > 0: - missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] - self._add_variables(missing_vars) - coef_list = [value(i) for i in repn.linear_coefs] - vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] - new_expr = gurobipy.LinExpr(coef_list, vlist) - else: - new_expr = 0.0 - - if len(repn.quadratic_vars) > 0: - missing_vars = {} - for x, y in repn.quadratic_vars: - for v in [x, y]: - vid = id(v) - if vid not in self._vars: - missing_vars[vid] = v - self._add_variables(list(missing_vars.values())) - for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): - gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] - gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] - new_expr += value(coef) * gurobi_x * gurobi_y - - return new_expr - - def _add_constraints(self, cons: List[ConstraintData]): - gurobi_expr_list = [] - for con in cons: - lb, body, ub = con.to_bounded_expression(evaluate_bounds=True) - repn = generate_standard_repn(body, quadratic=True, compute_values=True) - if len(repn.quadratic_vars) > 0: - self._quadratic_cons.add(con) - else: - self._linear_cons.add(con) - gurobi_expr = self._get_expr_from_pyomo_repn(repn) - if lb is None and ub is None: - raise ValueError( - "Constraint does not have a lower " f"or an upper bound: {con} \n" - ) - elif lb is None: - gurobi_expr_list.append(gurobi_expr <= float(ub - repn.constant)) - elif ub is None: - gurobi_expr_list.append(float(lb - repn.constant) <= gurobi_expr) - elif lb == ub: - gurobi_expr_list.append(gurobi_expr == float(lb - repn.constant)) - else: - gurobi_expr_list.append( - gurobi_expr - == [float(lb - repn.constant), float(ub - repn.constant)] - ) - - gurobi_cons = self._solver_model.addConstrs( - (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))) - ).values() - self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) - - def _add_sos_constraints(self, cons: List[SOSConstraintData]): - for con in cons: - level = con.level - if level == 1: - sos_type = gurobipy.GRB.SOS_TYPE1 - elif level == 2: - sos_type = gurobipy.GRB.SOS_TYPE2 - else: - raise ValueError( - f"Solver does not support SOS level {level} constraints" - ) - - gurobi_vars = [] - weights = [] - - missing_vars = { - id(v): v for v, w in con.get_items() if id(v) not in self._vars - } - self._add_variables(list(missing_vars.values())) - - for v, w in con.get_items(): - v_id = id(v) - gurobi_vars.append(self._pyomo_var_to_solver_var_map[v_id]) - weights.append(w) - - gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) - self._pyomo_sos_to_solver_sos_map[con] = gurobipy_con - - def _set_objective(self, obj): - if obj is None: - sense = gurobipy.GRB.MINIMIZE - gurobi_expr = 0 - repn_constant = 0 - else: - if obj.sense == minimize: - sense = gurobipy.GRB.MINIMIZE - elif obj.sense == maximize: - sense = gurobipy.GRB.MAXIMIZE - else: - raise ValueError(f'Objective sense is not recognized: {obj.sense}') - - repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=True) - gurobi_expr = self._get_expr_from_pyomo_repn(repn) - repn_constant = repn.constant - - self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) - - class _GurobiObserver(Observer): def __init__(self, opt: GurobiPersistent) -> None: self.opt = opt @@ -605,12 +308,16 @@ def __init__( self.auto_updates: bool = self.declare('auto_updates', AutoUpdateConfig()) -class GurobiPersistent(GurobiDirectQuadratic, PersistentSolverBase): +class GurobiPersistent(GurobiDirectBase, PersistentSolverBase): _minimum_version = (7, 0, 0) CONFIG = GurobiPersistentConfig() def __init__(self, **kwds): super().__init__(**kwds) + self._solver_model = None + self._pyomo_var_to_solver_var_map = ComponentMap() + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} self._pyomo_model = None self._objective = None self._mutable_helpers = {} @@ -628,7 +335,10 @@ def __init__(self, **kwds): self._should_update_parameters = False def _clear(self): - super()._clear() + self._solver_model = None + self._pyomo_var_to_solver_var_map = ComponentMap() + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} self._pyomo_model = None self._objective = None self._mutable_helpers = {} @@ -649,15 +359,15 @@ def _create_solver_model(self, pyomo_model): solution_loader = GurobiPersistentSolutionLoader( solver_model=self._solver_model, - var_id_map=self._vars, var_map=self._pyomo_var_to_solver_var_map, con_map=self._pyomo_con_to_solver_con_map, - linear_cons=self._linear_cons, - quadratic_cons=self._quadratic_cons, ) has_obj = self._objective is not None return self._solver_model, solution_loader, has_obj + def _pyomo_gurobi_var_iter(self): + return self._pyomo_var_to_solver_var_map.items() + def release_license(self): self._clear() self.__class__.release_license() @@ -668,7 +378,28 @@ def solve(self, model, **kwds) -> Results: return res def _process_domain_and_bounds(self, var): - res = super()._process_domain_and_bounds(var) + lb, ub, step = var.domain.get_interval() + if lb is None: + lb = -gurobipy.GRB.INFINITY + if ub is None: + ub = gurobipy.GRB.INFINITY + if step == 0: + vtype = gurobipy.GRB.CONTINUOUS + elif step == 1: + if lb == 0 and ub == 1: + vtype = gurobipy.GRB.BINARY + else: + vtype = gurobipy.GRB.INTEGER + else: + raise ValueError(f'Unrecognized domain: {var.domain}') + if var.fixed: + lb = var.value + ub = lb + else: + if var._lb is not None: + lb = max(lb, value(var._lb)) + if var._ub is not None: + ub = min(ub, value(var._ub)) if not is_constant(var._lb): mutable_lb = _MutableLowerBound( id(var), var.lower, self._pyomo_var_to_solver_var_map @@ -679,11 +410,26 @@ def _process_domain_and_bounds(self, var): id(var), var.upper, self._pyomo_var_to_solver_var_map ) self._mutable_bounds[id(var), 'ub'] = (var, mutable_ub) - return res + return lb, ub, vtype def _add_variables(self, variables: List[VarData]): self._invalidate_last_results() - super()._add_variables(variables) + vtypes = [] + lbs = [] + ubs = [] + for ndx, var in enumerate(variables): + self._vars[id(var)] = var + lb, ub, vtype = self._process_domain_and_bounds(var) + vtypes.append(vtype) + lbs.append(lb) + ubs.append(ub) + + gurobi_vars = self._solver_model.addVars( + len(variables), lb=lbs, ub=ubs, vtype=vtypes + ).values() + + for pyomo_var, gurobi_var in zip(variables, gurobi_vars): + self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var self._vars_added_since_update.update(variables) self._needs_updated = True @@ -720,6 +466,36 @@ def update(self): self._update_parameters([]) timer.stop('update') + def _get_expr_from_pyomo_repn(self, repn): + if repn.nonlinear_expr is not None: + raise IncompatibleModelError( + f'GurobiPersistent only supports linear and quadratic expressions: {repn}.' + ) + + if len(repn.linear_vars) > 0: + missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] + self._add_variables(missing_vars) + coef_list = [value(i) for i in repn.linear_coefs] + vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] + new_expr = gurobipy.LinExpr(coef_list, vlist) + else: + new_expr = 0.0 + + if len(repn.quadratic_vars) > 0: + missing_vars = {} + for x, y in repn.quadratic_vars: + for v in [x, y]: + vid = id(v) + if vid not in self._vars: + missing_vars[vid] = v + self._add_variables(list(missing_vars.values())) + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] + gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] + new_expr += value(coef) * gurobi_x * gurobi_y + + return new_expr + def _add_constraints(self, cons: List[ConstraintData]): self._invalidate_last_results() gurobi_expr_list = [] @@ -832,7 +608,32 @@ def _add_constraints(self, cons: List[ConstraintData]): def _add_sos_constraints(self, cons: List[SOSConstraintData]): self._invalidate_last_results() - super()._add_sos_constraints(cons) + for con in cons: + level = con.level + if level == 1: + sos_type = gurobipy.GRB.SOS_TYPE1 + elif level == 2: + sos_type = gurobipy.GRB.SOS_TYPE2 + else: + raise ValueError( + f"Solver does not support SOS level {level} constraints" + ) + + gurobi_vars = [] + weights = [] + + missing_vars = { + id(v): v for v, w in con.get_items() if id(v) not in self._vars + } + self._add_variables(list(missing_vars.values())) + + for v, w in con.get_items(): + v_id = id(v) + gurobi_vars.append(self._pyomo_var_to_solver_var_map[v_id]) + weights.append(w) + + gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) + self._pyomo_sos_to_solver_sos_map[con] = gurobipy_con self._constraints_added_since_update.update(cons) self._needs_updated = True diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp.py index 30ddd7eca4b..31d90770aca 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp.py @@ -13,7 +13,7 @@ import pyomo.common.unittest as unittest from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.common.results import TerminationCondition, SolutionStatus -from pyomo.contrib.solver.solvers.gurobi_direct_minlp import GurobiDirectMINLP +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiDirectMINLP from pyomo.core.base.constraint import Constraint from pyomo.environ import ( diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py index 14eab91f09c..eeae8a7d96e 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py @@ -14,7 +14,7 @@ from pyomo.core.expr import ProductExpression, SumExpression from pyomo.common.errors import InvalidValueError import pyomo.common.unittest as unittest -from pyomo.contrib.solver.solvers.gurobi_direct_minlp import GurobiMINLPVisitor +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiMINLPVisitor from pyomo.contrib.solver.tests.solvers.gurobi_to_pyomo_expressions import ( grb_nl_to_pyo_expr, ) diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py index f86ebd975c4..ab68aae046c 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py @@ -40,7 +40,7 @@ ) from pyomo.gdp import Disjunction from pyomo.opt import WriterFactory -from pyomo.contrib.solver.solvers.gurobi_direct_minlp import ( +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import ( GurobiDirectMINLP, GurobiMINLPVisitor, ) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 92232118b54..5eaa0791e73 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -33,7 +33,6 @@ ) from pyomo.contrib.solver.solvers.gurobi import ( GurobiDirect, - GurobiDirectQuadratic, GurobiPersistent, GurobiDirectMINLP, ) @@ -55,7 +54,6 @@ all_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), - ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('gurobi_direct_minlp', GurobiDirectMINLP), ('ipopt', Ipopt), ('highs', Highs), @@ -64,7 +62,6 @@ mip_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), - ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('gurobi_direct_minlp', GurobiDirectMINLP), ('highs', Highs), ('knitro_direct', KnitroDirectSolver), @@ -77,7 +74,6 @@ qcp_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_minlp', GurobiDirectMINLP), - ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('knitro_direct', KnitroDirectSolver), ] @@ -85,7 +81,6 @@ miqcqp_solvers = [ ('gurobi_direct_minlp', GurobiDirectMINLP), ('gurobi_persistent', GurobiPersistent), - ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('knitro_direct', KnitroDirectSolver), ] nl_solvers = [('ipopt', Ipopt)] From 5073ba0eb840e2fa7b5a65e90776239fb2e6ad58 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 1 Nov 2025 22:26:23 -0600 Subject: [PATCH 22/37] update gurobi persistent to use observer --- pyomo/contrib/observer/model_observer.py | 2 +- .../solvers/gurobi/gurobi_persistent.py | 311 ++++++++++++------ .../solver/tests/solvers/test_solvers.py | 8 +- 3 files changed, 213 insertions(+), 108 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 97910eb2bf1..8fa6b597cd2 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -910,7 +910,7 @@ def _update_variables(self, variables: Optional[Collection[VarData]] = None): reason = Reason.no_change if _fixed != fixed: reason |= Reason.fixed - elif _fixed and (value != _value): + elif (_fixed or fixed) and (value != _value): reason |= Reason.value if lb is not _lb or ub is not _ub: reason |= Reason.bounds diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 3462f50b437..af4d845fbed 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -42,6 +42,7 @@ Observer, ModelChangeDetector, AutoUpdateConfig, + Reason, ) @@ -247,47 +248,6 @@ def var2(self): return self.var_map[self.v2id] -class _GurobiObserver(Observer): - def __init__(self, opt: GurobiPersistent) -> None: - self.opt = opt - - def add_variables(self, variables: List[VarData]): - self.opt._add_variables(variables) - - def add_parameters(self, params: List[ParamData]): - pass - - def add_constraints(self, cons: List[ConstraintData]): - self.opt._add_constraints(cons) - - def add_sos_constraints(self, cons: List[SOSConstraintData]): - self.opt._add_sos_constraints(cons) - - def add_objectives(self, objs: List[ObjectiveData]): - self.opt._add_objectives(objs) - - def remove_objectives(self, objs: List[ObjectiveData]): - self.opt._remove_objectives(objs) - - def remove_constraints(self, cons: List[ConstraintData]): - self.opt._remove_constraints(cons) - - def remove_sos_constraints(self, cons: List[SOSConstraintData]): - self.opt._remove_sos_constraints(cons) - - def remove_variables(self, variables: List[VarData]): - self.opt._remove_variables(variables) - - def remove_parameters(self, params: List[ParamData]): - pass - - def update_variables(self, variables: List[VarData]): - self.opt._update_variables(variables) - - def update_parameters(self, params: List[ParamData]): - self.opt._update_parameters(params) - - class GurobiPersistentConfig(GurobiConfig): def __init__( self, @@ -308,12 +268,15 @@ def __init__( self.auto_updates: bool = self.declare('auto_updates', AutoUpdateConfig()) -class GurobiPersistent(GurobiDirectBase, PersistentSolverBase): +class GurobiPersistent(GurobiDirectBase, PersistentSolverBase, Observer): _minimum_version = (7, 0, 0) CONFIG = GurobiPersistentConfig() def __init__(self, **kwds): super().__init__(**kwds) + # we actually want to only grab the license when + # set_instance is called + self._release_env_client() self._solver_model = None self._pyomo_var_to_solver_var_map = ComponentMap() self._pyomo_con_to_solver_con_map = {} @@ -329,13 +292,16 @@ def __init__(self, **kwds): self._constraints_added_since_update = OrderedSet() self._vars_added_since_update = ComponentSet() self._last_results_object: Optional[Results] = None - self._observer = None self._change_detector = None self._constraint_ndx = 0 - self._should_update_parameters = False def _clear(self): + release = False + if self._solver_model is not None: + release = True self._solver_model = None + if release: + self._release_env_client() self._pyomo_var_to_solver_var_map = ComponentMap() self._pyomo_con_to_solver_con_map = {} self._pyomo_sos_to_solver_sos_map = {} @@ -349,6 +315,7 @@ def _clear(self): self._constraints_added_since_update = OrderedSet() self._vars_added_since_update = ComponentSet() self._last_results_object = None + self._change_detector = None self._constraint_ndx = 0 def _create_solver_model(self, pyomo_model): @@ -370,7 +337,7 @@ def _pyomo_gurobi_var_iter(self): def release_license(self): self._clear() - self.__class__.release_license() + super().release_license() def solve(self, model, **kwds) -> Results: res = super().solve(model, **kwds) @@ -418,7 +385,6 @@ def _add_variables(self, variables: List[VarData]): lbs = [] ubs = [] for ndx, var in enumerate(variables): - self._vars[id(var)] = var lb, ub, vtype = self._process_domain_and_bounds(var) vtypes.append(vtype) lbs.append(lb) @@ -429,7 +395,7 @@ def _add_variables(self, variables: List[VarData]): ).values() for pyomo_var, gurobi_var in zip(variables, gurobi_vars): - self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var + self._pyomo_var_to_solver_var_map[pyomo_var] = gurobi_var self._vars_added_since_update.update(variables) self._needs_updated = True @@ -439,13 +405,13 @@ def set_instance(self, pyomo_model): else: timer = self.config.timer self._clear() + self._register_env_client() self._pyomo_model = pyomo_model self._solver_model = gurobipy.Model(env=self.env()) - self._observer = _GurobiObserver(self) timer.start('set_instance') self._change_detector = ModelChangeDetector( model=self._pyomo_model, - observers=[self._observer], + observers=[self], **dict(self.config.auto_updates), ) self._change_detector.config = self.config.auto_updates @@ -462,8 +428,6 @@ def update(self): if self._needs_updated: self._update_gurobi_model() self._change_detector.update(timer=timer) - if self._should_update_parameters: - self._update_parameters([]) timer.stop('update') def _get_expr_from_pyomo_repn(self, repn): @@ -473,25 +437,25 @@ def _get_expr_from_pyomo_repn(self, repn): ) if len(repn.linear_vars) > 0: - missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] - self._add_variables(missing_vars) + #missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] + #self._add_variables(missing_vars) coef_list = [value(i) for i in repn.linear_coefs] - vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] + vlist = [self._pyomo_var_to_solver_var_map[v] for v in repn.linear_vars] new_expr = gurobipy.LinExpr(coef_list, vlist) else: new_expr = 0.0 if len(repn.quadratic_vars) > 0: - missing_vars = {} - for x, y in repn.quadratic_vars: - for v in [x, y]: - vid = id(v) - if vid not in self._vars: - missing_vars[vid] = v - self._add_variables(list(missing_vars.values())) + # missing_vars = {} + # for x, y in repn.quadratic_vars: + # for v in [x, y]: + # vid = id(v) + # if vid not in self._vars: + # missing_vars[vid] = v + # self._add_variables(list(missing_vars.values())) for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): - gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] - gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] + gurobi_x = self._pyomo_var_to_solver_var_map[x] + gurobi_y = self._pyomo_var_to_solver_var_map[y] new_expr += value(coef) * gurobi_x * gurobi_y return new_expr @@ -502,10 +466,6 @@ def _add_constraints(self, cons: List[ConstraintData]): for ndx, con in enumerate(cons): lb, body, ub = con.to_bounded_expression(evaluate_bounds=False) repn = generate_standard_repn(body, quadratic=True, compute_values=False) - if len(repn.quadratic_vars) > 0: - self._quadratic_cons.add(con) - else: - self._linear_cons.add(con) gurobi_expr = self._get_expr_from_pyomo_repn(repn) mutable_constant = None if lb is None and ub is None: @@ -751,53 +711,164 @@ def _remove_variables(self, variables: List[VarData]): v_id = id(var) if var in self._vars_added_since_update: self._update_gurobi_model() - solver_var = self._pyomo_var_to_solver_var_map[v_id] + solver_var = self._pyomo_var_to_solver_var_map.pop(var) self._solver_model.remove(solver_var) - del self._pyomo_var_to_solver_var_map[v_id] - del self._vars[v_id] - self._mutable_bounds.pop(v_id, None) + self._mutable_bounds.pop((v_id, 'lb'), None) + self._mutable_bounds.pop((v_id, 'ub'), None) self._needs_updated = True - def _update_variables(self, variables: List[VarData]): + def _update_variables(self, variables: Mapping[VarData, Reason]): self._invalidate_last_results() - for var in variables: - var_id = id(var) - if var_id not in self._pyomo_var_to_solver_var_map: - raise ValueError( - f'The Var provided to update_var needs to be added first: {var}' - ) - self._mutable_bounds.pop((var_id, 'lb'), None) - self._mutable_bounds.pop((var_id, 'ub'), None) - gurobipy_var = self._pyomo_var_to_solver_var_map[var_id] - lb, ub, vtype = self._process_domain_and_bounds(var) - gurobipy_var.setAttr('lb', lb) - gurobipy_var.setAttr('ub', ub) - gurobipy_var.setAttr('vtype', vtype) - if var.fixed: - self._should_update_parameters = True + new_vars = [] + old_vars = [] + mod_vars = [] + for v, reason in variables.items(): + if reason & Reason.added: + new_vars.append(v) + elif reason & Reason.removed: + old_vars.append(v) + else: + mod_vars.append(v) + + if new_vars: + self._add_variables(new_vars) + if old_vars: + self._remove_variables(old_vars) + + cons_to_reprocess = OrderedSet() + cons_to_update = OrderedSet() + reprocess_obj = False + update_obj = False + + for v in mod_vars: + reason = variables[v] + if reason & (Reason.bounds | Reason.domain | Reason.fixed | Reason.value): + var_id = id(v) + self._mutable_bounds.pop((var_id, 'lb'), None) + self._mutable_bounds.pop((var_id, 'ub'), None) + gurobipy_var = self._pyomo_var_to_solver_var_map[v] + lb, ub, vtype = self._process_domain_and_bounds(v) + gurobipy_var.setAttr('lb', lb) + gurobipy_var.setAttr('ub', ub) + gurobipy_var.setAttr('vtype', vtype) + if reason & Reason.fixed: + cons_to_reprocess.update(self._change_detector.get_constraints_impacted_by_var(v)) + objs = self._change_detector.get_objectives_impacted_by_var(v) + if objs: + assert len(objs) == 1 + assert objs[0] is self._objective + reprocess_obj = True + elif (reason & Reason.value) and v.fixed: + cons_to_update.update(self._change_detector.get_constraints_impacted_by_var(v)) + objs = self._change_detector.get_objectives_impacted_by_var(v) + if objs: + assert len(objs) == 1 + assert objs[0] is self._objective + update_obj = True + + self._remove_constraints(cons_to_reprocess) + self._add_constraints(cons_to_reprocess) + cons_to_update -= cons_to_reprocess + for c in cons_to_update: + if c in self._mutable_helpers: + for i in self._mutable_helpers[c]: + i.update() + self._update_quadratic_constraint(c) + + if reprocess_obj: + self._remove_objectives([self._objective]) + self._add_objectives([self._objective]) + elif update_obj: + self._mutable_objective_update() + self._needs_updated = True - def _update_parameters(self, params: List[ParamData]): + def _update_constraints(self, cons: Mapping[ConstraintData, Reason]): self._invalidate_last_results() - for con, helpers in self._mutable_helpers.items(): - for helper in helpers: - helper.update() - for k, (v, helper) in self._mutable_bounds.items(): - helper.update() + new_cons = [] + old_cons = [] + for c, reason in cons.items(): + if reason & Reason.added: + new_cons.append(c) + elif reason & Reason.removed: + old_cons.append(c) + elif reason & Reason.expr: + old_cons.append(c) + new_cons.append(c) + + if old_cons: + self._remove_constraints(old_cons) + if new_cons: + self._add_constraints(new_cons) + self._needs_updated = True - for con, helper in self._mutable_quadratic_helpers.items(): - if con in self._constraints_added_since_update: + def _update_sos_constraints(self, cons: Mapping[SOSConstraintData, Reason]): + self._invalidate_last_results() + new_cons = [] + old_cons = [] + for c, reason in cons.items(): + if reason & Reason.added: + new_cons.append(c) + elif reason & Reason.removed: + old_cons.append(c) + elif reason & Reason.sos_items: + old_cons.append(c) + new_cons.append(c) + + if old_cons: + self._remove_sos_constraints(old_cons) + if new_cons: + self._add_sos_constraints(new_cons) + self._needs_updated = True + + def _update_objectives(self, objs: Mapping[ObjectiveData, Reason]): + self._invalidate_last_results() + new_objs = [] + old_objs = [] + new_sense = [] + for obj, reason in objs.items(): + if reason & Reason.added: + new_objs.append(obj) + elif reason & Reason.removed: + old_objs.append(obj) + elif reason & Reason.expr: + old_objs.append(obj) + new_objs.append(obj) + elif reason & Reason.sense: + new_sense.append(obj) + + if old_objs: + self._remove_objectives(old_objs) + if new_objs: + self._add_objectives(new_objs) + if new_sense: + assert len(new_sense) == 1 + obj = new_sense[0] + assert obj is self._objective + if obj.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + elif obj.sense == maximize: + sense = gurobipy.GRB.MAXIMIZE + else: + raise ValueError(f'Objective sense is not recognized: {obj.sense}') + self._solver_model.ModelSense = sense + + def _update_quadratic_constraint(self, c: ConstraintData): + if c in self._mutable_quadratic_helpers: + if c in self._constraints_added_since_update: self._update_gurobi_model() + helper = self._mutable_quadratic_helpers[c] gurobi_con = helper.gurobi_con new_gurobi_expr = helper.get_updated_expression() new_rhs = helper.get_updated_rhs() new_sense = gurobi_con.qcsense self._solver_model.remove(gurobi_con) new_con = self._solver_model.addQConstr(new_gurobi_expr, new_sense, new_rhs) - self._pyomo_con_to_solver_con_map[con] = new_con - helper.pyomo_con = con - self._constraints_added_since_update.add(con) + self._pyomo_con_to_solver_con_map[c] = new_con + assert helper.pyomo_con is c + self._constraints_added_since_update.add(c) + def _mutable_objective_update(self): if self._mutable_objective is not None: new_gurobi_expr = self._mutable_objective.get_updated_expression() if new_gurobi_expr is not None: @@ -810,7 +881,43 @@ def _update_parameters(self, params: List[ParamData]): # parts have mutable coefficients self._solver_model.setObjective(new_gurobi_expr, sense=sense) - self._should_update_parameters = False + def _update_parameters(self, params: Mapping[ParamData, Reason]): + self._invalidate_last_results() + + cons_to_update = OrderedSet() + update_obj = False + vars_to_update = ComponentSet() + for p, reason in params.items(): + if reason & Reason.added: + continue + if reason & Reason.removed: + continue + if reason & Reason.value: + cons_to_update.update(self._change_detector.get_constraints_impacted_by_param(p)) + objs = self._change_detector.get_objectives_impacted_by_param(p) + if objs: + assert len(objs) == 1 + assert objs[0] is self._objective + update_obj = True + vars_to_update.update(self._change_detector.get_variables_impacted_by_param(p)) + + for c in cons_to_update: + if c in self._mutable_helpers: + for i in self._mutable_helpers[c]: + i.update() + self._update_quadratic_constraint(c) + + if update_obj: + self._mutable_objective_update() + + for v in vars_to_update: + vid = id(v) + if (vid, 'lb') in self._mutable_bounds: + self._mutable_bounds[(vid, 'lb')][1].update() + if (vid, 'ub') in self._mutable_bounds: + self._mutable_bounds[(vid, 'ub')][1].update() + + self._needs_updated = True def _invalidate_last_results(self): if self._last_results_object is not None: @@ -1212,9 +1319,6 @@ def cbUseSolution(self): def reset(self): self._solver_model.reset() - def add_variables(self, variables): - self._change_detector.add_variables(variables) - def add_constraints(self, cons): self._change_detector.add_constraints(cons) @@ -1230,9 +1334,6 @@ def remove_constraints(self, cons): def remove_sos_constraints(self, cons): self._change_detector.remove_sos_constraints(cons) - def remove_variables(self, variables): - self._change_detector.remove_variables(variables) - def update_variables(self, variables): self._change_detector.update_variables(variables) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 5eaa0791e73..76c7cb4be87 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -30,6 +30,8 @@ NoDualsError, NoReducedCostsError, NoSolutionError, + NoFeasibleSolutionError, + NoOptimalSolutionError, ) from pyomo.contrib.solver.solvers.gurobi import ( GurobiDirect, @@ -1089,10 +1091,12 @@ def test_results_infeasible( m.obj = pyo.Objective(expr=m.y) m.c1 = pyo.Constraint(expr=m.y >= m.x) m.c2 = pyo.Constraint(expr=m.y <= m.x - 1) - with self.assertRaises(Exception): + with self.assertRaises(NoOptimalSolutionError): res = opt.solve(m) - opt.config.load_solutions = False opt.config.raise_exception_on_nonoptimal_result = False + with self.assertRaises(NoFeasibleSolutionError): + res = opt.solve(m) + opt.config.load_solutions = False res = opt.solve(m) self.assertNotEqual(res.solution_status, SolutionStatus.optimal) if isinstance(opt, Ipopt): From 5e280dc49c23cf0db3996bbb244cd01c928f94ad Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 1 Nov 2025 23:14:18 -0600 Subject: [PATCH 23/37] gurobi refactor: bugs --- pyomo/contrib/solver/common/base.py | 32 ------------------- .../solver/solvers/gurobi/gurobi_direct.py | 9 +++++- .../solvers/gurobi/gurobi_direct_base.py | 16 ++++++++-- .../solvers/gurobi/gurobi_persistent.py | 22 +++---------- .../tests/solvers/test_gurobi_persistent.py | 7 ---- pyomo/contrib/solver/tests/unit/test_base.py | 12 ------- 6 files changed, 25 insertions(+), 73 deletions(-) diff --git a/pyomo/contrib/solver/common/base.py b/pyomo/contrib/solver/common/base.py index 63c1e97ffd6..9f6171b7773 100644 --- a/pyomo/contrib/solver/common/base.py +++ b/pyomo/contrib/solver/common/base.py @@ -321,22 +321,6 @@ def set_objective(self, obj: ObjectiveData): f"Derived class {self.__class__.__name__} failed to implement required method 'set_objective'." ) - def add_variables(self, variables: List[VarData]): - """ - Add variables to the model. - """ - raise NotImplementedError( - f"Derived class {self.__class__.__name__} failed to implement required method 'add_variables'." - ) - - def add_parameters(self, params: List[ParamData]): - """ - Add parameters to the model. - """ - raise NotImplementedError( - f"Derived class {self.__class__.__name__} failed to implement required method 'add_parameters'." - ) - def add_constraints(self, cons: List[ConstraintData]): """ Add constraints to the model. @@ -353,22 +337,6 @@ def add_block(self, block: BlockData): f"Derived class {self.__class__.__name__} failed to implement required method 'add_block'." ) - def remove_variables(self, variables: List[VarData]): - """ - Remove variables from the model. - """ - raise NotImplementedError( - f"Derived class {self.__class__.__name__} failed to implement required method 'remove_variables'." - ) - - def remove_parameters(self, params: List[ParamData]): - """ - Remove parameters from the model. - """ - raise NotImplementedError( - f"Derived class {self.__class__.__name__} failed to implement required method 'remove_parameters'." - ) - def remove_constraints(self, cons: List[ConstraintData]): """ Remove constraints from the model. diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 5ee8ad54f0e..bd3388b7fc3 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -114,7 +114,14 @@ def _create_solver_model(self, pyomo_model): self._gurobi_vars = x.tolist() var_map = ComponentMap(zip(repn.columns, self._gurobi_vars)) - con_map = dict(zip([i.constraint for i in repn.rows], A.tolist())) + con_map = {} + for row, gc in zip(repn.rows, A.tolist()): + pc = row.constraint + if pc in con_map: + # range constraint + con_map[pc] = (con_map[pc], gc) + else: + con_map[pc] = gc solution_loader = GurobiDirectSolutionLoader( solver_model=gurobi_model, var_map=var_map, con_map=con_map, ) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 506279ee37e..e156dbdca01 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -181,10 +181,20 @@ def _get_duals(solver_model, con_map, cons_to_load): duals = {} for c in cons_to_load: gurobi_con = con_map[c] - if gurobi_con in qcons: - duals[c] = gurobi_con.QCPi + if type(gurobi_con) is tuple: + # only linear range constraints are supported + gc1, gc2 = gurobi_con + d1 = gc1.Pi + d2 = gc2.Pi + if abs(d1) > abs(d2): + duals[c] = d1 + else: + duals[c] = d2 else: - duals[c] = gurobi_con.Pi + if gurobi_con in qcons: + duals[c] = gurobi_con.QCPi + else: + duals[c] = gurobi_con.Pi return duals diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index af4d845fbed..4950cd18919 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -437,8 +437,6 @@ def _get_expr_from_pyomo_repn(self, repn): ) if len(repn.linear_vars) > 0: - #missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] - #self._add_variables(missing_vars) coef_list = [value(i) for i in repn.linear_coefs] vlist = [self._pyomo_var_to_solver_var_map[v] for v in repn.linear_vars] new_expr = gurobipy.LinExpr(coef_list, vlist) @@ -446,13 +444,6 @@ def _get_expr_from_pyomo_repn(self, repn): new_expr = 0.0 if len(repn.quadratic_vars) > 0: - # missing_vars = {} - # for x, y in repn.quadratic_vars: - # for v in [x, y]: - # vid = id(v) - # if vid not in self._vars: - # missing_vars[vid] = v - # self._add_variables(list(missing_vars.values())) for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): gurobi_x = self._pyomo_var_to_solver_var_map[x] gurobi_y = self._pyomo_var_to_solver_var_map[y] @@ -582,14 +573,8 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): gurobi_vars = [] weights = [] - missing_vars = { - id(v): v for v, w in con.get_items() if id(v) not in self._vars - } - self._add_variables(list(missing_vars.values())) - for v, w in con.get_items(): - v_id = id(v) - gurobi_vars.append(self._pyomo_var_to_solver_var_map[v_id]) + gurobi_vars.append(self._pyomo_var_to_solver_var_map[v]) weights.append(w) gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) @@ -776,8 +761,9 @@ def _update_variables(self, variables: Mapping[VarData, Reason]): self._update_quadratic_constraint(c) if reprocess_obj: - self._remove_objectives([self._objective]) - self._add_objectives([self._objective]) + obj = self._objective + self._remove_objectives([obj]) + self._add_objectives([obj]) elif update_obj: self._mutable_objective_update() diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py index 24b53a19f2b..98df7236f25 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py @@ -584,13 +584,6 @@ def test_basics(self): self.assertEqual(opt.get_model_attr('NumConstrs'), 1) self.assertEqual(opt.get_model_attr('NumQConstrs'), 0) - m.z = pyo.Var() - opt.add_variables([m.z]) - self.assertEqual(opt.get_model_attr('NumVars'), 3) - opt.remove_variables([m.z]) - del m.z - self.assertEqual(opt.get_model_attr('NumVars'), 2) - def test_update1(self): m = pyo.ConcreteModel() m.x = pyo.Var() diff --git a/pyomo/contrib/solver/tests/unit/test_base.py b/pyomo/contrib/solver/tests/unit/test_base.py index 217b02b9999..08245d37f5e 100644 --- a/pyomo/contrib/solver/tests/unit/test_base.py +++ b/pyomo/contrib/solver/tests/unit/test_base.py @@ -74,15 +74,11 @@ def test_class_method_list(self): '_load_vars', 'add_block', 'add_constraints', - 'add_parameters', - 'add_variables', 'api_version', 'available', 'is_persistent', 'remove_block', 'remove_constraints', - 'remove_parameters', - 'remove_variables', 'set_instance', 'set_objective', 'solve', @@ -103,18 +99,10 @@ def test_init(self): self.assertEqual(instance.api_version(), SolverAPIVersion.V2) with self.assertRaises(NotImplementedError): self.assertEqual(instance.set_instance(None), None) - with self.assertRaises(NotImplementedError): - self.assertEqual(instance.add_variables(None), None) - with self.assertRaises(NotImplementedError): - self.assertEqual(instance.add_parameters(None), None) with self.assertRaises(NotImplementedError): self.assertEqual(instance.add_constraints(None), None) with self.assertRaises(NotImplementedError): self.assertEqual(instance.add_block(None), None) - with self.assertRaises(NotImplementedError): - self.assertEqual(instance.remove_variables(None), None) - with self.assertRaises(NotImplementedError): - self.assertEqual(instance.remove_parameters(None), None) with self.assertRaises(NotImplementedError): self.assertEqual(instance.remove_constraints(None), None) with self.assertRaises(NotImplementedError): From 8bff218837bb82ad6924ac59394df243b3f11645 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 1 Nov 2025 23:17:49 -0600 Subject: [PATCH 24/37] run black --- .../solver/solvers/gurobi/gurobi_direct.py | 8 +++-- .../solvers/gurobi/gurobi_direct_base.py | 8 ++--- .../solvers/gurobi/gurobi_direct_minlp.py | 2 +- .../solvers/gurobi/gurobi_persistent.py | 30 ++++++++++--------- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index bd3388b7fc3..f17497c4901 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -23,7 +23,11 @@ IncompatibleModelError, ) from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase -from .gurobi_direct_base import GurobiDirectBase, gurobipy, GurobiDirectSolutionLoaderBase +from .gurobi_direct_base import ( + GurobiDirectBase, + gurobipy, + GurobiDirectSolutionLoaderBase, +) import logging @@ -123,7 +127,7 @@ def _create_solver_model(self, pyomo_model): else: con_map[pc] = gc solution_loader = GurobiDirectSolutionLoader( - solver_model=gurobi_model, var_map=var_map, con_map=con_map, + solver_model=gurobi_model, var_map=var_map, con_map=con_map ) has_obj = len(repn.objectives) > 0 diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index e156dbdca01..da0a109a6d4 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -175,9 +175,9 @@ def _get_duals(solver_model, con_map, cons_to_load): if solver_model.IsMIP: # this will also return True for continuous, nonconvex models raise NoDualsError() - + qcons = set(solver_model.getQConstrs()) - + duals = {} for c in cons_to_load: gurobi_con = con_map[c] @@ -200,9 +200,7 @@ def _get_duals(solver_model, con_map, cons_to_load): class GurobiDirectSolutionLoaderBase(SolutionLoaderBase): - def __init__( - self, solver_model, var_map, con_map, - ) -> None: + def __init__(self, solver_model, var_map, con_map) -> None: super().__init__() self._solver_model = solver_model self._var_map = var_map diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py index 1fdbf27c018..2544e0acd40 100755 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py @@ -607,7 +607,7 @@ def _create_solver_model(self, pyomo_model): con_map = dict(zip(pyo_cons, grb_cons)) solution_loader = GurobiDirectSolutionLoader( - solver_model=grb_model, var_map=var_map, con_map=con_map, + solver_model=grb_model, var_map=var_map, con_map=con_map ) return grb_model, solution_loader, bool(pyo_obj) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 4950cd18919..1b4b58f3c02 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -50,12 +50,8 @@ class GurobiPersistentSolutionLoader(GurobiDirectSolutionLoaderBase): - def __init__( - self, solver_model, var_map, con_map, - ) -> None: - super().__init__( - solver_model, var_map, con_map, - ) + def __init__(self, solver_model, var_map, con_map) -> None: + super().__init__(solver_model, var_map, con_map) self._valid = True def invalidate(self): @@ -410,9 +406,7 @@ def set_instance(self, pyomo_model): self._solver_model = gurobipy.Model(env=self.env()) timer.start('set_instance') self._change_detector = ModelChangeDetector( - model=self._pyomo_model, - observers=[self], - **dict(self.config.auto_updates), + model=self._pyomo_model, observers=[self], **dict(self.config.auto_updates) ) self._change_detector.config = self.config.auto_updates timer.stop('set_instance') @@ -737,14 +731,18 @@ def _update_variables(self, variables: Mapping[VarData, Reason]): gurobipy_var.setAttr('ub', ub) gurobipy_var.setAttr('vtype', vtype) if reason & Reason.fixed: - cons_to_reprocess.update(self._change_detector.get_constraints_impacted_by_var(v)) + cons_to_reprocess.update( + self._change_detector.get_constraints_impacted_by_var(v) + ) objs = self._change_detector.get_objectives_impacted_by_var(v) if objs: assert len(objs) == 1 assert objs[0] is self._objective reprocess_obj = True elif (reason & Reason.value) and v.fixed: - cons_to_update.update(self._change_detector.get_constraints_impacted_by_var(v)) + cons_to_update.update( + self._change_detector.get_constraints_impacted_by_var(v) + ) objs = self._change_detector.get_objectives_impacted_by_var(v) if objs: assert len(objs) == 1 @@ -879,14 +877,18 @@ def _update_parameters(self, params: Mapping[ParamData, Reason]): if reason & Reason.removed: continue if reason & Reason.value: - cons_to_update.update(self._change_detector.get_constraints_impacted_by_param(p)) + cons_to_update.update( + self._change_detector.get_constraints_impacted_by_param(p) + ) objs = self._change_detector.get_objectives_impacted_by_param(p) if objs: assert len(objs) == 1 assert objs[0] is self._objective update_obj = True - vars_to_update.update(self._change_detector.get_variables_impacted_by_param(p)) - + vars_to_update.update( + self._change_detector.get_variables_impacted_by_param(p) + ) + for c in cons_to_update: if c in self._mutable_helpers: for i in self._mutable_helpers[c]: From d9dc14db49a38f8766fbcd40814c56ef1963ff49 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 5 Nov 2025 09:13:46 -0700 Subject: [PATCH 25/37] typo --- pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py index 2544e0acd40..50da9977c2b 100755 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py @@ -598,7 +598,7 @@ def _create_solver_model(self, pyomo_model): writer = GurobiMINLPWriter() grb_model, var_map, pyo_obj, grb_cons, pyo_cons = writer.write( - model, symbolic_solver_labels=config.symbolic_solver_labels + pyomo_model, symbolic_solver_labels=self.config.symbolic_solver_labels ) timer.stop('compile_model') From 1c26ae321890c03672d3196fdc10514546ed771c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 5 Nov 2025 09:17:55 -0700 Subject: [PATCH 26/37] contrib.solvers: bug in gurobi refactor --- .../contrib/solver/solvers/gurobi/gurobi_direct_minlp.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py index 50da9977c2b..593ee0c1daf 100755 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py @@ -604,7 +604,13 @@ def _create_solver_model(self, pyomo_model): timer.stop('compile_model') self._var_map = var_map - con_map = dict(zip(pyo_cons, grb_cons)) + con_map = {} + for pc, gc in zip(pyo_cons, grb_cons): + if pc in con_map: + # range constraint + con_map[pc] = (con_map[pc], gc) + else: + con_map[pc] = gc solution_loader = GurobiDirectSolutionLoader( solver_model=grb_model, var_map=var_map, con_map=con_map From a4858caf86b50f5712181df8406944b14916c971 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 6 Nov 2025 10:54:43 -0700 Subject: [PATCH 27/37] contrib.solver: update tests --- pyomo/contrib/solver/solvers/ipopt.py | 1 + .../contrib/solver/tests/solvers/test_gurobi_minlp_walker.py | 4 +++- .../contrib/solver/tests/solvers/test_gurobi_minlp_writer.py | 2 ++ pyomo/contrib/solver/tests/solvers/test_solvers.py | 2 -- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 1bf1fdb7bf9..4ad9729f4a2 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -530,6 +530,7 @@ def solve(self, model, **kwds) -> Results: results.solver_version = self.version(config) if config.load_solutions: + logger.error(f'solution_status: {results.solution_status}') if results.solution_status == SolutionStatus.noSolution: raise NoFeasibleSolutionError() results.solution_loader.load_vars() diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py index eeae8a7d96e..3c33f273121 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py @@ -14,7 +14,7 @@ from pyomo.core.expr import ProductExpression, SumExpression from pyomo.common.errors import InvalidValueError import pyomo.common.unittest as unittest -from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiMINLPVisitor +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiMINLPVisitor, GurobiDirectMINLP from pyomo.contrib.solver.tests.solvers.gurobi_to_pyomo_expressions import ( grb_nl_to_pyo_expr, ) @@ -39,6 +39,8 @@ if gurobipy_available: from gurobipy import GRB + if not GurobiDirectMINLP().available(): + gurobipy_available = False class CommonTest(unittest.TestCase): diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py index ab68aae046c..c114f3e12ca 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py @@ -51,6 +51,8 @@ gurobipy, gurobipy_available = attempt_import('gurobipy', minimum_version='12.0.0') if gurobipy_available: from gurobipy import GRB + if not GurobiDirectMINLP().available(): + gurobipy_available = False def make_model(): diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 76c7cb4be87..21d80c247ce 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -1094,8 +1094,6 @@ def test_results_infeasible( with self.assertRaises(NoOptimalSolutionError): res = opt.solve(m) opt.config.raise_exception_on_nonoptimal_result = False - with self.assertRaises(NoFeasibleSolutionError): - res = opt.solve(m) opt.config.load_solutions = False res = opt.solve(m) self.assertNotEqual(res.solution_status, SolutionStatus.optimal) From 122511b56759a002c5d9b022ff5b4fe132bf7419 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 7 Nov 2025 10:20:12 -0700 Subject: [PATCH 28/37] run black --- .../solver/tests/solvers/test_gurobi_minlp_walker.py | 6 +++++- .../solver/tests/solvers/test_gurobi_minlp_writer.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py index 3c33f273121..03350ed794d 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py @@ -14,7 +14,10 @@ from pyomo.core.expr import ProductExpression, SumExpression from pyomo.common.errors import InvalidValueError import pyomo.common.unittest as unittest -from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiMINLPVisitor, GurobiDirectMINLP +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import ( + GurobiMINLPVisitor, + GurobiDirectMINLP, +) from pyomo.contrib.solver.tests.solvers.gurobi_to_pyomo_expressions import ( grb_nl_to_pyo_expr, ) @@ -39,6 +42,7 @@ if gurobipy_available: from gurobipy import GRB + if not GurobiDirectMINLP().available(): gurobipy_available = False diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py index c114f3e12ca..f2b37b88e22 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py @@ -51,6 +51,7 @@ gurobipy, gurobipy_available = attempt_import('gurobipy', minimum_version='12.0.0') if gurobipy_available: from gurobipy import GRB + if not GurobiDirectMINLP().available(): gurobipy_available = False From 71963f14569f15f8ffb562603e006c343aba4426 Mon Sep 17 00:00:00 2001 From: Emma Johnson <12833636+emma58@users.noreply.github.com> Date: Fri, 7 Nov 2025 16:31:18 -0700 Subject: [PATCH 29/37] Changing the config option name for 'use_mipstart' to be 'warmstart_discrete_vars' --- pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index da0a109a6d4..3e362cff9fa 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -66,8 +66,8 @@ def __init__( implicit_domain=implicit_domain, visibility=visibility, ) - self.use_mipstart: bool = self.declare( - 'use_mipstart', + self.warmstart_discrete_vars: bool = self.declare( + 'warmstart_discrete_vars', ConfigValue( default=False, domain=bool, @@ -411,7 +411,7 @@ def solve(self, model, **kwds) -> Results: if config.abs_gap is not None: gurobi_model.setParam('MIPGapAbs', config.abs_gap) - if config.use_mipstart: + if config.warmstart_discrete_vars: self._mipstart() for key, option in options.items(): From 87128848750bdcc6d453fcb566d44c1482c571af Mon Sep 17 00:00:00 2001 From: Emma Johnson <12833636+emma58@users.noreply.github.com> Date: Fri, 7 Nov 2025 16:31:39 -0700 Subject: [PATCH 30/37] Adding tests for Gurobi warmstarts in all the interfaces --- .../tests/solvers/test_gurobi_warm_start.py | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 pyomo/contrib/solver/tests/solvers/test_gurobi_warm_start.py diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_warm_start.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_warm_start.py new file mode 100644 index 00000000000..93ef0eab347 --- /dev/null +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_warm_start.py @@ -0,0 +1,113 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import logging +from pyomo.common.log import LoggingIntercept +import pyomo.common.unittest as unittest +from pyomo.contrib.solver.common.factory import SolverFactory + +from pyomo.environ import ( + ConcreteModel, + Var, + Constraint, + value, + Binary, + NonNegativeReals, + Objective, + Set, +) + +gurobi_direct = SolverFactory('gurobi_direct') +gurobi_direct_minlp = SolverFactory('gurobi_direct_minlp') +gurobi_persistent = SolverFactory('gurobi_persistent') + + +class TestGurobiWarmStart(unittest.TestCase): + def make_model(self): + m = ConcreteModel() + m.S = Set(initialize=[1, 2, 3, 4, 5]) + m.y = Var(m.S, domain=Binary) + m.x = Var(m.S, domain=NonNegativeReals) + m.obj = Objective(expr=sum(m.x[i] for i in m.S)) + + @m.Constraint(m.S) + def cons(m, i): + if i % 2 == 0: + return m.x[i] + i * m.y[i] >= 3 * i + else: + return m.x[i] - i * m.y[i] >= 3 * i + + # define a suboptimal MIP start + for i in m.S: + m.y[i] = 1 + # objective will be 4 + 4 + 12 + 8 + 20 = 48 + + return m + + def check_optimal_soln(self, m): + # check that we got the optimal solution: + # y[1] = 0, x[1] = 3 + # y[2] = 1, x[2] = 4 + # y[3] = 0, x[3] = 9 + # y[4] = 1, x[4] = 8 + # y[5] = 0, x[5] = 15 + x = {1: 3, 2: 4, 3: 9, 4: 8, 5: 15} + self.assertEqual(value(m.obj), 39) + for i in m.S: + if i % 2 == 0: + self.assertEqual(value(m.y[i]), 1) + else: + self.assertEqual(value(m.y[i]), 0) + self.assertEqual(value(m.x[i]), x[i]) + + @unittest.skipUnless(gurobi_direct.available(), "needs Gurobi Direct interface") + def test_gurobi_direct_warm_start(self): + m = self.make_model() + + gurobi_direct.config.warmstart_discrete_vars = True + logger = logging.getLogger('tee') + with LoggingIntercept(module='tee', level=logging.INFO) as LOG: + gurobi_direct.solve(m, tee=logger) + self.assertIn( + "User MIP start produced solution with objective 48", LOG.getvalue() + ) + self.check_optimal_soln(m) + + @unittest.skipUnless( + gurobi_direct_minlp.available(), "needs Gurobi Direct MINLP interface" + ) + def test_gurobi_minlp_warmstart(self): + m = self.make_model() + + gurobi_direct_minlp.config.warmstart_discrete_vars = True + logger = logging.getLogger('tee') + with LoggingIntercept(module='tee', level=logging.INFO) as LOG: + gurobi_direct_minlp.solve(m, tee=logger) + self.assertIn( + "User MIP start produced solution with objective 48", LOG.getvalue() + ) + self.check_optimal_soln(m) + + @unittest.skipUnless( + gurobi_persistent.available(), "needs Gurobi persistent interface" + ) + def test_gurobi_persistent_warmstart(self): + m = self.make_model() + + gurobi_persistent.config.warmstart_discrete_vars = True + gurobi_persistent.set_instance(m) + logger = logging.getLogger('tee') + with LoggingIntercept(module='tee', level=logging.INFO) as LOG: + gurobi_persistent.solve(m, tee=logger) + self.assertIn( + "User MIP start produced solution with objective 48", LOG.getvalue() + ) + self.check_optimal_soln(m) From 7069f898c7e2940caa7f6729400dbf130644fc13 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 11 Nov 2025 15:04:43 -0700 Subject: [PATCH 31/37] revert modification to ipopt interface --- pyomo/contrib/solver/solvers/ipopt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 4ad9729f4a2..1bf1fdb7bf9 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -530,7 +530,6 @@ def solve(self, model, **kwds) -> Results: results.solver_version = self.version(config) if config.load_solutions: - logger.error(f'solution_status: {results.solution_status}') if results.solution_status == SolutionStatus.noSolution: raise NoFeasibleSolutionError() results.solution_loader.load_vars() From 9bfad144fc7fdf4565577ae0a99d36bfa68af360 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 16 Nov 2025 07:12:47 -0700 Subject: [PATCH 32/37] contrib.solver.gurobi: better handling of temporary config options --- pyomo/contrib/solver/plugins.py | 2 +- .../solver/solvers/gurobi/gurobi_direct.py | 4 +- .../solvers/gurobi/gurobi_direct_base.py | 43 ++++++------------- .../solvers/gurobi/gurobi_direct_minlp.py | 6 +-- .../solvers/gurobi/gurobi_persistent.py | 25 +++++------ 5 files changed, 32 insertions(+), 48 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 430207a736c..3bc1340544f 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -39,7 +39,7 @@ def load(): doc='Direct interface to Gurobi accommodating general MINLP', )(GurobiDirectMINLP) SolverFactory.register( - name="highs", legacy_name="highs_v2", doc="Persistent interface to HiGHS" + name="highs", legacy_name="highs", doc="Persistent interface to HiGHS" )(Highs) SolverFactory.register( name="knitro_direct", diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index f17497c4901..da7442d738f 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -59,8 +59,8 @@ def __init__(self, **kwds): def _pyomo_gurobi_var_iter(self): return zip(self._pyomo_vars, self._gurobi_vars) - def _create_solver_model(self, pyomo_model): - timer = self.config.timer + def _create_solver_model(self, pyomo_model, config): + timer = config.timer timer.start('compile_model') repn = LinearStandardFormCompiler().write( diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 3e362cff9fa..587362b658b 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -348,6 +348,8 @@ def _check_license(self): model.dispose() def version(self): + if not gurobipy_available: + return None version = ( gurobipy.GRB.VERSION_MAJOR, gurobipy.GRB.VERSION_MINOR, @@ -355,7 +357,7 @@ def version(self): ) return version - def _create_solver_model(self, pyomo_model): + def _create_solver_model(self, pyomo_model, config): # should return gurobi_model, solution_loader, has_objective raise NotImplementedError('should be implemented by derived classes') @@ -370,21 +372,10 @@ def _mipstart(self): def solve(self, model, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) - orig_config = self.config orig_cwd = os.getcwd() try: config = self.config(value=kwds, preserve_implicit=True) - # hack to work around legacy solver wrapper __setattr__ - # otherwise, this would just be self.config = config - object.__setattr__(self, 'config', config) - - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) if config.timer is None: config.timer = HierarchicalTimer() timer = config.timer @@ -396,7 +387,8 @@ def solve(self, model, **kwds) -> Results: os.chdir(config.working_dir) with capture_output(TeeStream(*ostreams), capture_fd=False): gurobi_model, solution_loader, has_obj = self._create_solver_model( - model + model, + config, ) options = config.solver_options @@ -421,17 +413,12 @@ def solve(self, model, **kwds) -> Results: gurobi_model.optimize(self._callback) timer.stop('optimize') - res = self._postsolve( - grb_model=gurobi_model, solution_loader=solution_loader, has_obj=has_obj + res = self._populate_results( + grb_model=gurobi_model, solution_loader=solution_loader, has_obj=has_obj, config=config ) finally: os.chdir(orig_cwd) - # hack to work around legacy solver wrapper __setattr__ - # otherwise, this would just be self.config = orig_config - object.__setattr__(self, 'config', orig_config) - self.config = orig_config - res.solver_log = ostreams[0].getvalue() end_timestamp = datetime.datetime.now(datetime.timezone.utc) res.timing_info.start_timestamp = start_timestamp @@ -461,7 +448,7 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map - def _postsolve(self, grb_model, solution_loader, has_obj): + def _populate_results(self, grb_model, solution_loader, has_obj, config): status = grb_model.Status results = Results() @@ -483,7 +470,7 @@ def _postsolve(self, grb_model, solution_loader, has_obj): if ( results.termination_condition != TerminationCondition.convergenceCriteriaSatisfied - and self.config.raise_exception_on_nonoptimal_result + and config.raise_exception_on_nonoptimal_result ): raise NoOptimalSolutionError() @@ -510,19 +497,15 @@ def _postsolve(self, grb_model, solution_loader, has_obj): results.extra_info.BarIterCount = grb_model.getAttr('BarIterCount') results.extra_info.NodeCount = grb_model.getAttr('NodeCount') - self.config.timer.start('load solution') - if self.config.load_solutions: + config.timer.start('load solution') + if config.load_solutions: if grb_model.SolCount > 0: results.solution_loader.load_vars() else: raise NoFeasibleSolutionError() - self.config.timer.stop('load solution') + config.timer.stop('load solution') - # self.config gets copied a the beginning of - # solve and restored at the end, so modifying - # results.solver_config will not actually - # modify self.config - results.solver_config = self.config + results.solver_config = config results.solver_name = self.name results.solver_version = self.version() diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py index 593ee0c1daf..56ca484b35b 100755 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py @@ -592,13 +592,13 @@ def __init__(self, **kwds): def _pyomo_gurobi_var_iter(self): return self._var_map.items() - def _create_solver_model(self, pyomo_model): - timer = self.config.timer + def _create_solver_model(self, pyomo_model, config): + timer = config.timer timer.start('compile_model') writer = GurobiMINLPWriter() grb_model, var_map, pyo_obj, grb_cons, pyo_cons = writer.write( - pyomo_model, symbolic_solver_labels=self.config.symbolic_solver_labels + pyomo_model, symbolic_solver_labels=config.symbolic_solver_labels ) timer.stop('compile_model') diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 1b4b58f3c02..d467cfc3d53 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -314,11 +314,11 @@ def _clear(self): self._change_detector = None self._constraint_ndx = 0 - def _create_solver_model(self, pyomo_model): + def _create_solver_model(self, pyomo_model, config): if pyomo_model is self._pyomo_model: - self.update() + self.update(**config) else: - self.set_instance(pyomo_model) + self.set_instance(pyomo_model, **config) solution_loader = GurobiPersistentSolutionLoader( solver_model=self._solver_model, @@ -395,33 +395,34 @@ def _add_variables(self, variables: List[VarData]): self._vars_added_since_update.update(variables) self._needs_updated = True - def set_instance(self, pyomo_model): - if self.config.timer is None: + def set_instance(self, pyomo_model, **kwds): + config = self.config(value=kwds, preserve_implicit=True) + if config.timer is None: timer = HierarchicalTimer() else: - timer = self.config.timer + timer = config.timer self._clear() self._register_env_client() self._pyomo_model = pyomo_model self._solver_model = gurobipy.Model(env=self.env()) timer.start('set_instance') self._change_detector = ModelChangeDetector( - model=self._pyomo_model, observers=[self], **dict(self.config.auto_updates) + model=self._pyomo_model, observers=[self], **config.auto_updates ) - self._change_detector.config = self.config.auto_updates timer.stop('set_instance') - def update(self): - if self.config.timer is None: + def update(self, **kwds): + config = self.config(value=kwds, preserve_implicit=True) + if config.timer is None: timer = HierarchicalTimer() else: - timer = self.config.timer + timer = config.timer if self._pyomo_model is None: raise RuntimeError('must call set_instance or solve before update') timer.start('update') if self._needs_updated: self._update_gurobi_model() - self._change_detector.update(timer=timer) + self._change_detector.update(timer=timer, **config.auto_updates) timer.stop('update') def _get_expr_from_pyomo_repn(self, repn): From 2ab061a9ca3da1f075dc90f6859c32d1eefe374f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 16 Nov 2025 07:16:46 -0700 Subject: [PATCH 33/37] fix error --- pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 587362b658b..033d183a7e0 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -154,7 +154,7 @@ def _get_reduced_costs(solver_model, var_map, vars_to_load): raise NoReducedCostsError() if solver_model.IsMIP: # this will also return True for continuous, nonconvex models - raise NoDualsError() + raise NoReducedCostsError() gurobi_vars_to_load = [var_map[v] for v in vars_to_load] vals = solver_model.getAttr("Rc", gurobi_vars_to_load) From f3d7f3a959fa8b5d68d4cc3139931176cdaec3fb Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 18 Nov 2025 14:08:33 -0700 Subject: [PATCH 34/37] contrib.solvers.gurobi: reworking the solution loader --- .../contrib/solver/common/solution_loader.py | 4 +- .../solver/solvers/gurobi/gurobi_direct.py | 24 +- .../solvers/gurobi/gurobi_direct_base.py | 337 ++++++++++-------- .../solvers/gurobi/gurobi_direct_minlp.py | 34 +- 4 files changed, 240 insertions(+), 159 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index 911d8bee50d..c83874397d4 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -9,7 +9,7 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -from typing import Sequence, Dict, Optional, Mapping, NoReturn +from typing import Sequence, Dict, Optional, Mapping from pyomo.core.base.constraint import ConstraintData from pyomo.core.base.var import VarData @@ -23,7 +23,7 @@ class SolutionLoaderBase: Intent of this class and its children is to load the solution back into the model. """ - def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> NoReturn: + def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> None: """ Load the solution of the primal variables into the value attribute of the variables. diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index da7442d738f..74c042e6485 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -35,6 +35,21 @@ class GurobiDirectSolutionLoader(GurobiDirectSolutionLoaderBase): + def __init__(self, solver_model, pyomo_vars, gurobi_vars, con_map) -> None: + super().__init__(solver_model) + self._pyomo_vars = pyomo_vars + self._gurobi_vars = gurobi_vars + self._con_map = con_map + + def _var_pair_iter(self): + return zip(self._pyomo_vars, self._gurobi_vars) + + def _get_var_map(self): + return ComponentMap(self._var_pair_iter()) + + def _get_con_map(self): + return self._con_map + def __del__(self): super().__del__() if python_is_shutting_down(): @@ -115,9 +130,12 @@ def _create_solver_model(self, pyomo_model, config): timer.stop('transfer_model') self._pyomo_vars = repn.columns + timer.start('tolist') self._gurobi_vars = x.tolist() + timer.stop('tolist') - var_map = ComponentMap(zip(repn.columns, self._gurobi_vars)) + timer.start('create maps') + timer.start('con map') con_map = {} for row, gc in zip(repn.rows, A.tolist()): pc = row.constraint @@ -126,8 +144,10 @@ def _create_solver_model(self, pyomo_model, config): con_map[pc] = (con_map[pc], gc) else: con_map[pc] = gc + timer.stop('con map') + timer.stop('create maps') solution_loader = GurobiDirectSolutionLoader( - solver_model=gurobi_model, var_map=var_map, con_map=con_map + solver_model=gurobi_model, pyomo_vars=self._pyomo_vars, gurobi_vars=self._gurobi_vars, con_map=con_map ) has_obj = len(repn.objectives) > 0 diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 033d183a7e0..7b15582f48d 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -42,6 +42,7 @@ TerminationCondition, ) from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +import time logger = logging.getLogger(__name__) @@ -77,135 +78,24 @@ def __init__( ) -def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_number): - """ - solver_model: gurobipy.Model - var_map: Mapping[VarData, gurobipy.Var] - Maps the pyomo variable to the gurobipy variable - vars_to_load: List[VarData] - solution_number: int - """ - if ( - solver_model.getAttr('NumIntVars') == 0 - and solver_model.getAttr('NumBinVars') == 0 - ): - raise ValueError('Cannot obtain suboptimal solutions for a continuous model') - original_solution_number = solver_model.getParamInfo('SolutionNumber')[2] - solver_model.setParam('SolutionNumber', solution_number) - gurobi_vars_to_load = [var_map[v] for v in vars_to_load] - vals = solver_model.getAttr("Xn", gurobi_vars_to_load) - res = ComponentMap(zip(vars_to_load, vals)) - solver_model.setParam('SolutionNumber', original_solution_number) - return res - - -def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): - """ - solver_model: gurobipy.Model - var_map: Mapping[VarData, gurobipy.Var] - Maps the pyomo variable to the gurobipy variable - vars_to_load: List[VarData] - solution_number: int - """ - for v, val in _get_primals( - solver_model=solver_model, - var_map=var_map, - vars_to_load=vars_to_load, - solution_number=solution_number, - ).items(): - v.set_value(val, skip_validation=True) - StaleFlagManager.mark_all_as_stale(delayed=True) - - -def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): - """ - solver_model: gurobipy.Model - var_map: Mapping[Vardata, gurobipy.Var] - Maps the pyomo variable to the gurobipy variable - vars_to_load: List[VarData] - solution_number: int - """ - if solver_model.SolCount == 0: - raise NoSolutionError() - - if solution_number != 0: - return _load_suboptimal_mip_solution( - solver_model=solver_model, - var_map=var_map, - vars_to_load=vars_to_load, - solution_number=solution_number, - ) - - gurobi_vars_to_load = [var_map[v] for v in vars_to_load] - vals = solver_model.getAttr("X", gurobi_vars_to_load) - - res = ComponentMap(zip(vars_to_load, vals)) - return res - - -def _get_reduced_costs(solver_model, var_map, vars_to_load): - """ - solver_model: gurobipy.Model - var_map: Mapping[VarData, gurobipy.Var] - Maps the pyomo variable to the gurobipy variable - vars_to_load: List[VarData] - """ - if solver_model.Status != gurobipy.GRB.OPTIMAL: - raise NoReducedCostsError() - if solver_model.IsMIP: - # this will also return True for continuous, nonconvex models - raise NoReducedCostsError() - - gurobi_vars_to_load = [var_map[v] for v in vars_to_load] - vals = solver_model.getAttr("Rc", gurobi_vars_to_load) - - res = ComponentMap(zip(vars_to_load, vals)) - return res - - -def _get_duals(solver_model, con_map, cons_to_load): - """ - solver_model: gurobipy.Model - con_map: Dict[ConstraintData, gurobipy.Constr] - Maps the pyomo constraint to the gurobipy constraint - cons_to_load: List[ConstraintData] - """ - if solver_model.Status != gurobipy.GRB.OPTIMAL: - raise NoDualsError() - if solver_model.IsMIP: - # this will also return True for continuous, nonconvex models - raise NoDualsError() - - qcons = set(solver_model.getQConstrs()) - - duals = {} - for c in cons_to_load: - gurobi_con = con_map[c] - if type(gurobi_con) is tuple: - # only linear range constraints are supported - gc1, gc2 = gurobi_con - d1 = gc1.Pi - d2 = gc2.Pi - if abs(d1) > abs(d2): - duals[c] = d1 - else: - duals[c] = d2 - else: - if gurobi_con in qcons: - duals[c] = gurobi_con.QCPi - else: - duals[c] = gurobi_con.Pi - - return duals - - class GurobiDirectSolutionLoaderBase(SolutionLoaderBase): - def __init__(self, solver_model, var_map, con_map) -> None: + def __init__(self, solver_model) -> None: super().__init__() self._solver_model = solver_model - self._var_map = var_map - self._con_map = con_map GurobiDirectBase._register_env_client() + self.timer = HierarchicalTimer() + + def _var_pair_iter(self): + """ + Should iterate over pairs of (pyomo var, gurobipy var) + """ + raise NotImplementedError('should be implemented by derived classes') + + def _get_var_map(self): + raise NotImplementedError('should be implemented by derived classes') + + def _get_con_map(self): + raise NotImplementedError('should be implemented by derived classes') def __del__(self): # Release the gurobi license if this is the last reference to @@ -213,51 +103,194 @@ def __del__(self): # interface) GurobiDirectBase._release_env_client() + def _load_all_vars_solution_0(self): + self.timer.start('vars to load') + gvars = [j for i, j in self._var_pair_iter()] + self.timer.stop('vars to load') + self.timer.start('getAttr') + vals = self._solver_model.getAttr("X", gvars) + self.timer.stop('getAttr') + self.timer.start('set_value') + for (pv, _), val in zip(self._var_pair_iter(), vals): + pv.set_value(val, skip_validation=True) + self.timer.stop('set_value') + + def _load_subset_vars_solution_0(self, vars_to_load): + var_map = self._get_var_map() + self.timer.start('vars_to_load') + gvars = [var_map[i] for i in vars_to_load] + self.timer.stop('vars_to_load') + self.timer.start('getAttr') + vals = self._solver_model.getAttr("X", gvars) + self.timer.stop('getAttr') + self.timer.start('set_value') + for pv, val in zip(vars_to_load, vals): + pv.set_value(val, skip_validation=True) + self.timer.stop('set_value') + + def _load_all_vars_solution_N(self, solution_number): + assert solution_number != 0 + if ( + self._solver_model.getAttr('NumIntVars') == 0 + and self._solver_model.getAttr('NumBinVars') == 0 + ): + raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] + self._solver_model.setParam('SolutionNumber', solution_number) + gvars = [j for i, j in self._var_pair_iter()] + vals = self._solver_model.getAttr("Xn", gvars) + for (pv, _), val in zip(self._var_pair_iter(), vals): + pv.set_value(val, skip_validation=True) + self._solver_model.setParam('SolutionNumber', original_solution_number) + + def _load_subset_vars_solution_N(self, vars_to_load, solution_number): + assert solution_number != 0 + if ( + self._solver_model.getAttr('NumIntVars') == 0 + and self._solver_model.getAttr('NumBinVars') == 0 + ): + raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] + self._solver_model.setParam('SolutionNumber', solution_number) + var_map = self._get_var_map() + gvars = [var_map[i] for i in vars_to_load] + vals = self._solver_model.getAttr("Xn", gvars) + for (pv, _), val in zip(self._var_pair_iter(), vals): + pv.set_value(val, skip_validation=True) + self._solver_model.setParam('SolutionNumber', original_solution_number) + + def _get_all_vars_solution_0(self): + gvars = [j for i, j in self._var_pair_iter()] + vals = self._solver_model.getAttr("X", gvars) + return ComponentMap((i[0], val) for i, val in zip(self._var_pair_iter(), vals)) + + def _get_subset_vars_solution_0(self, vars_to_load): + var_map = self._get_var_map() + gvars = [var_map[i] for i in vars_to_load] + vals = self._solver_model.getAttr("X", gvars) + return ComponentMap(zip(vars_to_load, vals)) + + def _get_all_vars_solution_N(self, solution_number): + assert solution_number != 0 + if ( + self._solver_model.getAttr('NumIntVars') == 0 + and self._solver_model.getAttr('NumBinVars') == 0 + ): + raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] + self._solver_model.setParam('SolutionNumber', solution_number) + gvars = [j for i, j in self._var_pair_iter()] + vals = self._solver_model.getAttr("Xn", gvars) + self._solver_model.setParam('SolutionNumber', original_solution_number) + return ComponentMap((i[0], val) for i, val in zip(self._var_pair_iter(), vals)) + + def _get_subset_vars_solution_N(self, vars_to_load, solution_number): + assert solution_number != 0 + if ( + self._solver_model.getAttr('NumIntVars') == 0 + and self._solver_model.getAttr('NumBinVars') == 0 + ): + raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] + self._solver_model.setParam('SolutionNumber', solution_number) + var_map = self._get_var_map() + gvars = [var_map[i] for i in vars_to_load] + vals = self._solver_model.getAttr("Xn", gvars) + self._solver_model.setParam('SolutionNumber', original_solution_number) + return ComponentMap(zip(vars_to_load, vals)) + def load_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> None: + if self._solver_model.SolCount == 0: + raise NoSolutionError() if vars_to_load is None: - vars_to_load = self._var_map - _load_vars( - solver_model=self._solver_model, - var_map=self._var_map, - vars_to_load=vars_to_load, - solution_number=solution_id, - ) + if solution_id == 0: + self._load_all_vars_solution_0() + else: + self._load_all_vars_solution_N(solution_number=solution_id) + else: + if solution_id == 0: + self._load_subset_vars_solution_0(vars_to_load=vars_to_load) + else: + self._load_subset_vars_solution_N(vars_to_load=vars_to_load, solution_number=solution_id) + StaleFlagManager.mark_all_as_stale(delayed=True) def get_primals( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> Mapping[VarData, float]: + if self._solver_model.SolCount == 0: + raise NoSolutionError() if vars_to_load is None: - vars_to_load = self._var_map - return _get_primals( - solver_model=self._solver_model, - var_map=self._var_map, - vars_to_load=vars_to_load, - solution_number=solution_id, - ) + if solution_id == 0: + res = self._get_all_vars_solution_0() + else: + res = self._get_all_vars_solution_N(solution_number=solution_id) + else: + if solution_id == 0: + res = self._get_subset_vars_solution_0(vars_to_load=vars_to_load) + else: + res = self._get_subset_vars_solution_N(vars_to_load=vars_to_load, solution_number=solution_id) + return res + + def _get_rc_all_vars(self): + gvars = [j for i, j in self._var_pair_iter()] + vals = self._solver_model.getAttr("Rc", gvars) + return ComponentMap((i[0], val) for i, val in zip(self._var_pair_iter(), vals)) + + def _get_rc_subset_vars(self, vars_to_load): + var_map = self._get_var_map() + gvars = [var_map[i] for i in vars_to_load] + vals = self._solver_model.getAttr("Rc", gvars) + return ComponentMap(zip(vars_to_load, vals)) def get_reduced_costs( self, vars_to_load: Optional[Sequence[VarData]] = None ) -> Mapping[VarData, float]: + if self._solver_model.Status != gurobipy.GRB.OPTIMAL: + raise NoReducedCostsError() + if self._solver_model.IsMIP: + # this will also return True for continuous, nonconvex models + raise NoReducedCostsError() if vars_to_load is None: - vars_to_load = self._var_map - return _get_reduced_costs( - solver_model=self._solver_model, - var_map=self._var_map, - vars_to_load=vars_to_load, - ) + res = self._get_rc_all_vars() + else: + res = self._get_rc_subset_vars(vars_to_load=vars_to_load) + return res def get_duals( self, cons_to_load: Optional[Sequence[ConstraintData]] = None ) -> Dict[ConstraintData, float]: + if self._solver_model.Status != gurobipy.GRB.OPTIMAL: + raise NoDualsError() + if self._solver_model.IsMIP: + # this will also return True for continuous, nonconvex models + raise NoDualsError() + + qcons = set(self._solver_model.getQConstrs()) + con_map = self._get_con_map() if cons_to_load is None: - cons_to_load = self._con_map - return _get_duals( - solver_model=self._solver_model, - con_map=self._con_map, - cons_to_load=cons_to_load, - ) + cons_to_load = con_map.keys() + + duals = {} + for c in cons_to_load: + gurobi_con = con_map[c] + if type(gurobi_con) is tuple: + # only linear range constraints are supported + gc1, gc2 = gurobi_con + d1 = gc1.Pi + d2 = gc2.Pi + if abs(d1) > abs(d2): + duals[c] = d1 + else: + duals[c] = d2 + else: + if gurobi_con in qcons: + duals[c] = gurobi_con.QCPi + else: + duals[c] = gurobi_con.Pi + + return duals class GurobiDirectBase(SolverBase): @@ -500,7 +533,7 @@ def _populate_results(self, grb_model, solution_loader, has_obj, config): config.timer.start('load solution') if config.load_solutions: if grb_model.SolCount > 0: - results.solution_loader.load_vars() + results.solution_loader.load_vars(timer=config.timer) else: raise NoFeasibleSolutionError() config.timer.stop('load solution') diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py index 56ca484b35b..217b489d635 100755 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py @@ -19,13 +19,13 @@ from pyomo.common.config import ConfigDict, ConfigValue from pyomo.common.errors import InvalidValueError from pyomo.common.numeric_types import native_complex_types +from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase from pyomo.contrib.solver.common.util import NoSolutionError -from .gurobi_direct_base import GurobiDirectBase -from .gurobi_direct import GurobiDirectSolutionLoader +from .gurobi_direct_base import GurobiDirectBase, GurobiDirectSolutionLoaderBase from pyomo.core.base import ( Binary, @@ -577,6 +577,34 @@ def write(self, model, **options): return grb_model, visitor.var_map, pyo_obj, grb_cons, pyo_cons +class GurobiDirectMINLPSolutionLoader(GurobiDirectSolutionLoaderBase): + def __init__(self, solver_model, var_map, con_map) -> None: + super().__init__(solver_model) + self._var_map = var_map + self._con_map = con_map + + def _var_pair_iter(self): + return self._var_map.items() + + def _get_var_map(self): + return self._var_map + + def _get_con_map(self): + return self._con_map + + def __del__(self): + super().__del__() + if python_is_shutting_down(): + return + # Free the associated model + if self._solver_model is not None: + self._var_map = None + self._con_map = None + # explicitly release the model + self._solver_model.dispose() + self._solver_model = None + + @SolverFactory.register( 'gurobi_direct_minlp', doc='Direct interface to Gurobi version 12 and up ' @@ -612,7 +640,7 @@ def _create_solver_model(self, pyomo_model, config): else: con_map[pc] = gc - solution_loader = GurobiDirectSolutionLoader( + solution_loader = GurobiDirectMINLPSolutionLoader( solver_model=grb_model, var_map=var_map, con_map=con_map ) From d08d9932c4874fda00bf5c5a2dc44c092915e3cf Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 18 Nov 2025 14:25:06 -0700 Subject: [PATCH 35/37] contrib.solvers.gurobi: reworking the solution loader --- .../solver/solvers/gurobi/gurobi_direct_base.py | 2 +- .../solver/solvers/gurobi/gurobi_persistent.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 7b15582f48d..05848e1a9cc 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -533,7 +533,7 @@ def _populate_results(self, grb_model, solution_loader, has_obj, config): config.timer.start('load solution') if config.load_solutions: if grb_model.SolCount > 0: - results.solution_loader.load_vars(timer=config.timer) + results.solution_loader.load_vars() else: raise NoFeasibleSolutionError() config.timer.stop('load solution') diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index d467cfc3d53..a9e5f095b3b 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -51,9 +51,20 @@ class GurobiPersistentSolutionLoader(GurobiDirectSolutionLoaderBase): def __init__(self, solver_model, var_map, con_map) -> None: - super().__init__(solver_model, var_map, con_map) + super().__init__(solver_model) + self._var_map = var_map + self._con_map = con_map self._valid = True + def _var_pair_iter(self): + return self._var_map.items() + + def _get_var_map(self): + return self._var_map + + def _get_con_map(self): + return self._con_map + def invalidate(self): self._valid = False From 3ee4e995dead895f249db1033351bb1fd966062e Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 18 Nov 2025 14:25:32 -0700 Subject: [PATCH 36/37] run black --- .../solver/solvers/gurobi/gurobi_direct.py | 5 ++- .../solvers/gurobi/gurobi_direct_base.py | 36 +++++++++++++------ .../solvers/gurobi/gurobi_persistent.py | 4 +-- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 74c042e6485..f7547b582f1 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -147,7 +147,10 @@ def _create_solver_model(self, pyomo_model, config): timer.stop('con map') timer.stop('create maps') solution_loader = GurobiDirectSolutionLoader( - solver_model=gurobi_model, pyomo_vars=self._pyomo_vars, gurobi_vars=self._gurobi_vars, con_map=con_map + solver_model=gurobi_model, + pyomo_vars=self._pyomo_vars, + gurobi_vars=self._gurobi_vars, + con_map=con_map, ) has_obj = len(repn.objectives) > 0 diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 05848e1a9cc..acd881dd0c4 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -90,10 +90,10 @@ def _var_pair_iter(self): Should iterate over pairs of (pyomo var, gurobipy var) """ raise NotImplementedError('should be implemented by derived classes') - + def _get_var_map(self): raise NotImplementedError('should be implemented by derived classes') - + def _get_con_map(self): raise NotImplementedError('should be implemented by derived classes') @@ -134,7 +134,9 @@ def _load_all_vars_solution_N(self, solution_number): self._solver_model.getAttr('NumIntVars') == 0 and self._solver_model.getAttr('NumBinVars') == 0 ): - raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + raise ValueError( + 'Cannot obtain suboptimal solutions for a continuous model' + ) original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] self._solver_model.setParam('SolutionNumber', solution_number) gvars = [j for i, j in self._var_pair_iter()] @@ -149,7 +151,9 @@ def _load_subset_vars_solution_N(self, vars_to_load, solution_number): self._solver_model.getAttr('NumIntVars') == 0 and self._solver_model.getAttr('NumBinVars') == 0 ): - raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + raise ValueError( + 'Cannot obtain suboptimal solutions for a continuous model' + ) original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] self._solver_model.setParam('SolutionNumber', solution_number) var_map = self._get_var_map() @@ -176,7 +180,9 @@ def _get_all_vars_solution_N(self, solution_number): self._solver_model.getAttr('NumIntVars') == 0 and self._solver_model.getAttr('NumBinVars') == 0 ): - raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + raise ValueError( + 'Cannot obtain suboptimal solutions for a continuous model' + ) original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] self._solver_model.setParam('SolutionNumber', solution_number) gvars = [j for i, j in self._var_pair_iter()] @@ -190,7 +196,9 @@ def _get_subset_vars_solution_N(self, vars_to_load, solution_number): self._solver_model.getAttr('NumIntVars') == 0 and self._solver_model.getAttr('NumBinVars') == 0 ): - raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + raise ValueError( + 'Cannot obtain suboptimal solutions for a continuous model' + ) original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] self._solver_model.setParam('SolutionNumber', solution_number) var_map = self._get_var_map() @@ -213,7 +221,9 @@ def load_vars( if solution_id == 0: self._load_subset_vars_solution_0(vars_to_load=vars_to_load) else: - self._load_subset_vars_solution_N(vars_to_load=vars_to_load, solution_number=solution_id) + self._load_subset_vars_solution_N( + vars_to_load=vars_to_load, solution_number=solution_id + ) StaleFlagManager.mark_all_as_stale(delayed=True) def get_primals( @@ -230,7 +240,9 @@ def get_primals( if solution_id == 0: res = self._get_subset_vars_solution_0(vars_to_load=vars_to_load) else: - res = self._get_subset_vars_solution_N(vars_to_load=vars_to_load, solution_number=solution_id) + res = self._get_subset_vars_solution_N( + vars_to_load=vars_to_load, solution_number=solution_id + ) return res def _get_rc_all_vars(self): @@ -420,8 +432,7 @@ def solve(self, model, **kwds) -> Results: os.chdir(config.working_dir) with capture_output(TeeStream(*ostreams), capture_fd=False): gurobi_model, solution_loader, has_obj = self._create_solver_model( - model, - config, + model, config ) options = config.solver_options @@ -447,7 +458,10 @@ def solve(self, model, **kwds) -> Results: timer.stop('optimize') res = self._populate_results( - grb_model=gurobi_model, solution_loader=solution_loader, has_obj=has_obj, config=config + grb_model=gurobi_model, + solution_loader=solution_loader, + has_obj=has_obj, + config=config, ) finally: os.chdir(orig_cwd) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index a9e5f095b3b..76479c1c3c4 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -58,10 +58,10 @@ def __init__(self, solver_model, var_map, con_map) -> None: def _var_pair_iter(self): return self._var_map.items() - + def _get_var_map(self): return self._var_map - + def _get_con_map(self): return self._con_map From b9ca201a3d0c392fdd549951c528861f6afee9d2 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 18 Nov 2025 16:32:54 -0700 Subject: [PATCH 37/37] remove some timing statements --- .../solver/solvers/gurobi/gurobi_direct_base.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index acd881dd0c4..81c6a72c199 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -83,7 +83,6 @@ def __init__(self, solver_model) -> None: super().__init__() self._solver_model = solver_model GurobiDirectBase._register_env_client() - self.timer = HierarchicalTimer() def _var_pair_iter(self): """ @@ -104,29 +103,17 @@ def __del__(self): GurobiDirectBase._release_env_client() def _load_all_vars_solution_0(self): - self.timer.start('vars to load') gvars = [j for i, j in self._var_pair_iter()] - self.timer.stop('vars to load') - self.timer.start('getAttr') vals = self._solver_model.getAttr("X", gvars) - self.timer.stop('getAttr') - self.timer.start('set_value') for (pv, _), val in zip(self._var_pair_iter(), vals): pv.set_value(val, skip_validation=True) - self.timer.stop('set_value') def _load_subset_vars_solution_0(self, vars_to_load): var_map = self._get_var_map() - self.timer.start('vars_to_load') gvars = [var_map[i] for i in vars_to_load] - self.timer.stop('vars_to_load') - self.timer.start('getAttr') vals = self._solver_model.getAttr("X", gvars) - self.timer.stop('getAttr') - self.timer.start('set_value') for pv, val in zip(vars_to_load, vals): pv.set_value(val, skip_validation=True) - self.timer.stop('set_value') def _load_all_vars_solution_N(self, solution_number): assert solution_number != 0