diff --git a/doc/OnlineDocs/explanation/solvers/pyros.rst b/doc/OnlineDocs/explanation/solvers/pyros.rst index 9311c8999fc..298b027af81 100644 --- a/doc/OnlineDocs/explanation/solvers/pyros.rst +++ b/doc/OnlineDocs/explanation/solvers/pyros.rst @@ -906,13 +906,13 @@ Observe that the log contains the following information: information, (UTC) time at which the solver was invoked, and, if available, information on the local Git branch and commit hash. -* **Summary of solver options** (lines 19--38). -* **Preprocessing information** (lines 39--41). +* **Summary of solver options** (lines 19--40). +* **Preprocessing information** (lines 41--43). Wall time required for preprocessing the deterministic model and associated components, i.e., standardizing model components and adding the decision rule variables and equations. -* **Model component statistics** (lines 42--58). +* **Model component statistics** (lines 44--61). Breakdown of model component statistics. Includes components added by PyROS, such as the decision rule variables and equations. @@ -927,7 +927,7 @@ Observe that the log contains the following information: The number of truly uncertain parameters detected during preprocessing is also noted in parentheses (in which "eff." is an abbreviation for "effective"). -* **Iteration log table** (lines 59--69). +* **Iteration log table** (lines 62--69). Summary information on the problem iterates and subproblem outcomes. The constituent columns are defined in detail in :ref:`the table following the snippet `. @@ -953,7 +953,7 @@ Observe that the log contains the following information: * **Termination statistics** (lines 89--94). Summary of statistics related to the iterate at which PyROS terminates. -* **Exit message** (lines 95--96). +* **Exit message** (lines 95--97). .. _solver-log-snippet: @@ -963,10 +963,10 @@ Observe that the log contains the following information: :linenos: ============================================================================== - PyROS: The Pyomo Robust Optimization Solver, v1.3.8. + PyROS: The Pyomo Robust Optimization Solver, v1.3.9. Pyomo version: 6.9.3dev0 Commit hash: unknown - Invoked at UTC 2025-05-05T00:00:00.000000+00:00 + Invoked at UTC 2025-07-21T00:00:00.000000+00:00 Developed by: Natalie M. Isenberg (1), Jason A. F. Sherman (1), John D. Siirola (2), Chrysanthos E. Gounaris (1) @@ -977,7 +977,7 @@ Observe that the log contains the following information: of Energy's Institute for the Design of Advanced Energy Systems (IDAES). ============================================================================== ================================= DISCLAIMER ================================= - PyROS is still under development. + PyROS is still under development. Please provide feedback and/or report any issues by creating a ticket at https://github.com/Pyomo/pyomo/issues/new/choose ============================================================================== @@ -998,6 +998,7 @@ Observe that the log contains the following information: backup_local_solvers=[] backup_global_solvers=[] subproblem_file_directory=None + subproblem_format_options={'bar': {'symbolic_solver_labels': True}} bypass_local_separation=False bypass_global_separation=False p_robustness={} @@ -1025,33 +1026,34 @@ Observe that the log contains the following information: ------------------------------------------------------------------------------ Itn Objective 1-Stg Shift 2-Stg Shift #CViol Max Viol Wall Time (s) ------------------------------------------------------------------------------ - 0 3.5838e+07 - - 5 1.8832e+04 0.759 - 1 3.5838e+07 2.9329e-09 5.0030e-10 5 2.1295e+04 1.573 - 2 3.6285e+07 7.6526e-01 2.0398e-01 2 2.2457e+02 2.272 - 3 3.6285e+07 7.7212e-13 1.2525e-10 0 7.2940e-08g 5.280 + 0 3.5838e+07 - - 5 1.8832e+04 0.611 + 1 3.5838e+07 1.2289e-09 1.5886e-12 5 2.8919e+02 1.702 + 2 3.6269e+07 3.1647e-01 1.0432e-01 4 2.9020e+02 3.407 + 3 3.6285e+07 7.6526e-01 1.4596e-04 7 7.5966e+03 5.919 + 4 3.6285e+07 1.1608e-11 2.2270e-01 0 1.5084e-12g 8.823 ------------------------------------------------------------------------------ Robust optimal solution identified. ------------------------------------------------------------------------------ Timing breakdown: - + Identifier ncalls cumtime percall % ----------------------------------------------------------- - main 1 5.281 5.281 100.0 + main 1 8.824 8.824 100.0 ------------------------------------------------------ - dr_polishing 3 0.155 0.052 2.9 - global_separation 27 1.280 0.047 24.2 - local_separation 108 2.200 0.020 41.7 - master 4 0.727 0.182 13.8 - master_feasibility 3 0.103 0.034 1.9 - preprocessing 1 0.021 0.021 0.4 - other n/a 0.794 n/a 15.0 + dr_polishing 4 0.547 0.137 6.2 + global_separation 27 0.978 0.036 11.1 + local_separation 135 4.645 0.034 52.6 + master 5 1.720 0.344 19.5 + master_feasibility 4 0.239 0.060 2.7 + preprocessing 1 0.013 0.013 0.2 + other n/a 0.681 n/a 7.7 ====================================================== =========================================================== - + ------------------------------------------------------------------------------ Termination stats: - Iterations : 4 - Solve time (wall s) : 5.281 + Iterations : 5 + Solve time (wall s) : 8.824 Final objective value : 3.6285e+07 Termination condition : pyrosTerminationCondition.robust_optimal ------------------------------------------------------------------------------ diff --git a/pyomo/contrib/pyros/CHANGELOG.txt b/pyomo/contrib/pyros/CHANGELOG.txt index 302d188c9e2..206361022d3 100644 --- a/pyomo/contrib/pyros/CHANGELOG.txt +++ b/pyomo/contrib/pyros/CHANGELOG.txt @@ -3,6 +3,15 @@ PyROS CHANGELOG =============== +------------------------------------------------------------------------------- +PyROS 1.3.9 19 Jul 2025 +------------------------------------------------------------------------------- +- Update uncertainty set validation methods with efficient, set-specific checks +- Adjust PyROS handling of separation objective evaluation errors +- Allow user to configure formats to which PyROS should export subproblems + not solved to an acceptable level + + ------------------------------------------------------------------------------- PyROS 1.3.8 28 Apr 2025 ------------------------------------------------------------------------------- diff --git a/pyomo/contrib/pyros/config.py b/pyomo/contrib/pyros/config.py index fde4f35375b..a7a6ae9391f 100644 --- a/pyomo/contrib/pyros/config.py +++ b/pyomo/contrib/pyros/config.py @@ -855,6 +855,26 @@ def pyros_config(): ), ), ) + CONFIG.declare( + "subproblem_format_options", + ConfigValue( + default={"bar": {"symbolic_solver_labels": True}}, + # note: we leave all validation of the dict entries + # to ``BlockData.write()`` + domain=dict, + description=( + """ + File format options for writing/exporting subproblems + that were not solved to an acceptable level + if ``keepfiles=True`` is specified. + Each entry of the dict should map a Pyomo WriterFactory + format (e.g., 'bar' for BARON, 'gams' for GAMS) + to a value for the argument ``io_options`` + to the method ``BlockData.write()``. + """ + ), + ), + ) # ================================================ # === Advanced Options diff --git a/pyomo/contrib/pyros/master_problem_methods.py b/pyomo/contrib/pyros/master_problem_methods.py index 50ebe78ec2b..93b9160686a 100644 --- a/pyomo/contrib/pyros/master_problem_methods.py +++ b/pyomo/contrib/pyros/master_problem_methods.py @@ -13,8 +13,6 @@ Functions for construction and solution of the PyROS master problem. """ -import os - from pyomo.common.collections import ComponentMap, ComponentSet from pyomo.common.modeling import unique_component_name from pyomo.core import TransformationFactory @@ -37,6 +35,7 @@ ObjectiveType, pyrosTerminationCondition, TIC_TOC_SOLVE_TIME_ATTR, + write_subproblem, ) @@ -828,31 +827,8 @@ def solver_call_master(master_data): # all solvers have failed to return an acceptable status. # we will terminate PyROS with subsolver error status. - # at this point, export subproblem to file, if desired. - # NOTE: subproblem is written with variables set to their - # initial values (not the final subsolver iterate) - save_dir = config.subproblem_file_directory - serialization_msg = "" - if save_dir and config.keepfiles: - output_problem_path = os.path.join( - save_dir, - ( - config.uncertainty_set.type - + "_" - + master_data.original_model_name - + "_master_" - + str(master_data.iteration) - + ".bar" - ), - ) - master_model.write( - output_problem_path, io_options={'symbolic_solver_labels': True} - ) - serialization_msg = ( - " For debugging, problem has been serialized to the file " - f"{output_problem_path!r}." - ) + # log subproblem solve failure warning deterministic_model_qual = ( " (i.e., the deterministic model)" if master_data.iteration == 0 else "" ) @@ -867,7 +843,6 @@ def solver_call_master(master_data): if master_data.iteration == 0 else "" ) - master_soln.pyros_termination_condition = pyrosTerminationCondition.subsolver_error subsolver_termination_conditions = [ res.solver.termination_condition for res in master_soln.master_results_list @@ -879,9 +854,22 @@ def solver_call_master(master_data): f"(Termination statuses: " f"{[term_cond for term_cond in subsolver_termination_conditions]}.)" f"{deterministic_msg}" - f"{serialization_msg}" ) + # at this point, export subproblem to file, if desired. + # NOTE: subproblem is written with variables set to their + # initial values (not the final subsolver iterate) + if config.keepfiles and config.subproblem_file_directory is not None: + write_subproblem( + model=master_model, + fname=( + f"{config.uncertainty_set.type}" + f"_{master_data.original_model_name}" + f"_master_{master_data.iteration}" + ), + config=config, + ) + return master_soln diff --git a/pyomo/contrib/pyros/pyros.py b/pyomo/contrib/pyros/pyros.py index 4da7b6672cd..0205d54a00e 100644 --- a/pyomo/contrib/pyros/pyros.py +++ b/pyomo/contrib/pyros/pyros.py @@ -33,7 +33,7 @@ ) -__version__ = "1.3.8" +__version__ = "1.3.9" default_pyros_solver_logger = setup_pyros_logger() diff --git a/pyomo/contrib/pyros/separation_problem_methods.py b/pyomo/contrib/pyros/separation_problem_methods.py index d05bbfa0f37..09c7e67b9cd 100644 --- a/pyomo/contrib/pyros/separation_problem_methods.py +++ b/pyomo/contrib/pyros/separation_problem_methods.py @@ -15,7 +15,6 @@ """ from itertools import product -import os from pyomo.common.collections import ComponentSet, ComponentMap from pyomo.common.dependencies import numpy as np @@ -39,6 +38,7 @@ call_solver, check_time_limit_reached, get_all_first_stage_eq_cons, + write_subproblem, ) @@ -1093,30 +1093,6 @@ def solver_call_separation( # termination condition. PyROS will terminate with subsolver # error. At this point, export model if desired solve_call_results.subsolver_error = True - save_dir = config.subproblem_file_directory - serialization_msg = "" - if save_dir and config.keepfiles: - objective = separation_obj.name - output_problem_path = os.path.join( - save_dir, - ( - config.uncertainty_set.type - + "_" - + separation_model.name - + "_separation_" - + str(separation_data.iteration) - + "_obj_" - + objective - + ".bar" - ), - ) - separation_model.write( - output_problem_path, io_options={'symbolic_solver_labels': True} - ) - serialization_msg = ( - " For debugging, problem has been serialized to the file " - f"{output_problem_path!r}." - ) solve_call_results.message = ( "Could not successfully solve separation problem of iteration " f"{separation_data.iteration} " @@ -1124,10 +1100,20 @@ def solver_call_separation( f"provided subordinate {solve_mode} optimizers. " f"(Termination statuses: " f"{[str(term_cond) for term_cond in solver_status_dict.values()]}.)" - f"{serialization_msg}" ) config.progress_logger.warning(solve_call_results.message) + if config.keepfiles and config.subproblem_file_directory is not None: + write_subproblem( + model=separation_model, + fname=( + f"{config.uncertainty_set.type}_{separation_model.name}" + f"_separation_{separation_data.iteration}" + f"_obj_{separation_obj.name}" + ), + config=config, + ) + separation_obj.deactivate() return solve_call_results diff --git a/pyomo/contrib/pyros/tests/test_config.py b/pyomo/contrib/pyros/tests/test_config.py index 9adb10432b5..db4c4c9eaf6 100644 --- a/pyomo/contrib/pyros/tests/test_config.py +++ b/pyomo/contrib/pyros/tests/test_config.py @@ -696,6 +696,38 @@ def test_config_objective_focus(self): with self.assertRaisesRegex(ValueError, exc_str): config.objective_focus = invalid_focus + def test_config_subproblem_formats(self): + config = self.CONFIG() + + # test default + self.assertEqual( + config.subproblem_format_options, + {"bar": {"symbolic_solver_labels": True}}, + msg=( + "Default value for PyROS config option " + "subproblem_format_options' not as expected." + ), + ) + + config.subproblem_format_options = {} + self.assertEqual(config.subproblem_format_options, {}) + + nondefault_test_val = {"fmt1": {"symbolic_solver_labels": False}, "fmt2": {}} + config.subproblem_format_options = nondefault_test_val + self.assertEqual(config.subproblem_format_options, nondefault_test_val) + + # anything castable to dict should also be acceptable + config.subproblem_format_options = list(nondefault_test_val.items()) + self.assertEqual(config.subproblem_format_options, nondefault_test_val) + + exc_str = ( + # contents of the error message are version dependent + "(cannot convert dictionary update sequence" + "|'int' object is not iterable)" + ) + with self.assertRaisesRegex(ValueError, exc_str): + config.subproblem_format_options = [1, 2, 3] + class TestPositiveIntOrMinusOne(unittest.TestCase): """ diff --git a/pyomo/contrib/pyros/tests/test_grcs.py b/pyomo/contrib/pyros/tests/test_grcs.py index 79a8816ad45..fd647e00f50 100644 --- a/pyomo/contrib/pyros/tests/test_grcs.py +++ b/pyomo/contrib/pyros/tests/test_grcs.py @@ -15,6 +15,7 @@ import logging import math +import os import time import pyomo.common.unittest as unittest @@ -32,6 +33,7 @@ scipy_available, ) from pyomo.common.errors import ApplicationError, InfeasibleConstraintException +from pyomo.common.tempfiles import TempfileManager from pyomo.core.expr import replace_expressions from pyomo.environ import assert_optimal_termination, maximize as pyo_max, units as u from pyomo.opt import ( @@ -3417,6 +3419,7 @@ def test_log_config(self): " backup_local_solvers=[]\n" " backup_global_solvers=[]\n" " subproblem_file_directory=None\n" + " subproblem_format_options={'bar': {'symbolic_solver_labels': True}}\n" " bypass_local_separation=False\n" " bypass_global_separation=False\n" " p_robustness={}\n" + "-" * 78 + "\n" @@ -3703,6 +3706,93 @@ def solve(self, model, **kwds): return res +class TestPyROSSubproblemWriter(unittest.TestCase): + """ + Test PyROS subproblem writers behave as expected when + solution of a subproblem fails. + """ + + @unittest.skipUnless(baron_available, "BARON not available.") + def test_pyros_write_master_problem(self): + m = build_leyffer() + + with TempfileManager.new_context() as TMP: + tmpdir = TMP.create_tempdir() + res = SolverFactory("pyros").solve( + model=m, + first_stage_variables=[m.x1, m.x2], + second_stage_variables=[], + uncertain_params=[m.u], + uncertainty_set=BoxSet([[1, 2]]), + local_solver=SimpleTestSolver(), + global_solver=SolverFactory("baron"), + solve_master_globally=False, + keepfiles=True, + subproblem_file_directory=tmpdir, + subproblem_format_options={ + "bar": {}, + "gams": {"symbolic_solver_labels": True}, + }, + ) + expected_subproblem_file = os.path.join(tmpdir, "box_unknown_master_0") + format_files_exist_dict = { + "bar": os.path.exists(f"{expected_subproblem_file}.bar"), + "gams": os.path.exists(f"{expected_subproblem_file}.gams"), + } + + self.assertTrue(format_files_exist_dict["bar"]) + self.assertTrue(format_files_exist_dict["gams"]) + self.assertEqual(res.iterations, 1) + self.assertEqual( + res.pyros_termination_condition, pyrosTerminationCondition.subsolver_error + ) + + @unittest.skipUnless(baron_available, "BARON not available.") + def test_pyros_write_separation_problem(self): + m = build_leyffer() + subproblem_format_options = { + "bar": {}, + "gams": {"symbolic_solver_labels": True}, + } + + with TempfileManager.new_context() as TMP: + tmpdir = TMP.create_tempdir() + expected_subproblem_filenames = [ + os.path.join( + tmpdir, f"box_unknown_separation_0_obj_separation_obj_0.{fmt}" + ) + for fmt in subproblem_format_options.keys() + ] + + res = SolverFactory("pyros").solve( + model=m, + first_stage_variables=[m.x1, m.x2], + second_stage_variables=[], + uncertain_params=[m.u], + uncertainty_set=BoxSet([[1, 2]]), + local_solver=SimpleTestSolver(), + global_solver=SolverFactory("baron"), + solve_master_globally=True, + bypass_global_separation=True, + keepfiles=True, + subproblem_file_directory=tmpdir, + subproblem_format_options=subproblem_format_options, + ) + + subproblem_files_created = { + fname: os.path.exists(fname) for fname in expected_subproblem_filenames + } + + for fname, file_created in subproblem_files_created.items(): + self.assertTrue( + file_created, msg=f"Subproblem was not written to file {fname}." + ) + self.assertEqual(res.iterations, 1) + self.assertEqual( + res.pyros_termination_condition, pyrosTerminationCondition.subsolver_error + ) + + class TestPyROSSolverAdvancedValidation(unittest.TestCase): """ Test PyROS solver validation routines result in diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index d65cac37b75..25277de29d1 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -21,6 +21,7 @@ import itertools as it import logging import math +import os import timeit from pyomo.common.collections import ComponentMap, ComponentSet @@ -3179,6 +3180,31 @@ def load_final_solution(model_data, master_soln, original_user_var_partitioning) orig_var.set_value(master_blk_var.value, skip_validation=True) +def write_subproblem(model, fname, config): + """ + Write/export a subproblem to one or more files. + + Parameters + ---------- + model : ConcreteModel + Subproblem to be written/exported. + fname : str + Base name of the file(s) to be written. + Should not include any prefix directories. + config : ConfigDict + PyROS solver options. + A file will be written for each format provided in + ``config.subproblem_format_options``, + and in the directory ``config.subproblem_file_directory``. + """ + for fmt, io_options in config.subproblem_format_options.items(): + full_filename = os.path.join(config.subproblem_file_directory, f"{fname}.{fmt}") + model.write(filename=full_filename, format=fmt, io_options=io_options) + config.progress_logger.warning( + f"For debugging, subproblem has been written to the file {full_filename!r}." + ) + + def call_solver(model, solver, config, timing_obj, timer_name, err_msg): """ Solve a model with a given optimizer, keeping track of