User-defined Output Folder

docs/_toc.yml

@@ -4,9 +4,10 @@
 format: jb-book
 root: index
 parts:
-  - caption: Installation
+  - caption: Installation and Contribution
     chapters:
     - file: install
+    - file: contribute
   - caption: Running Hercules
     chapters:
     - file: running_hercules

docs/examples/07_open_cycle_gas_turbine.md

@@ -48,15 +48,22 @@ No manual setup is required. The example uses only the OCGT component which requ
 To run the example, execute the following command in the terminal:
 
 ```bash
+python examples/07_open_cycle_gas_turbine/hercules_runscript.py
+
+# OR
+
+cd examples/07_open_cycle_gas_turbine
 python hercules_runscript.py
 ```
 
 ## Outputs
 
+The output files `hercules_output.h5` and `hercules_dict.echo` are written to the folder `examples/07_open_cycle_gas_turbine/outputs_07/` and log files are written to the folder `examples/07_open_cycle_gas_turbine/logger_outputs_07/`
+
 To plot the outputs, run:
 
 ```bash
-python plot_outputs.py
+python examples/07_open_cycle_gas_turbine/plot_outputs.py
 ```
 
 The plot shows:

docs/h_dict.md

@@ -16,14 +16,25 @@ The `h_dict` is a Python dictionary that contains all the configurations for eac
 | `endtime` | float | Simulation end time in seconds | - |
 | `step` | int | Current simulation step | 0 |
 | `time` | float | Current simulation time | starttime |
+| **Output File Configuration** |
+| `output_dir` | str | Output folder name | "outputs" |
+| `output_file` | str | Output HDF5 file name | "hercules_output.h5" |
+| `overwrite_outputs` | bool | If True, removes the existing output files in the `output_dir` | True |
+| **Logging Configuration** |
+| `logging` | dict | Logging configuration | - |
+| `logging.logger_name` | str | Name of logger | "hercules" |
+| `logging.logger_file` | dict | Log file name | "log_hercules.log" |
+| `logging.console_output` | bool | Whether to log to console | True |
+| `logging.console_prefix` | str | Logger prefix | "HERCULES" |
+| `logging.log_level` | str | Logging level | "INFO" |
+| `logging.logging_dir` | str | Logger output dir | "outputs" |
 | **Plant Configuration** |
 | `plant` | dict | Plant-level configuration | - |
 | `plant.interconnect_limit` | float | Maximum power limit in kW | - |
 | **Optional Global Parameters** |
 | `verbose` | bool | Enable verbose logging | False |
 | `name` | str | Simulation name | - |
 | `description` | str | Simulation description | - |
-| `output_file` | str | Output HDF5 file path | "outputs/hercules_output.h5" |
 | `log_every_n` | int | Log every N simulation steps to output log (default: 1) | 1 |
 | `external_data` | dict | External data configuration | - |
 | `external_data_file` | str | External data file path (deprecated, use `external_data` instead) | - |

examples/07_open_cycle_gas_turbine/hercules_input.yaml

@@ -14,6 +14,12 @@ endtime_utc: "2020-01-01T06:00:00Z"  # 6 hours later
 verbose: False
 log_every_n: 1
 
+output_dir: "outputs_07"
+overwrite_outputs: True
+logging:
+  logging_dir: "logger_outputs_07"
+  log_file: "log_hercules_ex_07.log"
+
 plant:
   interconnect_limit: 100000  # kW (100 MW)
 

examples/07_open_cycle_gas_turbine/hercules_runscript.py

@@ -15,13 +15,18 @@
 - Simulation runs for 6 hours total with 1 minute time steps
 """
 
+import os
+from pathlib import Path
+
 from hercules.hercules_model import HerculesModel
-from hercules.utilities_examples import prepare_output_directory
+from hercules.utilities import load_yaml
+
+os.chdir(os.path.dirname(__file__))
 
-prepare_output_directory()
+hercules_dict = load_yaml(Path(__file__).parent / "hercules_input.yaml")
 
 # Initialize the Hercules model
-hmodel = HerculesModel("hercules_input.yaml")
+hmodel = HerculesModel(hercules_dict)
 
 
 class ControllerOCGT:

examples/07_open_cycle_gas_turbine/plot_outputs.py

@@ -4,7 +4,7 @@
 from hercules import HerculesOutput
 
 # Read the Hercules output file using HerculesOutput
-ho = HerculesOutput("outputs/hercules_output.h5")
+ho = HerculesOutput("outputs_07/hercules_output.h5")
 
 # Print metadata information
 print("Simulation Metadata:")

examples/hercules_input_example.yaml

@@ -14,10 +14,19 @@ verbose: False  # Enable verbose console output (True/False)
 log_every_n: 10  # Log output every N time steps (positive integer, default: 1)
 
 # Output file configuration
+output_dir: outputs # output folder
 output_file: outputs/hercules_output.h5  # Output HDF5 file path (automatically adds .h5 extension if missing)
+overwrite_outputs: True # if True, remove any files in output_dir prior to running, if False, creates a unique foldername
 output_use_compression: True  # Enable HDF5 compression (True/False, default: True)
 output_buffer_size: 50000  # Memory buffer size for writing data in rows (default: 50000)
 
+logging:
+  logger_name: "hercules"
+  log_file: "log_hercules.log"
+  console_output: True
+  console_prefix: "HERCULES"
+  log_level: "INFO"
+  logging_dir: "outputs" # folder to write the loggers to
 # Plant-level configuration
 plant:
   interconnect_limit: 201300  # kW - grid interconnection capacity limit

hercules/__init__.py

@@ -2,5 +2,10 @@
 
 __version__ = version("nlr-hercules")
 
+from pathlib import Path
+
 from .hercules_model import HerculesModel
 from .hercules_output import HerculesOutput
+
+HERCULES_ROOT_DIR = Path(__file__).resolve().parent
+HERCULES_EXAMPLE_DIR = HERCULES_ROOT_DIR.parent / "examples"

hercules/hercules_model.py

@@ -1,6 +1,7 @@
 import datetime as dt
 import json
 import os
+import shutil
 import sys
 import time as _time
 from pathlib import Path
@@ -16,13 +17,12 @@
     hercules_float_type,
     interpolate_df,
     load_hercules_input,
+    make_unique_folder_name,
     setup_logging,
 )
 
 LOGFILE = str(dt.datetime.now()).replace(":", "_").replace(" ", "_").replace(".", "_")
 
-Path("outputs").mkdir(parents=True, exist_ok=True)
-
 
 class HerculesModel:
     def __init__(self, input_file):
@@ -35,14 +35,44 @@ def __init__(self, input_file):
 
         """
 
+        # Load and validate the input file
+        h_dict = self._load_hercules_input(input_file)
+
+        # set default output directory to cwd / "outputs"
+        output_dir = Path(h_dict.get("output_dir", "outputs")).absolute()
+
+        # If the output directory exists, and overwrite_outputs is True
+        # The output folder will be deleted and recreated.
+        if h_dict.get("overwrite_outputs", True):
+            if output_dir.exists():
+                shutil.rmtree(output_dir)
+        else:
+            # Make a unique foldername with the same basename as the output_dir
+            proposed_dirname = str(output_dir.name)
+            output_dir = make_unique_folder_name(output_dir.parent, output_dir.name)
+            # If the proposed directory already existed and a new folder was proposed,
+            # update the logger directory to have the same unique number
+            if output_dir.name != proposed_dirname:
+                unique_folder_id_parts = output_dir.name.split(proposed_dirname)
+                unique_folder_id = "".join(p for p in unique_folder_id_parts)
+                if h_dict.get("logging", {}).get("logging_dir", None) is not None:
+                    if isinstance(h_dict["logging"]["logging_dir"], str):
+                        h_dict["logging"].update(
+                            {"logging_dir": Path(h_dict["logging"]["logging_dir"])}
+                        )
+
+                    logging_parent_dir = h_dict["logging"]["logging_dir"].parent
+                    logging_dir_basename = h_dict["logging"]["logging_dir"].name
+                    new_log_fpath = logging_parent_dir / f"{logging_dir_basename}{unique_folder_id}"
+
+                    h_dict["logging"].update({"logging_dir": new_log_fpath})
+
         # Make sure output folder exists
-        Path("outputs").mkdir(parents=True, exist_ok=True)
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
 
         # Set up logging
-        self.logger = self._setup_logging()
-
-        # Load and validate the input file
-        h_dict = self._load_hercules_input(input_file)
+        logging_inputs = {"logging_dir": output_dir} | h_dict.get("logging", {})
+        self.logger = self._setup_logging(**logging_inputs)
 
         # Initialize the flattened h_dict
         self.h_dict_flat = {}
@@ -83,8 +113,19 @@ def __init__(self, input_file):
             # Ensure .h5 extension
             if not self.output_file.endswith(".h5"):
                 self.output_file = self.output_file.rsplit(".", 1)[0] + ".h5"
+
+            if "/" in self.output_file:
+                # check if folder was specfied in output_file
+                if Path(self.output_file).parent != output_dir:
+                    # if folder of output_file does not match output_dir, then
+                    # just use the name of the output file
+                    self.output_file = output_dir / self.output_file.split("/")[-1]
+            else:
+                self.output_file = output_dir / self.output_file
         else:
-            self.output_file = "outputs/hercules_output.h5"
+            self.output_file = output_dir / "hercules_output.h5"
+
+        self.output_file = Path(self.output_file).absolute()
 
         # Initialize HDF5 output system
         self.hdf5_file = None
@@ -124,25 +165,28 @@ def __init__(self, input_file):
         # starttime_utc is required and should already be set, but ensure it's still present
         self.starttime_utc = self.h_dict["starttime_utc"]
 
-    def _setup_logging(self, logfile="log_hercules.log", console_output=True):
+    def _setup_logging(self, log_file_name="log_hercules.log", **kwargs):
         """Set up logging to file and console.
 
         Creates 'outputs' directory and configures file/console logging with timestamps.
         This method wraps the utilities.setup_logging function for backward compatibility.
 
         Args:
-            logfile (str, optional): Log file name. Defaults to "log_hercules.log".
-            console_output (bool, optional): Enable console output. Defaults to True.
+            log_file_name (str, optional): Log file name. Defaults to "log_hercules.log".
+            **kwargs (dict, optional): Extra arguments passed to setup_logging
 
         Returns:
             logging.Logger: Configured logger instance.
         """
-        return setup_logging(
-            logger_name="hercules",
-            log_file=logfile,
-            console_output=console_output,
-            console_prefix="HERCULES",
-        )
+
+        logging_defaults = {
+            "logger_name": "hercules",
+            "log_file": log_file_name,
+        }
+
+        # Update the defaults with any input kwargs
+        logging_inputs = logging_defaults | kwargs
+        return setup_logging(**logging_inputs)
 
     def _load_hercules_input(self, filename):
         """Load and validate Hercules input file.
@@ -437,7 +481,8 @@ def _save_h_dict_as_text(self):
         # to see full dictionary in interpreting log
 
         original_stdout = sys.stdout
-        with open("outputs/h_dict.echo", "w") as f_i:
+        echo_file_fpath = self.output_file.parent / "h_dict.echo"
+        with open(echo_file_fpath, "w") as f_i:
             sys.stdout = f_i  # Change the standard output to the file we created.
             print(self.h_dict)
             sys.stdout = original_stdout  # Reset the standard output to its original value

hercules/plant_components/component_base.py

@@ -1,6 +1,6 @@
 # Base class for plant components in Hercules.
 
-
+from pathlib import Path
 from typing import ClassVar
 
 from hercules.utilities import setup_logging
@@ -60,13 +60,32 @@ def __init__(self, h_dict, component_name):
         self.component_type = type(self).__name__
 
         # Set up logging
+        output_dir = Path(h_dict.get("output_dir", "outputs")).absolute()
+        # Get the default output folder
+        logging_inputs = {"logging_dir": output_dir} | {
+            "logging_dir": h_dict.get("logging", {}).get("logging_dir", output_dir)
+        }
+        if h_dict[component_name].get("logging", {}).get("logging_dir", None) is not None:
+            msg = (
+                f"Cannot specify log folder for component {component_name}, "
+                f"all log files will be saved to the logging_dir {logging_inputs['logging_dir']}"
+            )
+            raise ValueError(msg)
+
+        logging_inputs["logging_dir"] = Path(logging_inputs["logging_dir"]).absolute()
+        logging_inputs = (
+            logging_inputs | h_dict.get("logging", {}) | h_dict[component_name].get("logging", {})
+        )
         # Check if log_file_name is defined in the h_dict[component_name]
         if "log_file_name" in h_dict[component_name]:
             self.log_file_name = h_dict[component_name]["log_file_name"]
         else:
-            self.log_file_name = f"outputs/log_{component_name}.log"
+            self.log_file_name = f"log_{component_name}.log"
+
+        if "log_file" in logging_inputs:
+            logging_inputs["log_file"] = self.log_file_name
 
-        self.logger = self._setup_logging(self.log_file_name)
+        self.logger = self._setup_logging(self.log_file_name, **logging_inputs)
 
         # Parse log_channels from the h_dict
         if "log_channels" in h_dict[component_name]:
@@ -103,7 +122,7 @@ def __init__(self, h_dict, component_name):
         if self.verbose:
             self.logger.info(f"Verbose flag = {self.verbose}")
 
-    def _setup_logging(self, log_file_name):
+    def _setup_logging(self, log_file_name, **kwargs):
         """Set up logging for the component.
 
 
@@ -112,18 +131,21 @@ def _setup_logging(self, log_file_name):
 
 
         Args:
-            log_file_name (str): Full path to the log file.
+            log_file_name (str): Filename of the logger
+            **kwargs (dict, optional): Extra arguments passed to setup_logging
 
         Returns:
             logging.Logger: Configured logger instance for the component.
         """
-        return setup_logging(
-            logger_name=self.component_name,
-            log_file=log_file_name,
-            console_output=True,
-            console_prefix=self.component_name.upper(),
-            use_outputs_dir=False,  # log_file_name is already a full path
-        )
+        logging_defaults = {
+            "logger_name": self.component_name,
+            "log_file": log_file_name,
+            "console_prefix": self.component_name.upper(),
+        }
+
+        # Update the defaults with any input kwargs
+        logging_inputs = logging_defaults | kwargs
+        return setup_logging(**logging_inputs)
 
     def __del__(self):
         """Cleanup method to properly close log file handlers."""