Better minimal buildstock 2025 release 1

.github/workflows/config.yml

@@ -20,8 +20,8 @@ env:
   OPENSTUDIO_PLATFORM: Ubuntu-22.04-x86_64
   OPENSTUDIO_EXT: deb
   OPENSTUDIO_URL: https://github.com/NREL/OpenStudio/releases/download/v3.9.0
-  BUILDSTOCK_BRANCH: develop
-  BUILDSTOCK_QUERY_BRANCH: main
+  BUILDSTOCK_BRANCH: ppv2
+  BUILDSTOCK_QUERY_BRANCH: set_cover
 
 jobs:
   format-files:
@@ -274,9 +274,6 @@ jobs:
         run: |
           gunzip -v project_national/national_baseline/results_csvs/*.csv.gz
           gunzip -v project_testing/testing_baseline/results_csvs/*.csv.gz
-
-          tar -xvf project_national/national_baseline/simulation_output/*.tar.gz -C project_national/national_baseline/simulation_output --wildcards '*results_timeseries.csv' --wildcards '*.xml' --wildcards '*data_point_out.json' --wildcards '*.osw'
-          tar -xvf project_testing/testing_baseline/simulation_output/*.tar.gz -C project_testing/testing_baseline/simulation_output --wildcards '*results_timeseries.csv' --wildcards '*.xml' --wildcards '*data_point_out.json' --wildcards '*.osw' --wildcards '*in.osm' --wildcards '*in.idf'
 
       - name: Process results
         run: |
@@ -538,7 +535,7 @@ jobs:
       - name: Generate minimal buildstock and other files
         run: |
           cd project_national/resources
-          python3 -m buildstock_query.tools.upgrades_analyzer --yaml_file ../sdr_upgrades_tmy3.yml --buildstock_file national_550ksamples.csv --opt_sat_file options_saturations.csv --output_prefix sdr_
+          python3 -m buildstock_query.tools.upgrades_analyzer --yaml_file ../sdr_upgrades_tmy3.yml --buildstock_file national_550ksamples.csv --opt_sat_file options_saturations.csv --prev_minimal_buildstock_file sdr_minimal_buildstock.csv --output_prefix sdr_
           rm national_550ksamples.csv
           rm national_550ksamples.csv.gz
 
@@ -659,7 +656,7 @@ jobs:
           fi
 
           # Run buildstockbatch and tee output to a log file
-          buildstock_local --low-disk $EXTRA_ARGS project_national/sdr_upgrades_tmy3.yml 2>&1 | tee buildstock_local.log
+          buildstock_local --ultra-low-disk-no-timeseries $EXTRA_ARGS project_national/sdr_upgrades_tmy3.yml 2>&1 | tee buildstock_local.log
 
           # Check the log for schema mismatch warnings and fail if any are present
           SCHEMA_MISMATCH_PATTERN='^Missing columns in output data|^Extra columns in output data'
@@ -693,10 +690,10 @@ jobs:
           echo "Checking for failed simulations in results files..."
           FAILED=false
           for csv_file in project_national/sdr_upgrades_tmy3/results_csvs/*.csv; do
-            python -m resstockpostproc.get_failures "$csv_file" --verbose --exit-code || FAILED=true
+            python -m resstockpostproc.get_failures "$csv_file" --verbose --exit-code --failure-threshold 0.5 || FAILED=true
           done
           if [ "$FAILED" = true ]; then
-            echo "ERROR: Some simulations failed. See details above."
+            echo "ERROR: Significantly high failure rate detected. See details above."
             exit 1
           fi
 

docs/technical_development_guide/source/workflow_outputs/csv_tables.rb

@@ -2,8 +2,8 @@
 
 require 'csv'
 
-inputs = CSV.read(File.join(File.dirname(__FILE__), '../../../../resources/data/dictionary/inputs.csv'), headers: true)
-outputs = CSV.read(File.join(File.dirname(__FILE__), '../../../../resources/data/dictionary/outputs.csv'), headers: true)
+inputs = CSV.read(File.join(File.dirname(__FILE__), '../../../../postprocessing/resstockpostproc/resources/dictionary/inputs.csv'), headers: true)
+outputs = CSV.read(File.join(File.dirname(__FILE__), '../../../../postprocessing/resstockpostproc/resources/dictionary/outputs.csv'), headers: true)
 
 csv_tables = {
   'characteristics.csv' => { 'annual' => false, 'timeseries' => false, 'kws' => ['build_existing_model.'], 'usecols' => ['Input Name', 'Input Description'] },

docs/technical_development_guide/source/workflow_outputs/csv_tables/other_outputs.csv

@@ -1,10 +1,12 @@
 Input Name,Input Description
 building_id,The building unit number (between 1 and the number of samples).
+upgrade,The upgrade run number (will be present for upgrade run).
 job_id,The unique identifier of the job/processor.
 started_at,Simulation start time.
 completed_at,Simulation end time.
 completed_status,"Simulation Success, Fail, or Invalid."
 eplusout_err,Contents of the eplusout.err file.
+step_failures,Contents of the step_failures.
 build_existing_model.air_leakage_to_outside_ach_50,Total infiltration to the dwelling unit adjusted by ratio of exterior envelope surface area to total envelope surface area.
 build_existing_model.applicable,The measure was applied to the workflow.
 build_existing_model.buildstock_csv_path,Absolute/relative path of the buildstock CSV file. Relative is compared to the 'lib/housing_characteristics' directory.
@@ -109,4 +111,27 @@ report_utility_bills.register_monthly_bills,Registers monthly utility bills with
 upgrade_costs.applicable,The measure was applied to the workflow.
 upgrade_costs.debug,"If true, retain existing and upgraded intermediate files."
 upgrade_costs.upgrade_cost_usd,Total cost of the upgrade.
+upgrade_costs.door_area_ft_2,Total area of doors in the building in square feet.
+upgrade_costs.duct_unconditioned_surface_area_ft_2,Total unconditioned surface area of ducts in the building in square feet.
+upgrade_costs.floor_area_attic_ft_2,Total attic floor area in the building in square feet.
+upgrade_costs.floor_area_attic_insulation_increase_ft_2_delta_r_value,Total attic floor area in the building in square feet.
+upgrade_costs.floor_area_conditioned_ft_2,Total conditioned floor area in the building in square feet.
+upgrade_costs.floor_area_conditioned_infiltration_reduction_ft_2_delta_ach_50,Total conditioned floor area in the building in square feet.
+upgrade_costs.floor_area_foundation_ft_2,Total foundation floor area in the building in square feet.
+upgrade_costs.floor_area_lighting_ft_2,Total floor area in the building in square feet.
+upgrade_costs.flow_rate_mechanical_ventilation_cfm,Total flow rate of mechanical ventilation in cubic feet per minute.
+upgrade_costs.rim_joist_area_above_grade_exterior_ft_2,Total rim joist area above grade in the building in square feet.
+upgrade_costs.roof_area_ft_2,Total roof area in the building in square feet.
+upgrade_costs.size_cooling_system_primary_k_btu_h,Total size of primary cooling system in the building in kilowatt-hours.
+upgrade_costs.size_heat_pump_backup_primary_k_btu_h,Total size of primary heat pump backup in the building in kilowatt-hours.
+upgrade_costs.size_heating_system_primary_k_btu_h,Total size of primary heating system in the building in kilowatt-hours.
+upgrade_costs.size_heating_system_secondary_k_btu_h,Total size of secondary heating system in the building in kilowatt-hours.
+upgrade_costs.size_water_heater_gal,Total size of water heater in the building in gallons.
+upgrade_costs.slab_perimeter_exposed_conditioned_ft,Total perimeter of slab exposed in the building in feet.
+upgrade_costs.upgrade_cost_usd,Total cost of the upgrade in US dollars.
+upgrade_costs.wall_area_above_grade_conditioned_ft_2,Total wall area above grade in the building in square feet.
+upgrade_costs.wall_area_above_grade_exterior_ft_2,Total wall area above grade in the building in square feet.
+upgrade_costs.wall_area_below_grade_ft_2,Total wall area below grade in the building in square feet.
+upgrade_costs.window_area_ft_2,Total window area in the building in square feet.
+upgrade_costs.option_<option_number>_name,Name of the option.
 qoi_report.applicable,The measure was applied to the workflow.

docs/technical_development_guide/source/workflow_outputs/csv_tables/other_timeseries.csv

@@ -79,3 +79,6 @@ Electricity:Facility,electricity_facility__j,J,"Any user-specified EnergyPlus ou
 ,schedules_electric_vehicle_discharging,frac,Availability schedule for electric vehicle discharging.
 ,schedules_peak_period,frac,Load flexibility measure ouput that indicates when the system is in the peak demand period.
 ,schedules_pre_peak_period,frac,Load flexibility measure ouput that identifies when the system is in the period leading up to the peak demand period.
+People Sensible Heating Rate: Occupants,people_sensible_heating_rate__occupants__w,W,People Sensible Heating Rate
+People Total Heating Rate: Occupants,people_total_heating_rate__occupants__w,W,People Total Heating Rate
+Site Outdoor Air Humidity Ratio: Environment,site_outdoor_air_humidity_ratio__environment__kgwater/kgdryair,kgWater/kgDryAir,Site Outdoor Air Humidity Ratio Environment

docs/technical_development_guide/source/workflow_outputs/index.rst

@@ -11,12 +11,12 @@ See the `Residential HPXML Workflow Generator <https://buildstockbatch.readthedo
 Tables in the :ref:`default_outputs` and :ref:`optional_outputs` sections are generated based on `data dictionary files for inputs and outputs <https://github.com/NREL/resstock/tree/data-dictionary/resources/data/dictionary/>`_.
 The data dictionary files, along with their data columns, are shown below:
 
-- ``resources/data/dictionary/inputs.csv``
+- ``postprocessing/resstockpostproc/resources/dictionary/inputs.csv``
 
   - Input Name
   - Input Description
 
-- ``resources/data/dictionary/outputs.csv``:
+- ``postprocessing/resstockpostproc/resources/dictionary/outputs.csv``:
 
   - Row Index
   - Sums To

postprocessing/resstockpostproc/__init__.py

@@ -1 +1,2 @@
-from .process_metadata import publish_baseline_annual_results, publish_upgrade_annual_results
+from .process_metadata import publish_baseline_annual_results, publish_upgrade_annual_results
+from .process_data_dict import get_resstock_timeseries_schema, get_bsb_timeseries_schema, get_annual_results_schema

postprocessing/resstockpostproc/get_failures.py

@@ -8,7 +8,7 @@
 import sys
 from pathlib import Path
 from typing import Any
-
+import json
 import polars as pl
 
 
@@ -21,12 +21,12 @@ def get_failures(csv_path: str, verbose: bool = False) -> list[dict[str, Any]]:
         verbose: Whether to print detailed information during processing
 
     Returns:
-        List of dictionaries containing details of failed simulations
+        List of dictionaries containing details of failed simulations and the percentage of failed simulations
     """
     path = Path(csv_path)
     if not path.exists():
         print(f"Error: File {csv_path} does not exist.")
-        return [{"building_id": "N/A", "completed_status": "N/A", "step_failures": f"Path {csv_path} does not exist."}]
+        return [{"building_id": "N/A", "completed_status": "N/A", "step_failures": f"Path {csv_path} does not exist."}], 1.0
 
     if verbose:
         print(f"Checking file: {csv_path}")
@@ -35,7 +35,7 @@ def get_failures(csv_path: str, verbose: bool = False) -> list[dict[str, Any]]:
     try:
         # Scan the CSV lazily
         df_lazy = pl.scan_csv(csv_path)
-
+        row_count = df_lazy.select(pl.len()).collect().item()
         # Check if completed_status column exists
         schema = df_lazy.collect_schema()
         if "completed_status" not in schema:
@@ -46,7 +46,7 @@ def get_failures(csv_path: str, verbose: bool = False) -> list[dict[str, Any]]:
                     "completed_status": "N/A",
                     "step_failures": f"completed_status column not found in {csv_path}",
                 }
-            ]
+            ], 1.0
         columns_to_select = [pl.col("building_id"), pl.col("completed_status")]
 
         if "step_failures" in schema:
@@ -61,7 +61,7 @@ def get_failures(csv_path: str, verbose: bool = False) -> list[dict[str, Any]]:
 
         failed_sims = df_lazy.select(columns_to_select).filter(pl.col("completed_status") == "Fail").collect()
         failures = failed_sims.to_dicts() if not failed_sims.is_empty() else []
-        return failures
+        return failures, len(failures) * 1.0 / row_count
     except Exception as e:  # noqa: BLE001
         print(f"Error processing {csv_path}: {e!s}")
         return [
@@ -71,7 +71,7 @@ def get_failures(csv_path: str, verbose: bool = False) -> list[dict[str, Any]]:
                 "step_failures": f"Error processing {csv_path}: {e!s}",
                 "eplusout_err": "N/A",
             }
-        ]
+        ], 1.0
 
 
 def print_failures(failures: list[dict[str, Any]], csv_path: str) -> None:
@@ -102,7 +102,7 @@ def print_failures(failures: list[dict[str, Any]], csv_path: str) -> None:
             # If step_failures is a string representation of a Python list
             if isinstance(step_failures, str) and step_failures.strip().startswith("["):
                 # Parse the Python literal string into a Python object
-                failures_data = ast.literal_eval(step_failures)
+                failures_data = json.loads(step_failures)
                 print("     Step Failures:")
 
                 # Format each failure entry
@@ -137,13 +137,15 @@ def main():
     parser.add_argument("csv_file", help="Path to the CSV file to check for failures.")
     parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose output.")
     parser.add_argument("--exit-code", "-e", action="store_true", help="Exit with non-zero code if failures are found.")
+    parser.add_argument("--failure-threshold", "-f", type=float, default=0.0, help="Failure threshold percentage.")
 
     args = parser.parse_args()
 
-    failures = get_failures(args.csv_file, args.verbose)
+    failures, failure_rate = get_failures(args.csv_file, args.verbose)
     print_failures(failures, args.csv_file)
+    print(f"Failure rate: {failure_rate * 100:.2f}%")
 
-    if args.exit_code and failures:
+    if args.exit_code and failure_rate > args.failure_threshold:
         print(f"ERROR: {len(failures)} simulation(s) failed. Check the logs above for details.")
         sys.exit(1)