diff --git a/buildstock_query/aggregate_query.py b/buildstock_query/aggregate_query.py
index f449fcc9..7df53b71 100644
--- a/buildstock_query/aggregate_query.py
+++ b/buildstock_query/aggregate_query.py
@@ -34,9 +34,8 @@ def aggregate_annual(self, *,
else:
upgrade_id = self._bsq._validate_upgrade(params.upgrade_id)
enduse_cols = self._bsq._get_enduse_cols(params.enduses, table='upgrade')
-
total_weight = self._bsq._get_weight(weights)
- enduse_selection = [safunc.sum(enduse * total_weight).label(self._bsq._simple_label(enduse.name))
+ enduse_selection = [self._bsq._agg_column(enduse, total_weight, params.agg_func)
for enduse in enduse_cols]
if params.get_quartiles:
enduse_selection += [sa.func.approx_percentile(enduse, [0, 0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98, 1]).label(
@@ -94,6 +93,7 @@ def _aggregate_timeseries_light(self,
new_query = params.copy()
new_query.enduses = [enduse.name]
new_query.split_enduses = False
+ new_query.get_query_only = True
query = self.aggregate_timeseries(params=new_query)
batch_queries_to_submit.append(query)
@@ -109,14 +109,17 @@ def _aggregate_timeseries_light(self,
result_dfs = self._bsq.get_batch_query_result(batch_id=batch_query_id, combine=False)
logger.info("Joining the individual enduses result into a single DataFrame")
group_by = self._bsq._clean_group_by(params.group_by)
- for res in result_dfs:
- res.set_index(group_by, inplace=True)
+ if not params.collapse_ts and 'time' not in group_by:
+ group_by.append('time')
+ for i, res in enumerate(result_dfs):
+ if group_by:
+ res.set_index(group_by, inplace=True)
+ if i > 0:
+ res.drop(columns=['sample_count', 'units_count'], inplace=True, errors='ignore')
self.result_dfs = result_dfs
- joined_enduses_df = result_dfs[0].drop(columns=['query_id'])
- for enduse, res in list(zip(params.enduses, result_dfs))[1:]:
- if not isinstance(enduse, str):
- enduse = enduse.name
- joined_enduses_df = joined_enduses_df.join(res[[enduse]])
+ joined_enduses_df = result_dfs[0]
+ for res in result_dfs[1:]:
+ joined_enduses_df = joined_enduses_df.join(res)
logger.info("Joining Completed.")
return joined_enduses_df.reset_index()
@@ -136,8 +139,7 @@ def aggregate_timeseries(self, params: TSQuery):
[self._bsq._get_table(jl[0]) for jl in params.join_list] # ingress all tables in join list
enduses_cols = self._bsq._get_enduse_cols(params.enduses, table='timeseries')
total_weight = self._bsq._get_weight(params.weights)
-
- enduse_selection = [safunc.sum(enduse * total_weight).label(self._bsq._simple_label(enduse.name))
+ enduse_selection = [self._bsq._agg_column(enduse, total_weight, params.agg_func)
for enduse in enduses_cols]
group_by = list(params.group_by)
if self._bsq.timestamp_column_name not in group_by and params.collapse_ts:
diff --git a/buildstock_query/aggregate_query.pyi b/buildstock_query/aggregate_query.pyi
index 8fdcf7c6..c75d7044 100644
--- a/buildstock_query/aggregate_query.pyi
+++ b/buildstock_query/aggregate_query.pyi
@@ -1,4 +1,4 @@
-from typing import Optional, Union, Literal, Sequence
+from typing import Optional, Union, Literal, Sequence, Callable
import pandas as pd
import typing
from buildstock_query.schema.query_params import TSQuery, AnnualQuery
@@ -24,6 +24,7 @@ class BuildStockAggregate:
avoid: Sequence[tuple[AnyColType, Union[str, int, Sequence[Union[int, str]]]]] = [],
get_quartiles: bool = False,
get_nonzero_count: bool = False,
+ agg_func: Optional[Union[str, Callable]] = 'sum'
) -> str:
...
@@ -40,6 +41,7 @@ class BuildStockAggregate:
avoid: Sequence[tuple[AnyColType, Union[str, int, Sequence[Union[int, str]]]]] = [],
get_quartiles: bool = False,
get_nonzero_count: bool = False,
+ agg_func: Optional[Union[str, Callable]] = 'sum'
) -> pd.DataFrame:
...
@@ -56,6 +58,7 @@ class BuildStockAggregate:
avoid: Sequence[tuple[AnyColType, Union[str, int, Sequence[Union[int, str]]]]] = [],
get_quartiles: bool = False,
get_nonzero_count: bool = False,
+ agg_func: Optional[Union[str, Callable]] = 'sum'
) -> Union[pd.DataFrame, str]:
"""
Aggregates the baseline annual result on select enduses.
@@ -93,6 +96,9 @@ class BuildStockAggregate:
get_query_only: Skips submitting the query to Athena and just returns the query string. Useful for batch
submitting multiple queries or debugging
+ agg_func: The aggregation function to use. Defaults to 'sum'.
+ See other options in https://prestodb.io/docs/current/functions/aggregate.html
+
Returns:
if get_query_only is True, returns the query_string, otherwise returns the dataframe
"""
@@ -118,7 +124,8 @@ class BuildStockAggregate:
split_enduses: bool = False,
collapse_ts: bool = False,
timestamp_grouping_func: Optional[str] = None,
- limit: Optional[int] = None
+ limit: Optional[int] = None,
+ agg_func: Optional[Union[str, Callable]] = 'sum'
) -> str:
...
@@ -136,7 +143,8 @@ class BuildStockAggregate:
collapse_ts: bool = False,
timestamp_grouping_func: Optional[str] = None,
get_query_only: Literal[False] = False,
- limit: Optional[int] = None
+ limit: Optional[int] = None,
+ agg_func: Optional[Union[str, Callable]] = 'sum'
) -> pd.DataFrame:
...
@@ -154,7 +162,8 @@ class BuildStockAggregate:
split_enduses: bool = False,
collapse_ts: bool = False,
timestamp_grouping_func: Optional[str] = None,
- limit: Optional[int] = None
+ limit: Optional[int] = None,
+ agg_func: Optional[Union[str, Callable]] = 'sum'
) -> Union[str, pd.DataFrame]:
"""
Aggregates the timeseries result on select enduses.
@@ -188,7 +197,8 @@ class BuildStockAggregate:
get_query_only: Skips submitting the query to Athena and just returns the query string. Useful for batch
submitting multiple queries or debugging
-
+ agg_func: The aggregation function to use. Defaults to 'sum'.
+ See other options in https://prestodb.io/docs/current/functions/aggregate.html
Returns:
if get_query_only is True, returns the query_string, otherwise, returns the DataFrame
diff --git a/buildstock_query/db_schema/comstock_default.toml b/buildstock_query/db_schema/comstock_default.toml
index 47f07ab1..e553472d 100644
--- a/buildstock_query/db_schema/comstock_default.toml
+++ b/buildstock_query/db_schema/comstock_default.toml
@@ -16,6 +16,7 @@ timestamp = "time"
completed_status = "completed_status"
unmet_hours_cooling_hr = ""
unmet_hours_heating_hr = ""
+upgrade = "apply_upgrade.upgrade"
fuel_totals = [
"simulation_output_report.total_site_electricity_kwh",
"simulation_output_report.total_site_energy_mbtu",
diff --git a/buildstock_query/db_schema/db_schema_model.py b/buildstock_query/db_schema/db_schema_model.py
index 7d67b278..cc319962 100644
--- a/buildstock_query/db_schema/db_schema_model.py
+++ b/buildstock_query/db_schema/db_schema_model.py
@@ -17,6 +17,7 @@ class ColumnNames(BaseModel):
sample_weight: str
sqft: str
timestamp: str
+ upgrade: str
completed_status: str
unmet_hours_cooling_hr: str
unmet_hours_heating_hr: str
diff --git a/buildstock_query/db_schema/resstock_default.toml b/buildstock_query/db_schema/resstock_default.toml
index 9c8d2b63..f29ef838 100644
--- a/buildstock_query/db_schema/resstock_default.toml
+++ b/buildstock_query/db_schema/resstock_default.toml
@@ -16,6 +16,7 @@ timestamp = "time"
completed_status = "completed_status"
unmet_hours_cooling_hr = "report_simulation_output.unmet_hours_cooling_hr"
unmet_hours_heating_hr = "report_simulation_output.unmet_hours_heating_hr"
+upgrade = "upgrade"
fuel_totals = [
'report_simulation_output.energy_use_total_m_btu',
'report_simulation_output.fuel_use_coal_total_m_btu',
diff --git a/buildstock_query/db_schema/resstock_oedi.toml b/buildstock_query/db_schema/resstock_oedi.toml
index acc4645b..ca0331a4 100644
--- a/buildstock_query/db_schema/resstock_oedi.toml
+++ b/buildstock_query/db_schema/resstock_oedi.toml
@@ -15,6 +15,7 @@ timestamp = "timestamp"
completed_status = "applicability"
unmet_hours_cooling_hr = ""
unmet_hours_heating_hr = ""
+upgrade = "upgrade"
fuel_totals = ["out.electricity.total.energy_consumption",
"out.natural_gas.total.energy_consumption",
"out.fuel_oil.total.energy_consumption",
diff --git a/buildstock_query/db_schema/resstock_oedi_vu.toml b/buildstock_query/db_schema/resstock_oedi_vu.toml
new file mode 100644
index 00000000..fe59be39
--- /dev/null
+++ b/buildstock_query/db_schema/resstock_oedi_vu.toml
@@ -0,0 +1,31 @@
+[table_suffix]
+baseline = "_metadata_state_vu"
+timeseries = "_by_state_vu"
+upgrades = "_metadata_state_vu"
+
+[column_prefix]
+characteristics = "in."
+output = "out."
+
+[column_names]
+building_id = "bldg_id"
+sample_weight = "weight"
+sqft = "in.sqft"
+timestamp = "timestamp"
+completed_status = "applicability"
+unmet_hours_cooling_hr = ""
+unmet_hours_heating_hr = ""
+fuel_totals = ["out.electricity.total.energy_consumption",
+ "out.natural_gas.total.energy_consumption",
+ "out.fuel_oil.total.energy_consumption",
+ "out.propane.total.energy_consumption",
+ ]
+upgrade="upgrade"
+
+[completion_values]
+success = "true"
+fail = ""
+unapplicable = "false"
+
+[structure]
+unapplicables_have_ts = "true"
diff --git a/buildstock_query/main.py b/buildstock_query/main.py
index be4a0e7c..0432278b 100644
--- a/buildstock_query/main.py
+++ b/buildstock_query/main.py
@@ -44,6 +44,7 @@ def __init__(self,
region_name: str = 'us-west-2',
execution_history: Optional[str] = None,
skip_reports: bool = False,
+ skip_integrity_check: bool = True,
athena_query_reuse: bool = True,
**kwargs,
) -> None:
@@ -69,6 +70,8 @@ def __init__(self,
custom filename.
skip_reports (bool, optional): If true, skips report printing during initialization. If False (default),
prints report from `buildstock_query.report_query.BuildStockReport.get_success_report`.
+ skip_integrity_check (bool, optional): If true, skips integrity check during initialization. If False (default),
+ checks integrity between baseline and timeseries tables. Most people don't need to check this.
athena_query_reuse (bool, optional): When true, Athena will make use of its built-in 7 day query cache.
When false, it will not. Defaults to True. One use case to set this to False is when you have modified
the underlying s3 data or glue schema and want to make sure you are not using the cached results.
@@ -108,7 +111,7 @@ def __init__(self,
if not skip_reports:
logger.info("Getting Success counts...")
print(self.report.get_success_report())
- if self.ts_table is not None:
+ if self.ts_table is not None and not skip_integrity_check:
self.report.check_ts_bs_integrity()
self.save_cache()
@@ -164,10 +167,14 @@ def get_upgrade_names(self, get_query_only: Literal[True]) -> str:
def get_upgrade_names(self, get_query_only: bool = False) -> Union[str, dict]:
if self.up_table is None:
raise ValueError("This run has no upgrades")
- upgrade_table = self.up_table
+ if isinstance(self.up_table, sa.Table):
+ upgrade_table = self.up_table.name
+ else:
+ upgrade_table = self._compile(self.up_table)
+ upgrade_col = self.db_schema.column_names.upgrade
query = f"""
- Select cast(upgrade as integer) as upgrade, arbitrary("apply_upgrade.upgrade_name") as upgrade_name
- from {upgrade_table}
+ Select cast(upgrade as integer) as upgrade, arbitrary("{upgrade_col}") as upgrade_name
+ from ({upgrade_table})
group by 1 order by 1
"""
if get_query_only:
@@ -300,6 +307,31 @@ def get_results_csv(self,
result = self.execute(query)
return result.set_index(self.bs_bldgid_column.name)
+ def _get_table_location(self, db_table_name: str) -> str:
+ table_info = self._aws_glue.get_table(DatabaseName=self.db_name, Name=db_table_name)['Table']
+ if table_info.get('TableType') != 'VIRTUAL_VIEW':
+ return table_info['StorageDescriptor']['Location']
+
+ try:
+ import base64
+ import re
+ import json
+ view_original_text = table_info.get('ViewOriginalText', '')
+ view_original_text = view_original_text[len('/* Presto View: '):-len(' */')]
+ decoded_json= json.loads(base64.b64decode(view_original_text).decode('utf-8'))
+ sql_text = decoded_json['originalSql']
+ match = re.search(r'FROM\s+([^\s,]+)', sql_text, re.IGNORECASE)
+ if not match:
+ raise ValueError(f"Could not find source table in view definition for {db_table_name}")
+ source_table = match.group(1)
+ source_table_info = self._aws_glue.get_table(
+ DatabaseName=self.db_name, # Using same database as view
+ Name=source_table
+ )['Table']
+ return source_table_info['StorageDescriptor']['Location']
+ except Exception as e:
+ raise ValueError(f"Failed to parse view definition for {db_table_name}: {str(e)}")
+
def _download_results_csv(self) -> str:
"""Downloads the results csv from s3 and returns the path to the downloaded file.
Returns:
@@ -313,8 +345,7 @@ def _download_results_csv(self) -> str:
db_table_name = f'{self.table_name}{self.db_schema.table_suffix.baseline}'
else:
db_table_name = self.table_name[0]
- baseline_path = self._aws_glue.get_table(DatabaseName=self.db_name,
- Name=db_table_name)['Table']['StorageDescriptor']['Location']
+ baseline_path = self._get_table_location(db_table_name)
bucket = baseline_path.split('/')[2]
key = '/'.join(baseline_path.split('/')[3:])
s3_data = self._aws_s3.list_objects(Bucket=bucket, Prefix=key)
diff --git a/buildstock_query/query_core.py b/buildstock_query/query_core.py
index 23fc15a2..a8c743a8 100644
--- a/buildstock_query/query_core.py
+++ b/buildstock_query/query_core.py
@@ -5,10 +5,11 @@
from pyathena.error import OperationalError
from pyathena.sqlalchemy.base import AthenaDialect
import sqlalchemy as sa
+from sqlalchemy.sql import func as safunc
from pyathena.pandas.async_cursor import AsyncPandasCursor
from pyathena.pandas.cursor import PandasCursor
import os
-from typing import Union, Optional, Literal, Sequence
+from typing import Union, Optional, Literal, Sequence, Callable
import typing
import time
import logging
@@ -155,7 +156,7 @@ def load_cache(self, path: Optional[str] = None):
pickle_path = pathlib.Path(path) if path else self._get_cache_file_path()
before_count = len(self._query_cache)
saved_cache = load_pickle(pickle_path)
- logger.info(f"{len(saved_cache)} queries cache read from {path}.")
+ logger.info(f"{len(saved_cache)} queries cache read from {pickle_path}.")
self._query_cache.update(saved_cache)
self.last_saved_queries = set(saved_cache)
after_count = len(self._query_cache)
@@ -518,10 +519,12 @@ def get_pandas(future):
return exe_id, AthenaFutureDf(result_future)
else:
if query not in self._query_cache:
- self._query_cache[query] = self._conn.cursor().execute(query,
- result_reuse_enable=self.athena_query_reuse,
- result_reuse_minutes=60 * 24 * 7,
- ).as_pandas()
+ cursor = self._conn.cursor()
+ self._query_cache[query] = cursor.execute(query,
+ result_reuse_enable=self.athena_query_reuse,
+ result_reuse_minutes=60 * 24 * 7,
+ ).as_pandas()
+ self._log_execution_cost(cursor.query_id)
return self._query_cache[query].copy()
def print_all_batch_query_status(self) -> None:
@@ -967,9 +970,14 @@ def get_cols(self, table: AnyTableType, fuel_type=None) -> Sequence[DBColType]:
tbl = self._get_table(table)
return [col for col in tbl.columns]
- def _simple_label(self, label: str):
+ def _simple_label(self, label: str, agg_func: Optional[Union[Callable, str]] = None):
label = label.removeprefix(self.db_schema.column_prefix.characteristics)
label = label.removeprefix(self.db_schema.column_prefix.output)
+
+ if callable(agg_func):
+ label += f"__{agg_func.__name__}"
+ elif isinstance(agg_func, str) and agg_func != 'sum':
+ label += f"__{agg_func}"
return label
def _add_restrict(self, query, restrict, *, bs_only=False):
@@ -1035,9 +1043,9 @@ def _add_order_by(self, query, order_by_selection):
query = query.order_by(*a)
return query
- def _get_weight(self, weights):
+ def _get_weight(self, weight_cols):
total_weight = self.sample_wt
- for weight_col in weights:
+ for weight_col in weight_cols:
if isinstance(weight_col, tuple):
tbl = self._get_table(weight_col[1])
total_weight *= tbl.c[weight_col[0]]
@@ -1045,6 +1053,18 @@ def _get_weight(self, weights):
total_weight *= self._get_column(weight_col)
return total_weight
+ def _agg_column(self, column: DBColType, weights, agg_func=None):
+ label = self._simple_label(column.name, agg_func)
+ if callable(agg_func):
+ return agg_func(column).label(label)
+ if agg_func is None or agg_func in ['sum']:
+ return safunc.sum(column * weights).label(label)
+ if agg_func in ['avg']:
+ return (safunc.sum(column * weights) / safunc.sum(weights)).label(label)
+ assert isinstance(agg_func, str), f"agg_func {agg_func} is not a string or callable"
+ agg_func = getattr(safunc, agg_func)
+ return agg_func(column).label(label)
+
def delete_everything(self):
"""Deletes the athena tables and data in s3 for the run.
"""
diff --git a/buildstock_query/savings_query.py b/buildstock_query/savings_query.py
index 6047368b..94d8fc42 100644
--- a/buildstock_query/savings_query.py
+++ b/buildstock_query/savings_query.py
@@ -89,6 +89,9 @@ def savings_shape(
) -> Union[pd.DataFrame, str]:
[self._bsq._get_table(jl[0]) for jl in params.join_list] # ingress all tables in join list
+ if params.agg_func != 'sum':
+ raise ValueError("Only 'sum' is supported for savings_shape")
+
upgrade_id = self._bsq._validate_upgrade(params.upgrade_id)
if params.timestamp_grouping_func and \
params.timestamp_grouping_func not in ['hour', 'day', 'month']:
diff --git a/buildstock_query/schema/query_params.py b/buildstock_query/schema/query_params.py
index a4f1d4e0..08d64053 100644
--- a/buildstock_query/schema/query_params.py
+++ b/buildstock_query/schema/query_params.py
@@ -1,5 +1,5 @@
from pydantic import BaseModel, Field
-from typing import Optional, Union, Sequence
+from typing import Optional, Union, Sequence, Callable
from typing import Literal
from buildstock_query.schema.utilities import AnyTableType, AnyColType
@@ -17,6 +17,7 @@ class AnnualQuery(BaseModel):
get_nonzero_count: bool = False
get_query_only: bool = False
limit: Optional[int] = None
+ agg_func: Optional[Union[str, Callable]] = 'sum'
class Config:
arbitrary_types_allowed = True
diff --git a/buildstock_query/tools/upgrades_visualizer/__init__.py b/buildstock_query/tools/upgrades_visualizer/__init__.py
deleted file mode 100644
index e06aa670..00000000
--- a/buildstock_query/tools/upgrades_visualizer/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from .upgrades_visualizer import main
-__all__ = ['main']
diff --git a/buildstock_query/tools/visualizer/__init__.py b/buildstock_query/tools/visualizer/__init__.py
new file mode 100644
index 00000000..87086465
--- /dev/null
+++ b/buildstock_query/tools/visualizer/__init__.py
@@ -0,0 +1,4 @@
+from .upgrades_visualizer import main as upgrades_visualizer
+from .timeseries_visualizer import main as timeseries_visualizer
+__all__ = ['upgrades_visualizer', 'timeseries_visualizer']
+
diff --git a/buildstock_query/tools/upgrades_visualizer/figure.py b/buildstock_query/tools/visualizer/figure.py
similarity index 98%
rename from buildstock_query/tools/upgrades_visualizer/figure.py
rename to buildstock_query/tools/visualizer/figure.py
index 9e15e964..23553bc0 100644
--- a/buildstock_query/tools/upgrades_visualizer/figure.py
+++ b/buildstock_query/tools/visualizer/figure.py
@@ -1,5 +1,5 @@
-from buildstock_query.tools.upgrades_visualizer.plot_utils import PlotParams, ValueTypes, human_sort_key, flatten_list
-from buildstock_query.tools.upgrades_visualizer.viz_data import VizData
+from buildstock_query.tools.visualizer.plot_utils import PlotParams, ValueTypes, human_sort_key, flatten_list
+from buildstock_query.tools.visualizer.viz_data import VizData
import plotly.graph_objects as go
import polars as pl
import re
diff --git a/buildstock_query/tools/upgrades_visualizer/plot_utils.py b/buildstock_query/tools/visualizer/plot_utils.py
similarity index 100%
rename from buildstock_query/tools/upgrades_visualizer/plot_utils.py
rename to buildstock_query/tools/visualizer/plot_utils.py
diff --git a/buildstock_query/tools/visualizer/timeseries_visualizer.py b/buildstock_query/tools/visualizer/timeseries_visualizer.py
new file mode 100644
index 00000000..039e9f38
--- /dev/null
+++ b/buildstock_query/tools/visualizer/timeseries_visualizer.py
@@ -0,0 +1,869 @@
+from functools import reduce
+import re
+import os
+os.environ['POLARS_MAX_THREADS'] = '4'
+from collections import defaultdict, Counter
+import dash_bootstrap_components as dbc
+from dash import html, ALL, dcc, ctx, Dash, dcc, html, Input, Output, ALL, MATCH, Patch, callback
+import dash
+from dash.dependencies import Input, Output, State
+from dash.exceptions import PreventUpdate
+from dash_iconify import DashIconify
+from InquirerPy import inquirer
+from buildstock_query.tools.visualizer.viz_data import VizData
+from buildstock_query.tools.visualizer.plot_utils import PlotParams, ValueTypes, SavingsTypes
+from buildstock_query.tools.visualizer.figure import UpgradesPlot
+from buildstock_query.helpers import load_script_defaults, save_script_defaults
+import polars as pl
+import pandas as pd
+from typing import Literal
+from buildstock_query.tools.visualizer.viz_utils import filter_cols, get_viz_data
+import json
+from sqlalchemy.sql import sqltypes
+import datetime
+import time
+import diskcache
+cache = diskcache.Cache("./diskcache")
+from dash import Dash, DiskcacheManager, CeleryManager, Input, Output, html, callback, set_props
+from pathlib import Path
+global metadata_df
+background_callback_manager = DiskcacheManager(cache)
+
+MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
+DAYS = [str(i) for i in range(1, 32)]
+
+# Add these styles at the top after imports
+CARD_STYLE = {
+ 'padding': '20px',
+ 'margin': '10px 0',
+ 'border-radius': '8px',
+ 'box-shadow': '0 2px 4px rgba(0,0,0,0.1)',
+ 'background-color': 'white'
+}
+
+SECTION_STYLE = {
+ 'margin-bottom': '20px'
+}
+
+cur_file_path = Path(__file__).resolve()
+# Add after imports
+COST_FILE = cur_file_path.parent / "costs.json"
+
+def load_costs():
+ """Load costs from JSON file"""
+ if COST_FILE.exists():
+ try:
+ with open(COST_FILE, 'r') as f:
+ return defaultdict(lambda: {'dollars': 0, 'gb': 0}, json.load(f))
+ except json.JSONDecodeError:
+ print(f"Warning: Could not parse {COST_FILE}, starting with empty costs")
+ return defaultdict(lambda: {'dollars': 0, 'gb': 0})
+
+def save_costs(costs):
+ """Save costs to JSON file"""
+ try:
+ with open(COST_FILE, 'w') as f:
+ json.dump(dict(costs), f)
+ except Exception as e:
+ print(f"Warning: Failed to save costs to {COST_FILE}: {e}")
+
+# Initialize cost tracker from file
+cost_tracker = load_costs()
+
+def get_cost_key(db_name, table_name):
+ """Generate a unique key for cost tracking based on db and table names"""
+ if isinstance(table_name, list):
+ table_name = '_'.join(sorted(table_name))
+ return f"{db_name}_{table_name}"
+
+def get_app(viz_data: VizData):
+ """Creates and returns the Dash application.
+
+ Args:
+ timeseries_data: Data source for timeseries values
+ char_data: Data source for building characteristics
+ """
+ cost_key = get_cost_key(viz_data.main_run.db_name, viz_data.main_run.table_name)
+
+ app = Dash(__name__,
+ external_stylesheets=[dbc.themes.BOOTSTRAP],
+ )
+ viz_data.init_metadata_df()
+ timeseries_columns = [col.name for col in viz_data.main_run.ts_table.columns
+ ]
+ build_cols = viz_data.metadata_df.columns
+ char_cols = [c.removeprefix(viz_data.main_run._char_prefix) for c in build_cols if 'applicable' not in c]
+ app.layout = dbc.Container([
+ # Header with spinner in the middle
+ dbc.Row([
+ dbc.Col(html.H2("Timeseries Visualizer", className="display-20"), width=4),
+ dbc.Col(
+ dcc.Loading(
+ id="loading-spinner",
+ children=html.Div(id="loading-output"),
+ fullscreen=False,
+ ),
+ width=2,
+ className="d-flex justify-content-center align-items-center"
+ ),
+ dbc.Col(html.Plaintext(id='update-status', className="text-muted"), width=2),
+ dbc.Col(
+ html.Div(
+ id='cost-display',
+ className="text-muted",
+ style={'textAlign': 'right', 'color': '#999'}
+ ),
+ width=4
+ ),
+ ], className="mb-0 align-items-center"), # Main Graph with card styling
+ dbc.Card([
+ dbc.CardBody([
+ dcc.Graph(id='main-graph')
+ ])
+ ], style=CARD_STYLE),
+
+ # Controls Section
+ dbc.Card([
+ dbc.CardBody([
+ # Top row - Plot Controls with Update button
+ dbc.Row([
+ dbc.Col([
+ html.Label("Upgrade:", className="mb-1", style={'fontSize': '14px'}),
+ dcc.Dropdown(
+ id='upgrade-dropdown',
+ options=list(viz_data.upgrade2shortname.keys()),
+ value=list(viz_data.upgrade2shortname.keys())[0] if viz_data.upgrade2shortname else None,
+ multi=True,
+ placeholder="Which Upgrade to Plot"
+ )
+ ], width=2),
+ dbc.Col([
+ html.Label("Columns to plot:", className="mb-1", style={'fontSize': '14px'}),
+ dcc.Dropdown(
+ id='plot-dropdown',
+ options=timeseries_columns,
+ value=timeseries_columns[20] if timeseries_columns else None,
+ multi=True,
+ placeholder="Select metrics to plot"
+ )
+ ], width=True),
+ dbc.Col([
+ dbc.Row([
+ dbc.Col(dbc.Button(
+ "Update",
+ id="update-plot-button",
+ color="primary",
+ className="w-100",
+ style={'minWidth': '120px'}
+ ), width="auto"),
+ dbc.Col(dbc.Button(
+ "Cancel",
+ id="cancel-button",
+ color="danger",
+ className="w-100",
+ style={'minWidth': '120px'},
+ disabled=True
+ ), width="auto")
+ ], className="g-2 h-100 align-items-end justify-content-end")
+ ], width="auto", className="d-flex align-items-end")
+ ], className="mb-3 g-2 align-items-end"),
+
+ # New split layout for aggregation controls
+ dbc.Row([
+ # Left Column - Building Aggregation
+ dbc.Col([
+ dbc.Card([
+ dbc.CardBody([
+ html.H5("Aggregate across buildings", className="mb-3"),
+ dbc.Row([
+ dbc.Col([
+ html.Label("Aggregation type:", className="mb-1", style={'fontSize': '14px'}),
+ dcc.Dropdown(
+ id='building-agg-dropdown',
+ options=[
+ {'label': x.capitalize(), 'value': x}
+ for x in ['sum', 'avg', 'max', 'min']
+ ],
+ value=['avg'],
+ multi=True
+ )
+ ])
+ ], className="mb-3"),
+
+ # Filters section
+ html.Label("Filters:", className="mb-1", style={'fontSize': '14px'}),
+ html.Div(id='restrictions-container', children=[], className="mb-2"),
+ dbc.Button(
+ [DashIconify(icon="mdi:plus", className="me-1"), "Add Filter"],
+ id="add-restriction-btn",
+ color="primary",
+ size="sm"
+ )
+ ])
+ ], className="h-100")
+ ], width=6),
+
+ # Right Column - Time Aggregation
+ dbc.Col([
+ dbc.Card([
+ dbc.CardBody([
+ html.H5("Aggregate across time", className="mb-3"),
+ dbc.Row([
+ dbc.Col([
+ html.Label("Shape type:", className="mb-1", style={'fontSize': '14px'}),
+ dcc.Dropdown(
+ id='shape-type-dropdown',
+ options=[
+ {'label': "Daily Shape", "value": "daily_shape"},
+ {'label': "Weekend Shape", "value": "weekend_shape"},
+ {'label': "Weekday Shape", "value": "weekday_shape"}
+ ],
+ value=[],
+ multi=True,
+ placeholder="Full year (no aggregation)"
+ )
+ ], width=4),
+ dbc.Col([
+ html.Label("Aggregation type:", className="mb-1", style={'fontSize': '14px'}),
+ dcc.Dropdown(
+ id='time-agg-dropdown',
+ options=[
+ {'label': x.capitalize(), 'value': x}
+ for x in ['avg', 'max', 'min']
+ ],
+ value=['avg'],
+ multi=True
+ )
+ ], width=4),
+ dbc.Col([
+ html.Label("Resolution:", className="mb-1", style={'fontSize': '14px'}),
+ dcc.Dropdown(
+ id='resolution-dropdown',
+ options=[], # Will be populated by callback
+ value=None
+ )
+ ], width=4)
+ ], className="mb-3"),
+
+ # Time range controls
+ dbc.Row([
+ dbc.Col([
+ html.Label("Start:", className="mb-1", style={'fontSize': '14px'}),
+ dbc.Row([
+ dbc.Col(dcc.Dropdown(
+ id='start-month-dropdown',
+ options=[{'label': m, 'value': m} for m in MONTHS],
+ value='Jan'
+ ), width=6),
+ dbc.Col(dcc.Dropdown(
+ id='start-day-dropdown',
+ options=[{'label': d, 'value': d} for d in DAYS],
+ value='1'
+ ), width=6)
+ ], className="g-1")
+ ], width=True),
+ dbc.Col([
+ html.Label("End:", className="mb-1", style={'fontSize': '14px'}),
+ dbc.Row([
+ dbc.Col(dcc.Dropdown(
+ id='end-month-dropdown',
+ options=[{'label': m, 'value': m} for m in MONTHS],
+ value='Dec'
+ ), width=6),
+ dbc.Col(dcc.Dropdown(
+ id='end-day-dropdown',
+ options=[{'label': d, 'value': d} for d in DAYS],
+ value='31'
+ ), width=6)
+ ], className="g-1")
+ ], width=True),
+ dbc.Col([
+ html.Label("Season:", className="mb-1", style={'fontSize': '14px'}),
+ dbc.Row([
+ dbc.Col(
+ dbc.Button(
+ "Summer",
+ id="summer-btn",
+ color="info",
+ size="sm",
+ className="w-100",
+ style={'minWidth': '80px'}
+ ),
+ width=4
+ ),
+ dbc.Col(
+ dbc.Button(
+ "Winter",
+ id="winter-btn",
+ color="info",
+ size="sm",
+ className="w-100",
+ style={'minWidth': '80px'}
+ ),
+ width=4
+ ),
+ dbc.Col(
+ dbc.Button(
+ "Full Year",
+ id="full-year-btn",
+ color="info",
+ size="sm",
+ className="w-100",
+ style={'minWidth': '80px'}
+ ),
+ width=4
+ )
+ ], className="g-1"),
+ ], width='auto')
+ ], className="g-2")
+ ])
+ ], className="h-100")
+ ], width=6)
+ ])
+ ], className="p-3")
+ ], style=CARD_STYLE),
+ # Keep the restrictions store
+ dcc.Store(id='restrictions-store', data={'count': 0}),
+
+ ], fluid=True, className="py-4", style={'background-color': '#f8f9fa'})
+
+ @app.callback(
+ Output('restrictions-container', 'children'),
+ Output('restrictions-store', 'data'),
+ Input('add-restriction-btn', 'n_clicks'),
+ Input({'type': 'remove-restriction', 'index': ALL}, 'n_clicks'),
+ State('restrictions-store', 'data'),
+ State('restrictions-container', 'children'),
+ prevent_initial_call=True
+ )
+ def manage_restrictions(add_clicks, remove_clicks, store_data, current_children):
+ if not dash.callback_context.triggered:
+ raise PreventUpdate
+ triggered_id = dash.callback_context.triggered[0]['prop_id']
+ # Initialize current_children and store_data if None
+ if current_children is None:
+ current_children = []
+ if store_data is None:
+ store_data = {'count': 0}
+
+ if triggered_id == 'add-restriction-btn.n_clicks':
+ new_index = store_data['count']
+ store_data['count'] += 1
+ new_restriction = get_restriction_row(new_index)
+ current_children.append(new_restriction)
+
+ elif 'remove-restriction' in triggered_id:
+ triggered_index = json.loads(triggered_id.split('.')[0])['index']
+ current_children = [
+ child for child in current_children
+ if child['props']['id'].get('index') != triggered_index
+ ]
+
+ return current_children, store_data
+
+ def get_restriction_row(new_index):
+ new_restriction = dbc.Row(
+ id={'type': 'restriction-row', 'index': new_index},
+ children=[
+ dbc.Col(
+ [
+ dcc.Dropdown(
+ id={'type': 'char-name', 'index': new_index},
+ options=char_cols,
+ placeholder="Select characteristic"
+ )
+ ], width=5),
+ dbc.Col([
+ dcc.Dropdown(
+ id={'type': 'char-value', 'index': new_index},
+ multi=True,
+ placeholder="Select values"
+ )
+ ], width=6),
+ dbc.Col([
+ dbc.Button(
+ DashIconify(icon="mdi:close"),
+ id={'type': 'remove-restriction', 'index': new_index},
+ color="danger",
+ size="sm",
+ style={'border-radius': '50%'}
+ )
+ ], width=1, className="d-flex align-items-center")
+ ], className="mb-2 g-2")
+
+ return new_restriction
+
+
+ @app.callback(
+ Output({'type': 'char-value', 'index': ALL}, 'options'),
+ Output({'type': 'char-value', 'index': ALL}, 'value'),
+ Input({'type': 'char-name', 'index': ALL}, 'value'),
+ Input({'type': 'char-value', 'index': ALL}, 'value'),
+ State({'type': 'char-value', 'index': ALL}, 'id'),
+ prevent_initial_call=False
+ )
+ def update_char_values(all_char_names, all_char_values, all_ids):
+ """Update characteristic value options and selections based on filter chain."""
+ if not is_valid_input(all_ids, all_char_names, viz_data.metadata_df):
+ return [[]] * len(all_ids), [None] * len(all_ids)
+
+ try:
+ return process_all_rows(all_char_names, all_char_values, all_ids, viz_data.metadata_df)
+ except Exception as e:
+ print(f"Unexpected error in update_char_values: {str(e)}")
+ return [[]] * len(all_ids), [None] * len(all_ids)
+
+ def is_valid_input(all_ids, all_char_names, metadata_df):
+ """Validate input parameters and data availability."""
+ if not all_ids:
+ return False
+ if not all_char_names:
+ return False
+ if metadata_df is None or metadata_df.is_empty():
+ print("Warning: Metadata DataFrame is empty or not initialized")
+ return False
+ return True
+
+ def process_all_rows(all_char_names, all_char_values, all_ids, metadata_df):
+ """Process each filter row to generate options and validate values."""
+ all_options = []
+ new_values = []
+
+ for i, (current_char_name, current_values, _) in enumerate(zip(all_char_names, all_char_values, all_ids)):
+ if not current_char_name:
+ all_options.append([])
+ new_values.append(None)
+ continue
+
+ try:
+ filtered_df = apply_preceding_filters(metadata_df, all_char_names, all_char_values, i)
+ if filtered_df.is_empty():
+ all_options.append([])
+ new_values.append(None)
+ continue
+
+ options, new_value = process_single_row(filtered_df, current_char_name, current_values)
+ all_options.append(options)
+ new_values.append(new_value)
+
+ except Exception as e:
+ print(f"Error processing row {i}: {str(e)}")
+ all_options.append([])
+ new_values.append(None)
+
+ return all_options, new_values
+
+ def apply_preceding_filters(df, all_char_names, all_char_values, current_index):
+ """Apply filters from rows preceding the current row."""
+ filtered_df = df
+ for j, (char_name, char_values) in enumerate(zip(all_char_names, all_char_values)):
+ if (char_name and char_values and j < current_index and char_name in filtered_df.columns):
+ filtered_df = filtered_df.filter(
+ pl.col(char_name).cast(pl.Utf8).is_in([str(v) for v in char_values])
+ )
+ return filtered_df
+
+ def process_single_row(filtered_df, current_char_name, current_values):
+ """Process a single row to generate options and validate current values."""
+ value_counts = (filtered_df.select(pl.col(current_char_name))
+ .filter(pl.col(current_char_name).is_not_null())
+ .group_by(current_char_name)
+ .count()
+ .sort(current_char_name)
+ .to_dict(as_series=False))
+ available_values = {str(v) for v in value_counts[current_char_name] if v is not None}
+
+ options = [
+ {
+ 'label': f"{str(value)} ({count:,})",
+ 'value': str(value),
+ 'title': f"{str(value)} - {count:,} buildings"
+ }
+ for value, count in zip(value_counts[current_char_name], value_counts['count'])
+ ]
+
+ new_value = None
+ if current_values:
+ current_values_str = [str(v) for v in current_values]
+ valid_values = [v for v in current_values_str if v in available_values]
+ new_value = valid_values if valid_values else None
+
+ return options, new_value
+
+
+ @app.callback(
+ Output('resolution-dropdown', 'options'),
+ Output('resolution-dropdown', 'value'),
+ Output('time-agg-dropdown', 'disabled'),
+ Input('shape-type-dropdown', 'value')
+ )
+ def update_resolution_options(shape_type):
+ shape_types = ['daily_shape', 'weekend_shape', 'weekday_shape']
+ has_shape_type = any(shape in shape_type for shape in shape_types)
+
+ if not shape_type:
+ return ['15min', 'hour', 'day', 'week', 'month'], '15min', True
+ else:
+ return ['15min', 'hour'], '15min', False
+
+ @app.callback(
+ Output('loading-output', 'children'),
+ Input('update-plot-button', 'n_clicks'),
+ [State('upgrade-dropdown', 'value'),
+ State('plot-dropdown', 'value'),
+ State('building-agg-dropdown', 'value'),
+ State('time-agg-dropdown', 'value'),
+ State('shape-type-dropdown', 'value'),
+ State('resolution-dropdown', 'value'),
+ State('start-month-dropdown', 'value'),
+ State('start-day-dropdown', 'value'),
+ State('end-month-dropdown', 'value'),
+ State('end-day-dropdown', 'value'),
+ State({'type': 'char-name', 'index': ALL}, 'value'),
+ State({'type': 'char-value', 'index': ALL}, 'value'),
+ State('main-graph', 'figure')],
+ prevent_initial_call=True,
+ background=True,
+ manager=background_callback_manager,
+ running=[
+ (Output('update-plot-button', 'disabled'), True, False),
+ (Output('cancel-button', 'disabled'), False, True),
+ ],
+ cancel=[Input('cancel-button', 'n_clicks')]
+ )
+ def update_plot(n_clicks, upgrades, plot_cols, building_aggs, time_aggs,
+ shape_types, resolution, start_month, start_day,
+ end_month, end_day, filter_names, filter_values, current_figure):
+ print(f"update_plot called with n_clicks: {n_clicks}")
+ if not n_clicks or not plot_cols or upgrades is None:
+ raise PreventUpdate
+
+ # Ensure all inputs are lists
+ upgrades = [upgrades] if not isinstance(upgrades, list) else upgrades
+ plot_cols = [plot_cols] if not isinstance(plot_cols, list) else plot_cols
+ building_aggs = [building_aggs] if not isinstance(building_aggs, list) else building_aggs
+ time_aggs = [time_aggs] if not isinstance(time_aggs, list) else time_aggs
+ shape_types = shape_types or [None]
+ shape_types = [shape_types] if not isinstance(shape_types, list) else shape_types
+ traces = []
+ total_plots = len(upgrades) * len(plot_cols) * len(building_aggs) * len(time_aggs) * len(shape_types)
+ completed = 0
+
+ # Initialize the figure structure, preserving existing layout settings if they exist
+ figure = {
+ 'data': [],
+ 'layout': current_figure.get('layout', {}) if current_figure else {
+ 'title': 'Energy Usage Over Time',
+ 'xaxis': {'title': 'Time'},
+ 'yaxis': {'title': 'Energy Usage'},
+ 'showlegend': True,
+ 'legend': {'orientation': 'h', 'y': -0.2},
+ 'margin': {'l': 60, 'r': 20, 't': 40, 'b': 60}
+ }
+ }
+
+ # Preserve any existing axis ranges if they exist
+ if current_figure and 'layout' in current_figure:
+ for axis in ['xaxis', 'yaxis']:
+ if axis in current_figure['layout']:
+ # Preserve range, autorange, and other axis settings
+ if 'range' in current_figure['layout'][axis]:
+ figure['layout'][axis]['range'] = current_figure['layout'][axis]['range']
+ if 'autorange' in current_figure['layout'][axis]:
+ figure['layout'][axis]['autorange'] = current_figure['layout'][axis]['autorange']
+
+ total_added_cost = 0
+ total_added_gb = 0
+ month_to_num = {month: idx for idx, month in enumerate(MONTHS, 1)}
+ unit_to_axis = {}
+ next_axis = 1
+ num_plots = 0
+ for upgrade in upgrades:
+ for enduse in plot_cols:
+ for building_agg in building_aggs:
+ for time_agg in time_aggs:
+ for shape_type in shape_types:
+ unit = enduse.split('__')[-1] if '__' in enduse else 'unknown'
+ if unit not in unit_to_axis:
+ if not unit_to_axis: # First unit uses primary axis
+ unit_to_axis[unit] = '' # Empty string for primary axis
+ figure['layout']['yaxis']['title'] = unit
+ else:
+ axis_name = f'yaxis{next_axis + 1}'
+ unit_to_axis[unit] = next_axis + 1
+ # Add new axis to layout
+ figure['layout'][axis_name] = {
+ 'title': unit,
+ 'side': 'right' if next_axis % 2 else 'left',
+ 'overlaying': 'y',
+ 'position': 1 + (0.08 * (next_axis // 2)) # Offset each pair of axes
+ }
+ next_axis += 1
+ # Update status
+ shape_agg_str = '' if not shape_type else f"-{shape_type}_{time_agg}"
+ name = f"Upgrade {upgrade} - {enduse} ({building_agg}{shape_agg_str})"
+ status = f"Processing {completed + 1}/{total_plots}: {name}"
+ print(status)
+ run_obj = viz_data.run_obj(upgrade)
+ run_obj.load_cache()
+ def get_restric_char_values(char, values):
+ if char in ['building_id']:
+ values = [int(v) for v in values]
+ return (f"{char}", values)
+
+ restrict = [get_restric_char_values(char, values) for char, values
+ in zip(filter_names, filter_values)
+ if values is not None and char is not None]
+
+ pre_dollars = run_obj.execution_cost.get('Dollars', 0)
+ pre_gb = run_obj.execution_cost.get('GB', 0)
+
+ df =run_obj.agg.aggregate_timeseries(
+ enduses=[enduse],
+ upgrade_id=upgrade,
+ restrict=restrict,
+ agg_func=building_agg,
+ )
+ viz_data.run_obj(upgrade).save_cache()
+ df = df.rename(columns={'timestamp': 'time'})
+
+ # Capture costs after query and calculate difference
+ post_dollars = run_obj.execution_cost.get('Dollars', 0)
+ post_gb = run_obj.execution_cost.get('GB', 0)
+ total_added_cost += post_dollars - pre_dollars
+ total_added_gb += post_gb - pre_gb
+
+ # Update server-side cost tracker
+ cost_tracker[cost_key]['dollars'] += total_added_cost
+ cost_tracker[cost_key]['gb'] += total_added_gb
+ # Save costs after each update
+ save_costs(cost_tracker)
+ if building_agg != 'sum':
+ new_col = f"{enduse}__{building_agg}"
+ else:
+ new_col = enduse
+ new_col = new_col.removeprefix(run_obj.db_schema.column_prefix.output)
+
+ df = df.sort_values('time')
+ if df.empty:
+ print(f"No data found for {upgrade} - {enduse}")
+ continue
+ # Convert to period begining
+ sample_count = df['sample_count'].iloc[0] if 'sample_count' in df.columns else 'N/A'
+ first_date = df['time'].iloc[0]
+ first_offset = first_date - datetime.datetime(first_date.year, 1, 1)
+ df['time'] = df['time'].dt.tz_localize(None) - first_offset
+
+ # Convert month names to numbers (1-12)
+ start_month_num = month_to_num[start_month]
+ end_month_num = month_to_num[end_month]
+
+ # Handle year wrapping
+ df = filter_range(df, start_day, end_day, start_month_num, end_month_num)
+
+ freq_map = {
+ 'day': '1D',
+ 'week': '1W',
+ 'hour': '1H',
+ '15min': '15T',
+ 'month': '1M'
+ }
+
+ agg_func = {
+ 'avg': 'mean',
+ 'max': 'max',
+ 'min': 'min',
+ }.get(time_agg, 'sum')
+
+ if enduse.startswith("schedules_"):
+ df = df.groupby(pd.Grouper(key='time', freq=freq_map[resolution])).agg({new_col: 'mean'})
+ else:
+ df = df.groupby(pd.Grouper(key='time', freq=freq_map[resolution])).agg({new_col: 'sum'})
+
+ if shape_type:
+ # Create a base date using the first date from the data
+ base_date = datetime.datetime(first_date.year, 1, 1)
+ if resolution == '15min':
+ periods = 96 # 24 hours * 4 periods per hour
+ freq = '15T'
+ else: # hourly
+ periods = 24
+ freq = 'H'
+ time_index = pd.date_range(base_date, periods=periods, freq=freq)
+ if shape_type == 'weekend_shape':
+ mask = df.index.weekday.isin([5, 6])
+ elif shape_type == 'weekday_shape':
+ mask = df.index.weekday.isin([0, 1, 2, 3, 4])
+ else: # daily_shape
+ mask = slice(None)
+ df = df[mask]
+ group_cols = [df.index.hour, df.index.minute] if resolution == '15min' else [df.index.hour]
+ df = df.groupby(group_cols).agg({new_col: agg_func})
+ df = pd.DataFrame({new_col: df[new_col].values}, index=time_index)
+
+ new_trace = {
+ 'x': df.index.tolist(),
+ 'y': df[new_col].tolist(),
+ 'name': name,
+ 'type': 'scatter',
+ 'mode': 'lines',
+ 'yaxis': f'y{unit_to_axis[unit]}' if unit_to_axis[unit] else 'y',
+ 'hovertemplate': f'Value: %{{y:,.2f}}
Time: %{{x}}
Sample Count: {sample_count}'
+ }
+
+ traces.append(new_trace)
+ completed += 1
+ num_plots += 1
+
+ # When updating the graph progressively
+ figure['data'] = traces
+ if not current_figure: # Only update title if there's no existing figure
+ figure['layout']['title'] = f"Completed {completed}/{total_plots}"
+ set_props('main-graph', {"figure": figure})
+
+ figure['layout']['title'] = f"Completed all {total_plots} plots."
+ # expand fig height based on number of plots
+ figure['layout']['height'] = 600 + (num_plots * 20)
+ set_props('main-graph', {"figure": figure, "config": {'edits': {"titleText": True, "axisTitleText": True, 'legendText': True}}})
+ return ""
+
+ def filter_range(df, start_day, end_day, start_month_num, end_month_num):
+ if start_month_num > end_month_num:
+ df = df[
+ # Include months after start_month
+ ((df['time'].dt.month >= start_month_num) &
+ ((df['time'].dt.month > start_month_num) |
+ (df['time'].dt.day >= int(start_day)))) |
+ # Include months before end_month
+ ((df['time'].dt.month <= end_month_num) &
+ ((df['time'].dt.month < end_month_num) |
+ (df['time'].dt.day <= int(end_day))))
+ ]
+ else:
+ df = df[
+ # Standard case when start month is before end month
+ ((df['time'].dt.month > start_month_num) &
+ (df['time'].dt.month < end_month_num)) |
+ ((df['time'].dt.month == start_month_num) &
+ (df['time'].dt.day >= int(start_day))) |
+ ((df['time'].dt.month == end_month_num) &
+ (df['time'].dt.day <= int(end_day)))
+ ]
+
+ return df
+
+ @app.callback(
+ Output('cancel-button', 'n_clicks'),
+ Input('cancel-button', 'n_clicks'),
+ prevent_initial_call=True
+ )
+ def cancel_queries(n_clicks):
+ if not n_clicks:
+ raise PreventUpdate
+
+ # Stop all queries for each upgrade
+ for upgrade in viz_data.upgrade2shortname.keys():
+ viz_data.run_obj(str(upgrade)).stop_all_queries()
+
+ return None
+
+ @app.callback(
+ Output('cost-display', 'children'),
+ Input('cost-display', 'data-update-trigger'), # Triggered by updates and also interval
+ Input('interval-component', 'n_intervals'), # Regular refresh
+ )
+ def update_cost_display(_, __):
+ """Update cost display from server-side tracker"""
+ cost_key = get_cost_key(viz_data.main_run.db_name, viz_data.main_run.table_name)
+ loaded_cost = load_costs()
+ cost_tracker.update(loaded_cost)
+ costs = cost_tracker[cost_key]
+ return f"Query Cost: ${costs['dollars']:.3f} ({costs['gb']:.3f} GB)"
+
+ @app.callback(
+ [Output('start-month-dropdown', 'value'),
+ Output('end-month-dropdown', 'value'),
+ Output('start-day-dropdown', 'value'),
+ Output('end-day-dropdown', 'value')],
+ [Input('summer-btn', 'n_clicks'),
+ Input('winter-btn', 'n_clicks'),
+ Input('full-year-btn', 'n_clicks')],
+ prevent_initial_call=True
+ )
+ def update_season_range(summer_clicks, winter_clicks, full_year_clicks):
+ if not dash.callback_context.triggered:
+ raise PreventUpdate
+
+ button_id = dash.callback_context.triggered[0]['prop_id'].split('.')[0]
+
+ if button_id == 'summer-btn':
+ return 'Jun', 'Sep', '1', '30'
+ elif button_id == 'winter-btn':
+ return 'Oct', 'May', '1', '31'
+ elif button_id == 'full-year-btn':
+ return 'Jan', 'Dec', '1', '31'
+
+ raise PreventUpdate
+
+ app.layout.children.append(
+ dcc.Interval(
+ id='interval-component',
+ interval=1000, # 1 second
+ n_intervals=0
+ )
+ )
+ return app
+
+def main():
+ print("Welcome to Upgrades Visualizer.")
+ defaults = load_script_defaults("project_info")
+ # check if variable 'done' exists and is 1
+ debug = False
+ if debug:
+ opt_sat_file = defaults.get("opt_sat_file", "")
+ workgroup = defaults.get("workgroup", "")
+ db_name = defaults.get("db_name", "")
+ table_name = defaults.get("table_name", "")
+ upgrades_selection = defaults.get("upgrades_selection", "")
+ db_schema = defaults.get("db_schema", "resstock_default")
+ else:
+ opt_sat_file = inquirer.text(message="Please enter path to the options saturation csv file:",
+ default=defaults.get("opt_sat_file", "")).execute().strip()
+ workgroup = inquirer.text(message="Please enter Athena workgroup name:",
+ default=defaults.get("workgroup", "")).execute().strip()
+ db_name = inquirer.text(message="Please enter database name "
+ "(found in postprocessing:aws:athena in the buildstock configuration file):",
+ default=defaults.get("db_name", "")).execute().strip()
+ table_name = inquirer.text(message="Please enter table name (same as output folder name; found under "
+ "output_directory in the buildstock configuration file). [Enter two names "
+ "separated by comma if baseline and upgrades are in different run]:",
+ default=defaults.get("table_name", "")
+ ).execute().strip()
+ upgrades_selection = inquirer.text(message="Please enter upgrade ids separated by comma and dashes "
+ "(example: `1-3,5,7,8-9`) or leave empty to include all upgrades.",
+ default=defaults.get("upgrades_selection", "")).execute()
+ db_schema = inquirer.text(message="Please enter database schema (found in db_schema/resstock_oedi_vu.toml):",
+ default=defaults.get("db_schema", "resstock_default")).execute().strip()
+ defaults.update({"opt_sat_file": opt_sat_file, "workgroup": workgroup,
+ "db_name": db_name, "table_name": table_name,
+ "upgrades_selection": upgrades_selection, "db_schema": db_schema})
+ save_script_defaults("project_info", defaults)
+ if ',' in table_name:
+ table_name = table_name.split(',')
+ viz_data = get_viz_data(opt_sat_path=opt_sat_file, db_name=db_name, db_schema=db_schema, table_name=table_name, workgroup=workgroup,
+ buildstock_type='resstock', include_monthly=False,
+ upgrades_selection_str=upgrades_selection, init_query=False)
+ viz_data.init_metadata_df()
+ global metadata_df
+ metadata_df = viz_data.metadata_df.to_pandas()
+ app = get_app(viz_data)
+ # get debug from env variable
+ import os
+ debug = os.getenv("DEBUG", "True").lower() == "true"
+ port = os.getenv("PORT", "8006")
+ print(f"Debug mode: {debug}, port: {port}")
+ app.run(debug=debug, port=port)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/buildstock_query/tools/upgrades_visualizer/upgrades_visualizer.py b/buildstock_query/tools/visualizer/upgrades_visualizer.py
similarity index 94%
rename from buildstock_query/tools/upgrades_visualizer/upgrades_visualizer.py
rename to buildstock_query/tools/visualizer/upgrades_visualizer.py
index e44ad9e2..bfdef2bd 100644
--- a/buildstock_query/tools/upgrades_visualizer/upgrades_visualizer.py
+++ b/buildstock_query/tools/visualizer/upgrades_visualizer.py
@@ -17,13 +17,15 @@
import dash_mantine_components as dmc
from dash_iconify import DashIconify
from InquirerPy import inquirer
-from buildstock_query.tools.upgrades_visualizer.viz_data import VizData
-from buildstock_query.tools.upgrades_visualizer.plot_utils import PlotParams, ValueTypes, SavingsTypes
-from buildstock_query.tools.upgrades_visualizer.figure import UpgradesPlot
+from buildstock_query.tools.visualizer.viz_data import VizData
+from buildstock_query.tools.visualizer.plot_utils import PlotParams, ValueTypes, SavingsTypes
+from buildstock_query.tools.visualizer.figure import UpgradesPlot
from buildstock_query.helpers import load_script_defaults, save_script_defaults
import polars as pl
from typing import Literal
+from buildstock_query.tools.visualizer.viz_utils import filter_cols, get_viz_data
+
# os.chdir("/Users/radhikar/Documents/eulpda/EULP-data-analysis/eulpda/smart_query/")
# from: https://github.com/thedirtyfew/dash-extensions/tree/1b8c6466b5b8522690442713eb421f622a1d7a59
# app = DashProxy(transforms=[
@@ -40,58 +42,6 @@
default_end_use = "fuel_use_electricity_total_m_btu"
-def filter_cols(all_columns, prefixes=[], suffixes=[]):
- cols = []
- for col in all_columns:
- for prefix in prefixes:
- if col.startswith(prefix):
- cols.append(col)
- break
- else:
- for suffix in suffixes:
- if col.endswith(suffix):
- cols.append(col)
- break
- return cols
-
-
-def get_int_set(input_str):
- """
- Convert "1,2,3-6,8,9" to [1, 2, 3, 4, 5, 6, 8, 9]
- """
- if not input_str:
- return set()
-
- pattern = r'^(\d+(-\d+)?,)*(\d+(-\d+)?)$'
- if not re.match(pattern, input_str):
- raise ValueError(f"{input_str} is not a valid pattern for list")
-
- result = set()
- segments = input_str.split(',')
- for segment in segments:
- if '-' in segment:
- start, end = map(int, segment.split('-'))
- result |= set(range(start, end + 1))
- else:
- result.add(int(segment))
-
- return result
-
-
-def _get_app(opt_sat_path: str, db_name: str = 'euss-tests',
- table_name: str = 'res_test_03_2018_10k_20220607',
- workgroup: str = 'largeee',
- buildstock_type: Literal['resstock', 'comstock'] = 'resstock',
- include_monthly: bool = True,
- upgrades_selection_str: str = ''):
- viz_data = VizData(opt_sat_path=opt_sat_path, db_name=db_name,
- run=table_name, workgroup=workgroup, buildstock_type=buildstock_type,
- include_monthly=include_monthly,
- upgrades_selection=get_int_set(upgrades_selection_str)
- )
- return get_app(viz_data)
-
-
def get_app(viz_data: VizData):
upgrades_plot = UpgradesPlot(viz_data)
upgrade2res = viz_data.upgrade2res
@@ -875,12 +825,10 @@ def main():
save_script_defaults("project_info", defaults)
if ',' in table_name:
table_name = table_name.split(',')
- app = _get_app(opt_sat_path=opt_sat_file,
- workgroup=workgroup,
- db_name=db_name,
- table_name=table_name,
- include_monthly=include_monthly,
- upgrades_selection_str=upgrades_selection)
+ viz_data = get_viz_data(opt_sat_path=opt_sat_file, db_name=db_name, table_name=table_name, workgroup=workgroup,
+ buildstock_type='resstock', include_monthly=include_monthly,
+ upgrades_selection_str=upgrades_selection, init_query=True)
+ app = get_app(viz_data)
app.run_server(debug=False, port=8006)
diff --git a/buildstock_query/tools/upgrades_visualizer/viz_data.py b/buildstock_query/tools/visualizer/viz_data.py
similarity index 94%
rename from buildstock_query/tools/upgrades_visualizer/viz_data.py
rename to buildstock_query/tools/visualizer/viz_data.py
index f2325f7a..96e2c920 100644
--- a/buildstock_query/tools/upgrades_visualizer/viz_data.py
+++ b/buildstock_query/tools/visualizer/viz_data.py
@@ -1,7 +1,7 @@
from buildstock_query import BuildStockQuery, KWH2MBTU
from pydantic import validate_arguments
import polars as pl
-from buildstock_query.tools.upgrades_visualizer.plot_utils import PlotParams
+from buildstock_query.tools.visualizer.plot_utils import PlotParams
from typing import Union
from typing import Literal
import datetime
@@ -16,20 +16,22 @@ class VizData:
@validate_arguments(config=dict(arbitrary_types_allowed=True, smart_union=True))
def __init__(self, opt_sat_path: str,
db_name: str,
+ db_schema: str,
run: Union[str, tuple[str, str]],
workgroup: str = 'largeee',
buildstock_type: Literal['resstock', 'comstock'] = 'resstock',
skip_init: bool = False,
- include_monthly: bool = True,
upgrades_selection: set[str] = set()):
if isinstance(run, tuple):
# Allows for separate baseline and upgrade runs
# In this case, run[0] is the baseline run and run[1] is the upgrade run
self.baseline_run = BuildStockQuery(workgroup=workgroup,
db_name=db_name,
+ db_schema=db_schema,
buildstock_type=buildstock_type,
table_name=run[0],
- skip_reports=skip_init)
+ skip_reports=skip_init,
+ )
baseline_table_name = run[0] + "_baseline"
upgrade_table_name = run[1] + "_upgrades"
ts_table_name = run[1] + "_timeseries"
@@ -40,12 +42,12 @@ def __init__(self, opt_sat_path: str,
table = run
self.main_run = BuildStockQuery(workgroup=workgroup,
db_name=db_name,
+ db_schema=db_schema,
buildstock_type=buildstock_type,
table_name=table,
skip_reports=skip_init)
self.opt_sat_path = opt_sat_path
self.upgrades_selection = upgrades_selection
- self.include_monthly = include_monthly
if not skip_init:
self.initialize()
@@ -62,13 +64,13 @@ def initialize(self):
if self.available_upgrades:
upgrade_names = self.main_run.get_upgrade_names()
self.upgrade2name |= upgrade_names
+ self.upgrade2shortname = {0: "Baseline"}
+ self.upgrade2shortname |= {indx+1: f"Upgrade {indx+1}" for indx in range(len(self.available_upgrades))}
+ self.all_upgrade_plotting_df = None
+ self.metadata_df = None
- self.upgrade2shortname = {indx+1: f"Upgrade {indx+1}" for indx in range(len(self.available_upgrades) + 1)}
+ def init_change2bldgs(self):
self.chng2bldg = self.get_change2bldgs()
- self.init_annual_results()
- if self.include_monthly:
- self.init_monthly_results(self.metadata_df)
- self.all_upgrade_plotting_df = None
def run_obj(self, upgrade: str) -> BuildStockQuery:
if upgrade == '0' and self.baseline_run is not None:
@@ -110,9 +112,13 @@ def _get_metadata_df(self):
metadata_df = metadata_df.rename({x: x.split('.')[1] for x in metadata_df.columns if '.' in x})
return metadata_df
+ def init_metadata_df(self):
+ if self.metadata_df is None:
+ self.metadata_df = self._get_metadata_df()
+
def init_annual_results(self):
self.bs_res_df = self._get_results_csv_clean('0')
- self.metadata_df = self._get_metadata_df()
+
self.sample_weight = self.metadata_df['sample_weight'][0]
self.upgrade2res = {'0': self.bs_res_df}
for upgrade in self.available_upgrades:
@@ -128,7 +134,9 @@ def _get_ts_enduse_cols(self, upgrade: str):
enduse_cols = filter(lambda x: x.endswith(('_kbtu', '_kwh', 'lb')), all_cols)
return list(enduse_cols)
- def init_monthly_results(self, metadata_df):
+ def init_monthly_results(self):
+ if self.metadata_df is None:
+ self.metadata_df = self._get_metadata_df()
self.upgrade2res_monthly: dict[str, pl.DataFrame] = {}
for upgrade in ['0'] + self.available_upgrades:
ts_cols = self._get_ts_enduse_cols(upgrade)
@@ -169,7 +177,7 @@ def init_monthly_results(self, metadata_df):
.alias(col.replace("__", "_")))
monthly_df = monthly_df.select(['building_id', 'month'] + modified_cols
+ [pl.lit(upgrade).alias("upgrade")])
- monthly_df = monthly_df.join(metadata_df, on='building_id')
+ monthly_df = monthly_df.join(self.metadata_df, on='building_id')
self.upgrade2res_monthly[upgrade] = monthly_df
def get_values(self,
diff --git a/buildstock_query/tools/visualizer/viz_utils.py b/buildstock_query/tools/visualizer/viz_utils.py
new file mode 100644
index 00000000..59bacff0
--- /dev/null
+++ b/buildstock_query/tools/visualizer/viz_utils.py
@@ -0,0 +1,54 @@
+import re
+
+from buildstock_query.tools.visualizer.viz_data import VizData
+
+
+def filter_cols(all_columns, prefixes=[], suffixes=[]):
+ cols = []
+ for col in all_columns:
+ for prefix in prefixes:
+ if col.startswith(prefix):
+ cols.append(col)
+ break
+ else:
+ for suffix in suffixes:
+ if col.endswith(suffix):
+ cols.append(col)
+ break
+ return cols
+
+
+def get_int_set(input_str):
+ """
+ Convert "1,2,3-6,8,9" to [1, 2, 3, 4, 5, 6, 8, 9]
+ """
+ if not input_str:
+ return set()
+
+ pattern = r'^(\d+(-\d+)?,)*(\d+(-\d+)?)$'
+ if not re.match(pattern, input_str):
+ raise ValueError(f"{input_str} is not a valid pattern for list")
+
+ result = set()
+ segments = input_str.split(',')
+ for segment in segments:
+ if '-' in segment:
+ start, end = map(int, segment.split('-'))
+ result |= set(range(start, end + 1))
+ else:
+ result.add(int(segment))
+
+ return result
+
+
+def get_viz_data(opt_sat_path, db_name, db_schema, table_name, workgroup, buildstock_type, include_monthly, upgrades_selection_str, init_query):
+ viz_data = VizData(opt_sat_path=opt_sat_path, db_name=db_name, db_schema=db_schema,
+ run=table_name, workgroup=workgroup, buildstock_type=buildstock_type,
+ upgrades_selection=get_int_set(upgrades_selection_str)
+ )
+ if init_query:
+ viz_data.init_change2bldgs()
+ viz_data.init_annual_results()
+ if include_monthly:
+ viz_data.init_monthly_results()
+ return viz_data
\ No newline at end of file
diff --git a/example_usage/aggregates_and_savings.ipynb b/example_usage/aggregates_and_savings.ipynb
index 184180bf..e75a7ef5 100644
--- a/example_usage/aggregates_and_savings.ipynb
+++ b/example_usage/aggregates_and_savings.ipynb
@@ -22,20 +22,23 @@
"output_type": "stream",
"text": [
"INFO:buildstock_query.query_core:Loading euss_res_final_2018_550k_20220901 ...\n",
- "INFO:buildstock_query.query_core:12 queries cache read from .bsq_cache/euss_res_final_2018_550k_20220901_query_cache.pkl.\n",
- "INFO:buildstock_query.query_core:12 queries cache is updated.\n",
+ "INFO:botocore.tokens:Loading cached SSO token for nrel-sso\n",
+ "INFO:buildstock_query.query_core:21 queries cache read from None.\n",
+ "INFO:buildstock_query.query_core:21 queries cache is updated.\n",
"INFO:buildstock_query.main:Getting Success counts...\n",
- "INFO:buildstock_query.report_query:Checking integrity with ts_tables ...\n"
+ "INFO:buildstock_query.report_query:Checking integrity with ts_tables ...\n",
+ "INFO:buildstock_query.query_core:No new queries to save.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- " Fail Unapplicaple Success Sum Applied % no-chng bad-chng \\\n",
+ " fail unapplicable success Sum Applied % no-chng bad-chng \\\n",
"upgrade \n",
"0 1084 0 548916 550000 0.0 0 0 \n",
"1 1 17092 531823 548916 96.9 6097 949 \n",
+ "10 0 0 548916 548916 100.0 1 5310 \n",
"2 2 12117 536797 548916 97.8 6527 879 \n",
"3 0 656 548260 548916 99.9 807 29954 \n",
"4 0 656 548260 548916 99.9 4 20705 \n",
@@ -44,12 +47,12 @@
"7 2 148335 400579 548916 73.0 4169 7603 \n",
"8 3 0 548913 548916 100.0 2 6456 \n",
"9 0 0 548916 548916 100.0 1 5443 \n",
- "10 0 0 548916 548916 100.0 1 5310 \n",
"\n",
" ok-chng true-bad-chng true-ok-chng null any no-chng % \\\n",
"upgrade \n",
"0 0 0 0 0 0 0.0 \n",
"1 524777 277 525480 0 531823 1.1 \n",
+ "10 543605 3039 545876 0 548916 0.0 \n",
"2 529391 354 529940 0 536797 1.2 \n",
"3 517499 19686 527777 0 548260 0.1 \n",
"4 527551 14086 534173 0 548260 0.0 \n",
@@ -58,12 +61,12 @@
"7 388807 4177 392234 0 400579 1.0 \n",
"8 542455 3494 545418 0 548913 0.0 \n",
"9 543472 3102 545813 0 548916 0.0 \n",
- "10 543605 3039 545876 0 548916 0.0 \n",
"\n",
" bad-chng % ok-chng % true-ok-chng % true-bad-chng % \n",
"upgrade \n",
"0 0.0 0.0 0.0 0.0 \n",
"1 0.2 98.7 98.8 0.1 \n",
+ "10 1.0 99.0 99.4 0.6 \n",
"2 0.2 98.6 98.7 0.1 \n",
"3 5.5 94.4 96.3 3.6 \n",
"4 3.8 96.2 97.4 2.6 \n",
@@ -72,29 +75,21 @@
"7 1.9 97.1 97.9 1.0 \n",
"8 1.2 98.8 99.4 0.6 \n",
"9 1.0 99.0 99.4 0.6 \n",
- "10 1.0 99.0 99.4 0.6 \n",
"\u001b[92mAnnual and timeseries tables are verified to have the same number of buildings.\u001b[0m\n",
"\u001b[92mAll buildings are verified to have the same number of (35040) timeseries rows.\u001b[0m\n"
]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "INFO:buildstock_query.query_core:12 queries cache saved to .bsq_cache/euss_res_final_2018_550k_20220901_query_cache.pkl\n"
- ]
}
],
"source": [
"my_run = BuildStockQuery(db_name='euss-final',\n",
" table_name='euss_res_final_2018_550k_20220901',\n",
- " workgroup='largeee',\n",
+ " workgroup='rescore',\n",
" buildstock_type='resstock')"
]
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 3,
"id": "708912ad",
"metadata": {},
"outputs": [
@@ -129,37 +124,37 @@
" \n",
" | 0 | \n",
" Mobile Home | \n",
- " 703 | \n",
- " 9.361996e+06 | \n",
- " 4.506122e+08 | \n",
+ " 34491 | \n",
+ " 8.351341e+06 | \n",
+ " 1.399963e+06 | \n",
"
\n",
" \n",
" | 1 | \n",
" Multi-Family with 2 - 4 Units | \n",
- " 446 | \n",
- " 5.939474e+06 | \n",
- " 1.385683e+08 | \n",
+ " 43868 | \n",
+ " 1.062180e+07 | \n",
+ " 1.215178e+06 | \n",
"
\n",
" \n",
" | 2 | \n",
" Multi-Family with 5+ Units | \n",
- " 1189 | \n",
- " 1.583416e+07 | \n",
- " 4.099084e+08 | \n",
+ " 100345 | \n",
+ " 2.429664e+07 | \n",
+ " 2.494200e+06 | \n",
"
\n",
" \n",
" | 3 | \n",
" Single-Family Attached | \n",
- " 530 | \n",
- " 7.058119e+06 | \n",
- " 2.094629e+08 | \n",
+ " 32125 | \n",
+ " 7.778459e+06 | \n",
+ " 1.039893e+06 | \n",
"
\n",
" \n",
" | 4 | \n",
" Single-Family Detached | \n",
- " 7121 | \n",
- " 9.483182e+07 | \n",
- " 4.413546e+09 | \n",
+ " 338087 | \n",
+ " 8.186135e+07 | \n",
+ " 1.571373e+07 | \n",
"
\n",
" \n",
"\n",
@@ -167,21 +162,21 @@
],
"text/plain": [
" geometry_building_type_recs sample_count units_count \\\n",
- "0 Mobile Home 703 9.361996e+06 \n",
- "1 Multi-Family with 2 - 4 Units 446 5.939474e+06 \n",
- "2 Multi-Family with 5+ Units 1189 1.583416e+07 \n",
- "3 Single-Family Attached 530 7.058119e+06 \n",
- "4 Single-Family Detached 7121 9.483182e+07 \n",
+ "0 Mobile Home 34491 8.351341e+06 \n",
+ "1 Multi-Family with 2 - 4 Units 43868 1.062180e+07 \n",
+ "2 Multi-Family with 5+ Units 100345 2.429664e+07 \n",
+ "3 Single-Family Attached 32125 7.778459e+06 \n",
+ "4 Single-Family Detached 338087 8.186135e+07 \n",
"\n",
" fuel_use_electricity_total_m_btu \n",
- "0 4.506122e+08 \n",
- "1 1.385683e+08 \n",
- "2 4.099084e+08 \n",
- "3 2.094629e+08 \n",
- "4 4.413546e+09 "
+ "0 1.399963e+06 \n",
+ "1 1.215178e+06 \n",
+ "2 2.494200e+06 \n",
+ "3 1.039893e+06 \n",
+ "4 1.571373e+07 "
]
},
- "execution_count": 5,
+ "execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
@@ -194,17 +189,384 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 4,
+ "id": "1e25b6ea",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "INFO:botocore.tokens:Loading cached SSO token for nrel-sso\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " geometry_building_type_recs | \n",
+ " sample_count | \n",
+ " units_count | \n",
+ " fuel_use_electricity_total_m_btu__max | \n",
+ " fuel_use_electricity_total_m_btu__nonzero_units_count | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " Mobile Home | \n",
+ " 34491 | \n",
+ " 8.351341e+06 | \n",
+ " 635.684 | \n",
+ " 8.351341e+06 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " Multi-Family with 2 - 4 Units | \n",
+ " 43868 | \n",
+ " 1.062180e+07 | \n",
+ " 535.279 | \n",
+ " 1.062180e+07 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " Multi-Family with 5+ Units | \n",
+ " 100345 | \n",
+ " 2.429664e+07 | \n",
+ " 397.650 | \n",
+ " 2.429664e+07 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " Single-Family Attached | \n",
+ " 32125 | \n",
+ " 7.778459e+06 | \n",
+ " 616.660 | \n",
+ " 7.778459e+06 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " Single-Family Detached | \n",
+ " 338087 | \n",
+ " 8.186135e+07 | \n",
+ " 757.359 | \n",
+ " 8.186135e+07 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " geometry_building_type_recs | \n",
+ " sample_count | \n",
+ " units_count | \n",
+ " fuel_use_natural_gas_total_m_btu__my_non_zero_count | \n",
+ " fuel_use_natural_gas_total_m_btu__nonzero_units_count | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " Mobile Home | \n",
+ " 34491 | \n",
+ " 8.351341e+06 | \n",
+ " 3.045282e+06 | \n",
+ " 3.045282e+06 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " Multi-Family with 2 - 4 Units | \n",
+ " 43868 | \n",
+ " 1.062180e+07 | \n",
+ " 7.284269e+06 | \n",
+ " 7.284269e+06 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " Multi-Family with 5+ Units | \n",
+ " 100345 | \n",
+ " 2.429664e+07 | \n",
+ " 1.338089e+07 | \n",
+ " 1.338089e+07 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " Single-Family Attached | \n",
+ " 32125 | \n",
+ " 7.778459e+06 | \n",
+ " 5.766592e+06 | \n",
+ " 5.766592e+06 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " Single-Family Detached | \n",
+ " 338087 | \n",
+ " 8.186135e+07 | \n",
+ " 5.395333e+07 | \n",
+ " 5.395333e+07 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " geometry_building_type_recs | \n",
+ " sample_count | \n",
+ " units_count | \n",
+ " fuel_use_electricity_total_m_btu | \n",
+ " fuel_use_electricity_total_m_btu__nonzero_units_count | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " Mobile Home | \n",
+ " 34491 | \n",
+ " 8.351341e+06 | \n",
+ " 1.399963e+06 | \n",
+ " 8.351341e+06 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " Multi-Family with 2 - 4 Units | \n",
+ " 43868 | \n",
+ " 1.062180e+07 | \n",
+ " 1.215178e+06 | \n",
+ " 1.062180e+07 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " Multi-Family with 5+ Units | \n",
+ " 100345 | \n",
+ " 2.429664e+07 | \n",
+ " 2.494200e+06 | \n",
+ " 2.429664e+07 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " Single-Family Attached | \n",
+ " 32125 | \n",
+ " 7.778459e+06 | \n",
+ " 1.039893e+06 | \n",
+ " 7.778459e+06 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " Single-Family Detached | \n",
+ " 338087 | \n",
+ " 8.186135e+07 | \n",
+ " 1.571373e+07 | \n",
+ " 8.186135e+07 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " geometry_building_type_recs | \n",
- " state | \n",
- " time | \n",
- " sample_count | \n",
- " units_count | \n",
- " rows_per_sample | \n",
- " fuel_use__electricity__total__kwh | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | 0 | \n",
- " Mobile Home | \n",
- " AL | \n",
- " 2018-01-01 | \n",
- " 1233 | \n",
- " 298547.538693 | \n",
- " 2976 | \n",
- " 5.749913e+08 | \n",
- "
\n",
- " \n",
- " | 1 | \n",
- " Mobile Home | \n",
- " AL | \n",
- " 2018-02-01 | \n",
- " 1233 | \n",
- " 298547.538693 | \n",
- " 2688 | \n",
- " 2.702862e+08 | \n",
- "
\n",
- " \n",
- " | 2 | \n",
- " Mobile Home | \n",
- " AL | \n",
- " 2018-03-01 | \n",
- " 1233 | \n",
- " 298547.538693 | \n",
- " 2976 | \n",
- " 2.812449e+08 | \n",
- "
\n",
- " \n",
- " | 3 | \n",
- " Mobile Home | \n",
- " AL | \n",
- " 2018-04-01 | \n",
- " 1233 | \n",
- " 298547.538693 | \n",
- " 2880 | \n",
- " 2.391579e+08 | \n",
- "
\n",
- " \n",
- " | 4 | \n",
- " Mobile Home | \n",
- " AL | \n",
- " 2018-05-01 | \n",
- " 1233 | \n",
- " 298547.538693 | \n",
- " 2976 | \n",
- " 2.993396e+08 | \n",
- "
\n",
- " \n",
- " | ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- "
\n",
- " \n",
- " | 2923 | \n",
- " Single-Family Detached | \n",
- " WY | \n",
- " 2018-08-01 | \n",
- " 725 | \n",
- " 175544.984227 | \n",
- " 2976 | \n",
- " 1.149405e+08 | \n",
- "
\n",
- " \n",
- " | 2924 | \n",
- " Single-Family Detached | \n",
- " WY | \n",
- " 2018-09-01 | \n",
- " 725 | \n",
- " 175544.984227 | \n",
- " 2880 | \n",
- " 1.090155e+08 | \n",
- "
\n",
- " \n",
- " | 2925 | \n",
- " Single-Family Detached | \n",
- " WY | \n",
- " 2018-10-01 | \n",
- " 725 | \n",
- " 175544.984227 | \n",
- " 2976 | \n",
- " 1.492910e+08 | \n",
- "
\n",
- " \n",
- " | 2926 | \n",
- " Single-Family Detached | \n",
- " WY | \n",
- " 2018-11-01 | \n",
- " 725 | \n",
- " 175544.984227 | \n",
- " 2880 | \n",
- " 2.082698e+08 | \n",
- "
\n",
- " \n",
- " | 2927 | \n",
- " Single-Family Detached | \n",
- " WY | \n",
- " 2018-12-01 | \n",
- " 725 | \n",
- " 175544.984227 | \n",
- " 2976 | \n",
- " 2.658665e+08 | \n",
- "
\n",
- " \n",
- "
\n",
- "
2928 rows × 7 columns
\n",
- "
\n",
" | 0 | \n",
" 2018-01-01 00:15:00 | \n",
- " 9989 | \n",
- " 1.330256e+08 | \n",
- " 7.148833e+07 | \n",
- " 1.966400e+07 | \n",
+ " 9105 | \n",
+ " 2.204603e+06 | \n",
+ " 2.344104e+06 | \n",
+ " 798942.753525 | \n",
"
\n",
" \n",
" | 1 | \n",
" 2018-01-01 00:30:00 | \n",
- " 9989 | \n",
- " 1.330256e+08 | \n",
- " 7.227770e+07 | \n",
- " 1.960490e+07 | \n",
+ " 9105 | \n",
+ " 2.204603e+06 | \n",
+ " 2.380520e+06 | \n",
+ " 804806.924523 | \n",
"
\n",
" \n",
" | 2 | \n",
" 2018-01-01 00:45:00 | \n",
- " 9989 | \n",
- " 1.330256e+08 | \n",
- " 7.189331e+07 | \n",
- " 1.947231e+07 | \n",
+ " 9105 | \n",
+ " 2.204603e+06 | \n",
+ " 2.377988e+06 | \n",
+ " 802971.087184 | \n",
"
\n",
" \n",
" | 3 | \n",
" 2018-01-01 01:00:00 | \n",
- " 9989 | \n",
- " 1.330256e+08 | \n",
- " 7.202095e+07 | \n",
- " 1.937339e+07 | \n",
+ " 9105 | \n",
+ " 2.204603e+06 | \n",
+ " 2.370643e+06 | \n",
+ " 801110.794613 | \n",
"
\n",
" \n",
" | 4 | \n",
" 2018-01-01 01:15:00 | \n",
- " 9989 | \n",
- " 1.330256e+08 | \n",
- " 7.083665e+07 | \n",
- " 1.915390e+07 | \n",
+ " 9105 | \n",
+ " 2.204603e+06 | \n",
+ " 2.327645e+06 | \n",
+ " 794221.925170 | \n",
"
\n",
" \n",
" | ... | \n",
@@ -575,42 +925,42 @@
"
\n",
" | 35035 | \n",
" 2018-12-31 23:00:00 | \n",
- " 9989 | \n",
- " 1.330256e+08 | \n",
- " 4.153427e+07 | \n",
- " 5.265636e+06 | \n",
+ " 9105 | \n",
+ " 2.204603e+06 | \n",
+ " 5.392025e+05 | \n",
+ " 35243.137427 | \n",
"
\n",
" \n",
" | 35036 | \n",
" 2018-12-31 23:15:00 | \n",
- " 9989 | \n",
- " 1.330256e+08 | \n",
- " 3.732818e+07 | \n",
- " 5.140162e+06 | \n",
+ " 9105 | \n",
+ " 2.204603e+06 | \n",
+ " 4.739228e+05 | \n",
+ " 36440.475284 | \n",
"
\n",
" \n",
" | 35037 | \n",
" 2018-12-31 23:30:00 | \n",
- " 9989 | \n",
- " 1.330256e+08 | \n",
- " 3.620637e+07 | \n",
- " 5.337922e+06 | \n",
+ " 9105 | \n",
+ " 2.204603e+06 | \n",
+ " 4.598230e+05 | \n",
+ " 39609.485979 | \n",
"
\n",
" \n",
" | 35038 | \n",
" 2018-12-31 23:45:00 | \n",
- " 9989 | \n",
- " 1.330256e+08 | \n",
- " 3.537120e+07 | \n",
- " 5.329013e+06 | \n",
+ " 9105 | \n",
+ " 2.204603e+06 | \n",
+ " 4.500661e+05 | \n",
+ " 41160.092985 | \n",
"
\n",
" \n",
" | 35039 | \n",
" 2019-01-01 00:00:00 | \n",
- " 9989 | \n",
- " 1.330256e+08 | \n",
- " 3.489996e+07 | \n",
- " 5.477100e+06 | \n",
+ " 9105 | \n",
+ " 2.204603e+06 | \n",
+ " 4.322545e+05 | \n",
+ " 43056.705207 | \n",
"
\n",
" \n",
"\n",
@@ -619,54 +969,55 @@
],
"text/plain": [
" time sample_count units_count \\\n",
- "0 2018-01-01 00:15:00 9989 1.330256e+08 \n",
- "1 2018-01-01 00:30:00 9989 1.330256e+08 \n",
- "2 2018-01-01 00:45:00 9989 1.330256e+08 \n",
- "3 2018-01-01 01:00:00 9989 1.330256e+08 \n",
- "4 2018-01-01 01:15:00 9989 1.330256e+08 \n",
+ "0 2018-01-01 00:15:00 9105 2.204603e+06 \n",
+ "1 2018-01-01 00:30:00 9105 2.204603e+06 \n",
+ "2 2018-01-01 00:45:00 9105 2.204603e+06 \n",
+ "3 2018-01-01 01:00:00 9105 2.204603e+06 \n",
+ "4 2018-01-01 01:15:00 9105 2.204603e+06 \n",
"... ... ... ... \n",
- "35035 2018-12-31 23:00:00 9989 1.330256e+08 \n",
- "35036 2018-12-31 23:15:00 9989 1.330256e+08 \n",
- "35037 2018-12-31 23:30:00 9989 1.330256e+08 \n",
- "35038 2018-12-31 23:45:00 9989 1.330256e+08 \n",
- "35039 2019-01-01 00:00:00 9989 1.330256e+08 \n",
+ "35035 2018-12-31 23:00:00 9105 2.204603e+06 \n",
+ "35036 2018-12-31 23:15:00 9105 2.204603e+06 \n",
+ "35037 2018-12-31 23:30:00 9105 2.204603e+06 \n",
+ "35038 2018-12-31 23:45:00 9105 2.204603e+06 \n",
+ "35039 2019-01-01 00:00:00 9105 2.204603e+06 \n",
"\n",
" fuel_use__electricity__total__kwh__baseline \\\n",
- "0 7.148833e+07 \n",
- "1 7.227770e+07 \n",
- "2 7.189331e+07 \n",
- "3 7.202095e+07 \n",
- "4 7.083665e+07 \n",
+ "0 2.344104e+06 \n",
+ "1 2.380520e+06 \n",
+ "2 2.377988e+06 \n",
+ "3 2.370643e+06 \n",
+ "4 2.327645e+06 \n",
"... ... \n",
- "35035 4.153427e+07 \n",
- "35036 3.732818e+07 \n",
- "35037 3.620637e+07 \n",
- "35038 3.537120e+07 \n",
- "35039 3.489996e+07 \n",
+ "35035 5.392025e+05 \n",
+ "35036 4.739228e+05 \n",
+ "35037 4.598230e+05 \n",
+ "35038 4.500661e+05 \n",
+ "35039 4.322545e+05 \n",
"\n",
" fuel_use__electricity__total__kwh__savings \n",
- "0 1.966400e+07 \n",
- "1 1.960490e+07 \n",
- "2 1.947231e+07 \n",
- "3 1.937339e+07 \n",
- "4 1.915390e+07 \n",
+ "0 798942.753525 \n",
+ "1 804806.924523 \n",
+ "2 802971.087184 \n",
+ "3 801110.794613 \n",
+ "4 794221.925170 \n",
"... ... \n",
- "35035 5.265636e+06 \n",
- "35036 5.140162e+06 \n",
- "35037 5.337922e+06 \n",
- "35038 5.329013e+06 \n",
- "35039 5.477100e+06 \n",
+ "35035 35243.137427 \n",
+ "35036 36440.475284 \n",
+ "35037 39609.485979 \n",
+ "35038 41160.092985 \n",
+ "35039 43056.705207 \n",
"\n",
"[35040 rows x 5 columns]"
]
},
- "execution_count": 7,
+ "execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"my_run.savings.savings_shape(upgrade_id=2, enduses=['fuel_use__electricity__total__kwh'],\n",
+ " restrict=[('build_existing_model.state', ['AL'])],\n",
" annual_only=False)"
]
},
diff --git a/setup.py b/setup.py
index ff7826e8..c9be901f 100644
--- a/setup.py
+++ b/setup.py
@@ -45,13 +45,14 @@
"dash-mantine-components == 0.10.2",
"dash-iconify >= 0.1.2",
"plotly >= 5.10.0",
- "dash >= 2.6.2",
+ "dash[diskcache] >= 2.18.2",
]
},
entry_points={
'console_scripts': ['upgrades_analyzer=buildstock_query.tools.upgrades_analyzer:main',
- 'upgrades_visualizer=buildstock_query.tools.upgrades_visualizer:main']
+ 'upgrades_visualizer=buildstock_query.tools.visualizer:upgrades_visualizer',
+ 'timeseries_visualizer=buildstock_query.tools.visualizer:timeseries_visualizer']
},
)
diff --git a/tests/generate_reference_viz_data_files.py b/tests/generate_reference_viz_data_files.py
index 52b671c6..073252f7 100644
--- a/tests/generate_reference_viz_data_files.py
+++ b/tests/generate_reference_viz_data_files.py
@@ -1,6 +1,6 @@
from tests.utils import save_ref_pkl
-from buildstock_query.tools.upgrades_visualizer.viz_data import VizData
-from buildstock_query.tools.upgrades_visualizer.plot_utils import PlotParams, SavingsTypes, ValueTypes
+from buildstock_query.tools.visualizer.viz_data import VizData
+from buildstock_query.tools.visualizer.plot_utils import PlotParams, SavingsTypes, ValueTypes
import pathlib
import itertools as it
from buildstock_query import BuildStockQuery
@@ -27,6 +27,9 @@ def save_viz_data_reference_data():
buildstock_type='resstock',
)
assert viz_data.baseline_run is not None
+ viz_data.init_change2bldgs()
+ viz_data.init_annual_results()
+ viz_data.init_monthly_results()
all_cols = viz_data.get_all_end_use_cols("annual")
for resolution, value_type, savings_type in it.product(["annual", "monthly"], ValueTypes, SavingsTypes):
params = PlotParams(enduses=all_cols, savings_type=savings_type, value_type=value_type,
diff --git a/tests/test_BuildStockQuery.py b/tests/test_BuildStockQuery.py
index 6231c260..3c99f822 100644
--- a/tests/test_BuildStockQuery.py
+++ b/tests/test_BuildStockQuery.py
@@ -12,7 +12,7 @@
import time
from typing_extensions import assert_type
from typing import Union
-
+from sqlalchemy import func as safunc
query_core.sa.Table = load_tbl_from_pkl # mock the sqlalchemy table loading
query_core.sa.create_engine = MagicMock() # mock creating engine
query_core.Connection = MagicMock() # type: ignore # NOQA
@@ -261,6 +261,18 @@ def test_aggregate_annual(temp_history_file):
""" # noqa: E501
assert_query_equal(query5, valid_query_string5)
+ # Custom agg_func
+ query6 = my_athena2.agg.aggregate_annual(enduses=enduses,
+ agg_func='max',
+ get_query_only=True,
+ )
+ valid_query_string5 = """
+ select sum(1) as sample_count, sum(29.1) as units_count, max(res_n250_hrly_v1_baseline."report_simulation_output.fuel_use_electricity_net_m_btu" * 1) as fuel_use_electricity_net_m_btu__max,
+ max(res_n250_hrly_v1_baseline."report_simulation_output.end_use_electricity_cooling_m_btu" * 1) as
+ end_use_electricity_cooling_m_btu__max from res_n250_hrly_v1_baseline where res_n250_hrly_v1_baseline.completed_status = 'Success'
+ """ # noqa: E501
+ assert_query_equal(query6, valid_query_string5)
+
def test_get_upgrade_names(temp_history_file):
my_athena = BuildStockQuery(
@@ -448,6 +460,27 @@ def test_aggregate_ts(temp_history_file):
timestamp_grouping_func='month',
get_query_only=True)
assert_query_equal(query9, valid_query_string9)
+ # Test that the agg_func is applied correctly
+
+ def min_func(col):
+ return safunc.min(col)
+
+ query10 = my_athena2.agg.aggregate_timeseries(enduses=enduses,
+ collapse_ts=False,
+ agg_func=min_func,
+ timestamp_grouping_func='month',
+ get_query_only=True)
+ valid_query_string10 = """
+ select date_trunc('month', res_n250_hrly_v1_timeseries.time) AS time,
+ count(distinct(res_n250_hrly_v1_timeseries.building_id)) AS sample_count,
+ (count(distinct(res_n250_hrly_v1_timeseries.building_id)) * sum(29.1)) / sum(1) AS units_count,
+ sum(1) / count(distinct(res_n250_hrly_v1_timeseries.building_id)) AS rows_per_sample,
+ min(res_n250_hrly_v1_timeseries."fuel use: electricity: total" * 1) as
+ "fuel use: electricity: total__min_func", min(res_n250_hrly_v1_timeseries."end use: electricity: cooling" * 1)
+ as "end use: electricity: cooling__min_func" from res_n250_hrly_v1_timeseries join res_n250_hrly_v1_baseline on
+ res_n250_hrly_v1_baseline.building_id = res_n250_hrly_v1_timeseries.building_id group by 1 order by 1
+ """ # noqa: E501
+ assert_query_equal(query10, valid_query_string10)
def test_batch_query(temp_history_file):
diff --git a/tests/test_Viz.py b/tests/test_Viz.py
index e0402787..ed4bdeec 100644
--- a/tests/test_Viz.py
+++ b/tests/test_Viz.py
@@ -1,7 +1,7 @@
-from buildstock_query.tools.upgrades_visualizer.viz_data import VizData
-from buildstock_query.tools.upgrades_visualizer.figure import UpgradesPlot
-from buildstock_query.tools.upgrades_visualizer.plot_utils import PlotParams, SavingsTypes, ValueTypes
-from buildstock_query.tools.upgrades_visualizer.upgrades_visualizer import get_app
+from buildstock_query.tools.visualizer.viz_data import VizData
+from buildstock_query.tools.visualizer.figure import UpgradesPlot
+from buildstock_query.tools.visualizer.plot_utils import PlotParams, SavingsTypes, ValueTypes
+from buildstock_query.tools.visualizer.upgrades_visualizer import get_app
import pathlib
import itertools as it
import pytest
@@ -35,6 +35,9 @@ def viz_data(self):
mydata.baseline_run.load_cache()
mydata.main_run.load_cache()
mydata.initialize()
+ mydata.init_change2bldgs()
+ mydata.init_annual_results()
+ mydata.init_monthly_results()
return mydata
@pytest.fixture(scope='class')