diff --git a/sample_data/boston_floods.json b/sample_data/boston_floods.json index 90656ab4..6e644200 100644 --- a/sample_data/boston_floods.json +++ b/sample_data/boston_floods.json @@ -211,62 +211,76 @@ }, { "type": "Chart", - "name": "Parabolic Hyetograph", - "description": "Rainfall intensity over 24 hours following a parabolic model", + "name": "Charles River Hydrograph (Long Flow Length)", + "description": "Hourly hydrograph for Charles River, using a longer flow length measurement coming from Chester Brook. Uses the following units. Timestep is one hour. Values are volume without units - volume is a proportion of total flood volume. Since these are hourly timesteps, each value can also be thought of as a per-hour rate (volume/hour = discharge).", "project": "Boston Transportation", "files": [ { - "url": "https://data.kitware.com/api/v1/item/67d9a34e429cb34d95af01c5/download", - "hash": "0af8013362c94a3d043c88a277abcbf8e2c44999812f7b8da33dd18e4b22686e", - "path": "boston/parabolic_hyetograph.csv" + "url": "https://data.kitware.com/api/v1/item/68d950f73ba8f1c07a875e68/download", + "hash": "c2c7164b0b9ad8272c225c496e2f17a40a84d0712b00fe6e5aa3f677910cef1f", + "path": "boston/hydrograph_charles_long.csv" } ], "metadata": { - "precipitation_model": "Parabolic" + "method": "NRCS lag equation", + "watershed_slope_percent": 3, + "curve_number": 85, + "flow_length_ft": 124080, + "time_of_concentration_hours": 12.3, + "peak_rate_factor": 484, + "time_to_peak_hours": 7.2, + "attribution": "Calculated by August Posch, September 2025, Northeastern University." }, "chart_options": { - "chart_title": "Rainfall Intensity over 24 hours", + "chart_title": "Proportional Discharge over 24 hours", "x_title": "Hour", - "y_title": "Precipitation rate (mm/hr)" + "y_title": "Proportional discharge rate (volume/hour)" }, "conversion_options": { "labels": "hour", "datasets": [ - "precipitation" + "discharge" ], "palette": { - "precipitation": "blue" + "discharge": "blue" } } }, { "type": "Chart", - "name": "NCRS Type II Hyetograph", - "description": "Rainfall intensity over 24 hours following the NCRS Type II model", + "name": "Charles River Hydrograph (Short Flow Length)", + "description": "Hourly hydrograph for Charles River, using a shorter flow length measurement directly from Waltham gage to mouth. Uses the following units. Timestep is one hour. Values are volume without units - volume is a proportion of total flood volume. Since these are hourly timesteps, each value can also be thought of as a per-hour rate (volume/hour = discharge).", "project": "Boston Transportation", "files": [ { - "url": "https://data.kitware.com/api/v1/item/67d9a34e429cb34d95af01c2/download", - "hash": "aa83acf52af4dcc8e384fda18bcbba5d85d84ffdd30314950e8aacce772111e7", - "path": "boston/ncrs_type_2_hyetograph.csv" + "url": "https://data.kitware.com/api/v1/item/68d950f63ba8f1c07a875e65/download", + "hash": "d0ce5335c53bc2ee80b8762b5c458d06f0cdce8e03cfce935964d365da118c4d", + "path": "boston/hydrograph_charles_short.csv" } ], "metadata": { - "precipitation_model": "NCRS Type II" + "method": "NRCS lag equation", + "watershed_slope_percent": 3, + "curve_number": 85, + "flow_length_ft": 81840, + "time_of_concentration_hours": 8.8, + "peak_rate_factor": 484, + "time_to_peak_hours": 5.2, + "attribution": "Calculated by August Posch, September 2025, Northeastern University." }, "chart_options": { - "chart_title": "Rainfall Intensity over 24 hours", + "chart_title": "Proportional Discharge over 24 hours", "x_title": "Hour", - "y_title": "Precipitation rate (mm/hr)" + "y_title": "Proportional discharge rate (volume/hour)" }, "conversion_options": { "labels": "hour", "datasets": [ - "precipitation" + "discharge" ], "palette": { - "precipitation": "blue" + "discharge": "blue" } } } -] \ No newline at end of file +] diff --git a/sample_data/tests/analytics.json b/sample_data/tests/analytics.json index f4218b76..5dca789e 100644 --- a/sample_data/tests/analytics.json +++ b/sample_data/tests/analytics.json @@ -34,32 +34,39 @@ }, { "type": "Chart", - "name": "Parabolic Hyetograph", - "description": "Rainfall intensity over 24 hours following a parabolic model", + "name": "Charles River Hydrograph (Short Flow Length)", + "description": "Hourly hydrograph for Charles River, using a shorter flow length measurement directly from Waltham gage to mouth. Uses the following units. Timestep is one hour. Values are volume without units - volume is a proportion of total flood volume. Since these are hourly timesteps, each value can also be thought of as a per-hour rate (volume/hour = discharge).", "project": "Boston Transportation", "files": [ { - "url": "https://data.kitware.com/api/v1/item/67d9a34e429cb34d95af01c5/download", - "hash": "0af8013362c94a3d043c88a277abcbf8e2c44999812f7b8da33dd18e4b22686e", - "path": "boston/parabolic_hyetograph.csv" + "url": "https://data.kitware.com/api/v1/item/68d950f63ba8f1c07a875e65/download", + "hash": "d0ce5335c53bc2ee80b8762b5c458d06f0cdce8e03cfce935964d365da118c4d", + "path": "boston/hydrograph_charles_short.csv" } ], "metadata": { - "precipitation_model": "Parabolic" + "method": "NRCS lag equation", + "watershed_slope_percent": 3, + "curve_number": 85, + "flow_length_ft": 81840, + "time_of_concentration_hours": 8.8, + "peak_rate_factor": 484, + "time_to_peak_hours": 5.2, + "attribution": "Calculated by August Posch, September 2025, Northeastern University." }, "chart_options": { - "chart_title": "Rainfall Intensity over 24 hours", + "chart_title": "Proportional Discharge over 24 hours", "x_title": "Hour", - "y_title": "Precipitation rate (mm/hr)" + "y_title": "Proportional discharge rate (volume/hour)" }, "conversion_options": { "labels": "hour", "datasets": [ - "precipitation" + "discharge" ], "palette": { - "precipitation": "blue" + "discharge": "blue" } } } -] \ No newline at end of file +] diff --git a/setup.py b/setup.py index 4f54789d..088925ab 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ 'django-guardian==2.4.0', 'django-oauth-toolkit==2.4.0', 'djangorestframework==3.15.2', - 'django-large-image==0.10.0', + 'django-large-image==0.10.2', 'drf-yasg==1.21.7', 'matplotlib==3.9.2', # for raster colormaps 'geopandas==1.1.1', diff --git a/uvdat/core/rest/analytics.py b/uvdat/core/rest/analytics.py index ee45ce0a..edbe51e1 100644 --- a/uvdat/core/rest/analytics.py +++ b/uvdat/core/rest/analytics.py @@ -1,9 +1,6 @@ -import inspect - from django.db.models import QuerySet from rest_framework.decorators import action from rest_framework.response import Response -from rest_framework.serializers import ModelSerializer from rest_framework.viewsets import ReadOnlyModelViewSet from uvdat.core.models import Project, TaskResult @@ -38,19 +35,8 @@ def list_types(self, request, project_id: int, **kwargs): filtered_queryset = filter_queryset_by_projects( v, Project.objects.filter(id=project_id) ) - queryset_serializer = next( - iter( - s - for name, s in inspect.getmembers(uvdat_serializers, inspect.isclass) - if issubclass(s, ModelSerializer) and s.Meta.model is v.model - ), - None, - ) - if queryset_serializer is None: - v = None - else: - v = [queryset_serializer(o).data for o in filtered_queryset] - else: + v = [dict(id=o.id, name=o.name) for o in filtered_queryset] + elif any(not isinstance(o, dict) for o in v): v = [dict(id=o, name=o) for o in v] filtered_input_options[k] = v serializer = uvdat_serializers.AnalysisTypeSerializer( diff --git a/uvdat/core/tasks/analytics/flood_network_failure.py b/uvdat/core/tasks/analytics/flood_network_failure.py index a63b27e0..839508e8 100644 --- a/uvdat/core/tasks/analytics/flood_network_failure.py +++ b/uvdat/core/tasks/analytics/flood_network_failure.py @@ -108,6 +108,7 @@ def flood_network_failure(result_id): result.save() node_failures = {} + n_nodes = network.nodes.count() flood_dataset_id = flood_sim.outputs.get('flood') flood_layer = Layer.objects.get(dataset__id=flood_dataset_id) @@ -128,24 +129,31 @@ def get_station_region(point): units='EPSG:4326', ) - for frame in flood_layer.frames.all(): - n_nodes = network.nodes.count() - result.write_status( - f'Evaluating flood levels at {n_nodes} nodes for frame {frame.index}...' - ) - raster_path = utilities.field_file_to_local_path( - frame.raster.cloud_optimized_geotiff - ) - source = tilesource.get_tilesource_from_path(raster_path) + # Precompute node regions + node_regions = { + node.id: get_station_region(node.location) for node in network.nodes.all() + } - node_failures[frame.index] = [] - for node in network.nodes.all(): - region = get_station_region(node.location) - region_data, _ = source.getRegion(region=region, format='numpy') - water_heights = numpy.take(region_data, 0, axis=2) - if numpy.any(numpy.where(water_heights > tolerance)): - node_failures[frame.index].append(node.id) + # Assume that all frames in flood_layer refer to frames of the same RasterData + raster = flood_layer.frames.first().raster + raster_path = utilities.field_file_to_local_path(raster.cloud_optimized_geotiff) + source = tilesource.get_tilesource_from_path(raster_path) + metadata = source.getMetadata() + for frame in metadata.get('frames', []): + frame_index = frame.get('Index') + result.write_status( + f'Evaluating flood levels at {n_nodes} nodes for frame {frame_index}...' + ) + node_failures[frame_index] = [] + for node_id, node_region in node_regions.items(): + region_data, _ = source.getRegion( + region=node_region, + frame=frame_index, + format='numpy', + ) + if numpy.any(numpy.where(region_data > tolerance)): + node_failures[frame_index].append(node_id) result.outputs = dict(failures=node_failures) except Exception as e: result.error = str(e) diff --git a/uvdat/core/tasks/analytics/flood_simulation.py b/uvdat/core/tasks/analytics/flood_simulation.py index ce4c39c2..a11c06b3 100644 --- a/uvdat/core/tasks/analytics/flood_simulation.py +++ b/uvdat/core/tasks/analytics/flood_simulation.py @@ -1,113 +1,46 @@ -from datetime import datetime +import datetime import os from pathlib import Path +import subprocess import tempfile -from urllib.request import urlretrieve from celery import shared_task from django.core.files.base import ContentFile -from uvdat.core.models import Chart, Dataset, FileItem, TaskResult +from uvdat.core.models import Chart, Colormap, Dataset, FileItem, LayerStyle, TaskResult from .analysis_type import AnalysisType -HYETOGRAPHS = [ - dict(label='Parabolic', value='parabolic'), - dict(label='NCRS Type II', value='type2'), -] - -LIKELIHOODS = [ - dict(label='1 in 25 year (4% chance)', value=0.04), - dict(label='1 in 100 year (1% chance)', value=0.01), -] - -TIME_PERIODS = [ - dict(label='2030-2050', value=[2030, 2050]), - dict(label='2080-2100', value=[2080, 2100]), - dict(label='Test', value='test'), -] - -DATA_PRODUCTS = [ - dict( - url='https://data.kitware.com/api/v1/item/67e4061f3e5f3e5e96b9753d/download', - precipitation='parabolic', - likelihood=0.04, - time_period=[2030, 2050], - ), - dict( - url='https://data.kitware.com/api/v1/item/67e408513e5f3e5e96b97544/download', - precipitation='type2', - likelihood=0.04, - time_period=[2030, 2050], - ), - dict( - url='https://data.kitware.com/api/v1/item/67e40d693e5f3e5e96b97547/download', - precipitation='parabolic', - likelihood=0.01, - time_period=[2030, 2050], - ), - dict( - url='https://data.kitware.com/api/v1/item/67e40f4e3e5f3e5e96b9754a/download', - precipitation='type2', - likelihood=0.01, - time_period=[2030, 2050], - ), - dict( - url='https://data.kitware.com/api/v1/item/67d875e5429cb34d95af01a4/download', - precipitation='parabolic', - likelihood=0.04, - time_period=[2080, 2100], - ), - dict( - url='https://data.kitware.com/api/v1/item/67d87697429cb34d95af01a7/download', - precipitation='type2', - likelihood=0.04, - time_period=[2080, 2100], - ), - dict( - url='https://data.kitware.com/api/v1/item/67d8773c429cb34d95af01ab/download', - precipitation='parabolic', - likelihood=0.01, - time_period=[2080, 2100], - ), - dict( - url='https://data.kitware.com/api/v1/item/67d87868429cb34d95af01ae/download', - precipitation='type2', - likelihood=0.01, - time_period=[2080, 2100], - ), - dict( - url='https://data.kitware.com/api/v1/item/68d438a2af4f192121e81684/download', - precipitation='parabolic', - likelihood=0.01, - time_period='test', - ), -] +MODULE_REPOSITORY = 'https://github.com/OpenGeoscience/uvdat-flood-sim.git' +MODULE_PATH = Path('/analytics/modules/uvdat-flood-sim') +VENV_PATH = Path('/venvs/flood_simulation') class FloodSimulation(AnalysisType): def __init__(self): super().__init__(self) self.name = 'Flood Simulation' - self.description = ( - 'Select a precipitation model, likelihood, and time period to simulate a flood event.' - ) + self.description = 'Select parameters to simulate a 24-hour flood of the Charles River' self.db_value = 'flood_simulation' self.input_types = { - 'precipitation': 'Chart', - 'likelihood': 'string', 'time_period': 'string', + 'hydrograph': 'Chart', + 'potential_evapotranspiration_percentile': 'number', + 'soil_moisture_percentile': 'number', + 'ground_water_percentile': 'number', + 'annual_probability': 'number', } self.output_types = {'flood': 'Dataset'} self.attribution = 'Northeastern University' def get_input_options(self): return { - 'precipitation': Chart.objects.filter(name__icontains='hyetograph'), - 'likelihood': [likelihood.get('label') for likelihood in LIKELIHOODS], - 'time_period': [ - period.get('label') for period in TIME_PERIODS if period.get('value') != 'test' - ], + 'time_period': ['2030-2050'], + 'hydrograph': Chart.objects.filter(name__icontains='hydrograph'), + 'potential_evapotranspiration_percentile': [dict(min=0, max=100, step=1)], + 'soil_moisture_percentile': [dict(min=0, max=100, step=1)], + 'ground_water_percentile': [dict(min=0, max=100, step=1)], + 'annual_probability': [dict(min=0, max=1, step=0.01)], } def run_task(self, project, **inputs): @@ -122,134 +55,196 @@ def run_task(self, project, **inputs): return result +def run_command(cmd, cwd): + result = subprocess.run( + cmd, + cwd=cwd, + capture_output=True, + ) + if result.stderr: + raise Exception(result.stderr) + + +def pull_module(): + MODULE_PATH.parent.mkdir(parents=True, exist_ok=True) + if not MODULE_PATH.exists(): + run_command( + ['git', 'clone', MODULE_REPOSITORY], + MODULE_PATH.parent, + ) + run_command(['git', 'pull', '-q'], MODULE_PATH) + + +def install_module_dependencies(): + VENV_PATH.parent.mkdir(parents=True, exist_ok=True) + if not VENV_PATH.exists(): + run_command(['python', '-m', 'venv', VENV_PATH], MODULE_PATH) + run_command( + [VENV_PATH / 'bin' / 'python', '-m', 'pip', 'install', '--upgrade', 'pip'], + MODULE_PATH, + ) + run_command( + [VENV_PATH / 'bin' / 'python', '-m', 'pip', 'install', '-r', 'requirements.txt'], + MODULE_PATH, + ) + + @shared_task def flood_simulation(result_id): result = TaskResult.objects.get(id=result_id) try: - # Verify inputs - chart = None - hyetograph = None - chart_id = result.inputs.get('precipitation') - if chart_id is None: - result.write_error('Precipitation hyetograph chart not provided') - else: - try: - chart = Chart.objects.get(id=chart_id) - except Chart.DoesNotExist: - result.write_error('Precipitation hyetograph chart not found') - if chart is not None: - hyetograph = next( - iter( - hyeto - for hyeto in HYETOGRAPHS - if hyeto.get('label') == chart.metadata.get('precipitation_model') - ), - None, - ) - if hyetograph is None: - result.write_error('Hyetograph selection not valid') - - likelihood = next( - iter(lik for lik in LIKELIHOODS if lik.get('label') == result.inputs.get('likelihood')), - None, + for input_key in [ + 'time_period', + 'hydrograph', + 'potential_evapotranspiration_percentile', + 'soil_moisture_percentile', + 'ground_water_percentile', + 'annual_probability', + ]: + if result.inputs.get(input_key) is None: + result.write_error(f'{input_key} not provided') + result.complete() + return + + result.write_status( + 'Ensuring that flood simulation module code and dependencies are up to date' ) - if likelihood is None: - result.write_error('Likelihood selection not valid') - - period = next( - iter(tp for tp in TIME_PERIODS if tp.get('label') == result.inputs.get('time_period')), - None, + pull_module() + install_module_dependencies() + + result.write_status('Interpreting input values') + time_period = result.inputs.get('time_period') + hydrograph_id = result.inputs.get('hydrograph') + hydrograph_chart = Chart.objects.get(id=hydrograph_id) + hydrograph = hydrograph_chart.chart_data.get('datasets')[0].get('data') + pet_percentile = result.inputs.get('potential_evapotranspiration_percentile') + sm_percentile = result.inputs.get('soil_moisture_percentile') + gw_percentile = result.inputs.get('ground_water_percentile') + annual_probability = result.inputs.get('annual_probability') + + name = ( + f'{time_period} {annual_probability} Flood Simulation ' + f'with {hydrograph_chart.name} and ' + f'percentiles {pet_percentile}, {sm_percentile}, {gw_percentile}' + ) + result.name = name + result.write_status('Running flood simulation module with specified inputs') + output_path = Path(tempfile.gettempdir(), 'flood_simulation.tif') + + run_command( + [ + VENV_PATH / 'bin' / 'python', + 'main.py', + '--time_period', + time_period, + '--hydrograph', + *[str(v) for v in hydrograph], + '--pet_percentile', + str(pet_percentile), + '--sm_percentile', + str(sm_percentile), + '--gw_percentile', + str(gw_percentile), + '--annual_probability', + str(annual_probability), + '--output_path', + output_path, + '--no_animation', + '--tiff-writer', + 'large_image', + ], + MODULE_PATH, ) - if period is None: - result.write_error('Time period selection not valid') - data_product = None - if hyetograph is not None: - data_product = next( - iter( - dp - for dp in DATA_PRODUCTS - if dp.get('precipitation') == hyetograph.get('value') - and dp.get('likelihood') == likelihood.get('value') - and dp.get('time_period') == period.get('value') + result.write_status('Saving result to database') + if output_path.exists(): + metadata = dict( + attribution='Simulation code by August Posch at Northeastern University', + simulation_steps=[ + 'downscaling_prediction', + 'hydrological_prediction', + 'hydrodynamic_prediction', + ], + module_repository=MODULE_REPOSITORY, + inputs=dict( + time_period=time_period, + hydrograph=hydrograph, + pet_percentile=pet_percentile, + sm_percentile=sm_percentile, + gw_percentile=gw_percentile, + annual_probability=annual_probability, ), - None, + uploaded=datetime.datetime.now(datetime.UTC).isoformat(), ) - if data_product is None: - result.write_error('Data product not found') - - # Run task - if result.error is None: - - # Update name - result.name = ( - f'{hyetograph["label"].title()} {likelihood["value"] * 100}% ' - f'Chance flood for {period["label"]}' + name_match = Dataset.objects.filter(name__icontains=name) + if name_match.count() > 0: + name += f' ({name_match.count() + 1})' + dataset = Dataset.objects.create( + name=name, + description='Generated by Flood Simulation Analytics Task', + category='flood', + metadata=metadata, ) - result.save() - - result.write_status('Downloading pregenerated data product as zip file...') - - folder = Path(tempfile.gettempdir(), 'flood_data_products') - folder.mkdir(exist_ok=True, parents=True) - file_name = ( - f'{hyetograph.get("value")}_{likelihood.get("value")}' - f'_{period.get("label")}_flood.zip' + file_item = FileItem.objects.create( + name=output_path.name, + dataset=dataset, + file_type='tif', + file_size=os.path.getsize(output_path), + metadata=metadata, ) - zip_path = Path(folder, file_name) - if not zip_path.exists(): - urlretrieve(data_product.get('url'), zip_path) - - dataset_name = hyetograph.get('label') + ' ' + likelihood.get('label') - dataset_name += ' Flood Simulation for ' + period.get('label') - dataset, created = Dataset.objects.get_or_create( - name=dataset_name, - description='Generated by Flood Simulation Analysis Task', - category='flood', - metadata=dict( - attribution=( - 'Simulated by August Posch and Jack Watson at Northeastern University' + with output_path.open('rb') as f: + file_item.file.save(output_path.name, ContentFile(f.read())) + dataset.spawn_conversion_task( + layer_options=[ + dict( + name='Flood Simulation', + source_files=[output_path.name], + frame_property='frame', ), - likelihood=likelihood.get('label'), - downscaling=dict( - climate_model='CESM2-LENS', - ensemble_member='r1i1p1f1', - emissions_scenario=370, - method='LOCA (Localized Constructed Analogs) to daily 1/16 degree', - time_period=period.get('label'), - return_period='100 years', - ), - precipitation=dict( - hyetograph=hyetograph.get('label'), - spatially_uniform=True, - dem_resolution='1/3 arcsecond = 10 m', - ), - ), + ], + network_options=None, + region_options=None, + asynchronous=False, ) - if created: - file_item = FileItem.objects.create( - name=file_name, - dataset=dataset, - file_type='zip', - file_size=os.path.getsize(zip_path), - metadata=dict( - **dataset.metadata, - uploaded=datetime.now().strftime('%m/%d/%Y %H:%M:%S'), - ), - ) - with zip_path.open('rb') as f: - file_item.file.save(zip_path, ContentFile(f.read())) - result.write_status('Interpreting data in zip file...') - dataset.spawn_conversion_task( - layer_options=[ - dict(name='Flood Simulation', source_file=zip_path.name), + # Create a default style for new layer + layer = dataset.layers.first() + style = LayerStyle.objects.create( + name='Flood Depth Viridis', + layer=layer, + project=result.project, + ) + layer.default_style = style + layer.save() + viridis = Colormap.objects.filter(name='viridis').first() + style.save_style_configs( + dict( + default_frame=0, + opacity=1, + colors=[ + dict( + name='all', + visible=True, + colormap=dict( + id=viridis.id, + discrete=False, + clamp=True, + color_by='value', + null_color='transparent', + range=[0, 2], + ), + ) + ], + sizes=[ + dict( + name='all', + zoom_scaling=True, + single_size=5, + ) ], - network_options=None, - region_options=None, - asynchronous=False, ) + ) result.outputs = dict(flood=dataset.id) except Exception as e: diff --git a/uvdat/core/tests/test_analytics.py b/uvdat/core/tests/test_analytics.py index 84031bbe..b6e939a9 100644 --- a/uvdat/core/tests/test_analytics.py +++ b/uvdat/core/tests/test_analytics.py @@ -74,14 +74,17 @@ def test_flood_analysis_chain(project): './tests/analytics.json', ) network = Network.objects.get(name='MBTA Rapid Transit Network 1') - chart = Chart.objects.get(name='Parabolic Hyetograph') + chart = Chart.objects.get(name='Charles River Hydrograph (Short Flow Length)') # run flood simulation task result_1 = FloodSimulation().run_task( project=project, - precipitation=chart.id, - likelihood='1 in 100 year (1% chance)', - time_period='Test', + hydrograph=chart.id, + time_period='2030-2050', + annual_probability=0.25, + potential_evapotranspiration_percentile=50, + soil_moisture_percentile=50, + ground_water_percentile=50, ) result_1.refresh_from_db() assert result_1.completed is not None @@ -91,17 +94,50 @@ def test_flood_analysis_chain(project): flood_dataset_id = result_1.outputs.get('flood') assert flood_dataset_id is not None flood_dataset = Dataset.objects.get(id=flood_dataset_id) - assert flood_dataset.name == 'Parabolic 1 in 100 year (1% chance) Flood Simulation for Test' - assert flood_dataset.description == 'Generated by Flood Simulation Analysis Task' + assert flood_dataset.name == ( + '2030-2050 0.25 Flood Simulation with Charles River ' + 'Hydrograph (Short Flow Length) and percentiles 50, 50, 50' + ) + assert flood_dataset.description == 'Generated by Flood Simulation Analytics Task' assert flood_dataset.category == 'flood' metadata = flood_dataset.metadata - assert metadata.get('likelihood') == '1 in 100 year (1% chance)' - assert metadata.get('downscaling', {}).get('time_period') == 'Test' - assert metadata.get('precipitation', {}).get('hyetograph') == 'Parabolic' + assert metadata.get('inputs', {}) == dict( + time_period='2030-2050', + hydrograph=[ + 0.006, + 0.026, + 0.066, + 0.111, + 0.138, + 0.143, + 0.128, + 0.102, + 0.08, + 0.054, + 0.038, + 0.03, + 0.021, + 0.016, + 0.012, + 0.008, + 0.006, + 0.005, + 0.003, + 0.002, + 0.002, + 0.001, + 0.001, + 0.001, + ], + pet_percentile=50, + sm_percentile=50, + gw_percentile=50, + annual_probability=0.25, + ) layer = flood_dataset.layers.first() - assert layer.frames.count() == 2 + assert layer.frames.count() == 24 # run flood network failure task result_2 = FloodNetworkFailure().run_task( @@ -117,9 +153,7 @@ def test_flood_analysis_chain(project): assert result_2.outputs is not None failures = result_2.outputs.get('failures') - assert len(failures) == 2 - assert len(failures['0']) == 51 - assert len(failures['1']) == 82 + assert len(failures) == 24 # run network recovery task result_3 = NetworkRecovery().run_task( @@ -133,6 +167,4 @@ def test_flood_analysis_chain(project): assert result_3.outputs is not None recoveries = result_3.outputs.get('recoveries') - assert len(recoveries) == 83 - assert len(recoveries['0']) == 82 - assert len(recoveries['82']) == 0 + assert len(recoveries) == 5 diff --git a/web/src/api/rest.ts b/web/src/api/rest.ts index a112d17c..bc26d9c6 100644 --- a/web/src/api/rest.ts +++ b/web/src/api/rest.ts @@ -18,6 +18,7 @@ import { LayerStyle, VectorSummary, LayerFrame, + TaskResult, Colormap, } from "@/types"; @@ -203,6 +204,10 @@ export async function getTaskResults( ).data; } +export async function getTaskResult(resultId: number): Promise { + return (await apiClient.get(`analytics/${resultId}`)).data; +} + export async function getVectorDataBounds(vectorId: number): Promise { return (await apiClient.get(`vectors/${vectorId}/bounds/`)).data; } diff --git a/web/src/components/sidebars/AnalyticsPanel.vue b/web/src/components/sidebars/AnalyticsPanel.vue index 0ae65744..ecedcd1b 100644 --- a/web/src/components/sidebars/AnalyticsPanel.vue +++ b/web/src/components/sidebars/AnalyticsPanel.vue @@ -6,8 +6,11 @@ import { getDataset, getProjectAnalysisTypes, getChart, + getTaskResult, + getNetwork, } from "@/api/rest"; import NodeAnimation from "./NodeAnimation.vue"; +import SliderNumericInput from "../SliderNumericInput.vue"; import { TaskResult } from "@/types"; import { @@ -113,23 +116,60 @@ function fetchResults() { }); } +function inputIsNumeric(key: string) { + return ( + analysisStore.currentAnalysisType && + analysisStore.currentAnalysisType.input_types[key] === 'number' && + analysisStore.currentAnalysisType.input_options[key].length == 1 && + analysisStore.currentAnalysisType.input_options[key][0].min !== undefined && + analysisStore.currentAnalysisType.input_options[key][0].max !== undefined && + analysisStore.currentAnalysisType.input_options[key][0].step !== undefined + ) +} + +async function getFullObject(type: string, value: any) { + if (type !== 'number' && typeof value === 'number') { + value = {id: value} + } + if (type == 'dataset') { + value = await getDataset(value.id) + } + if (type == 'chart') { + value = await getChart(value.id) + } + if (type == 'network') { + value = await getNetwork(value.id) + } + if (type == 'taskresult') { + value = await getTaskResult(value.id) + } + if (typeof value === 'object') { + value.type = type + value.visible = panelStore.isVisible({[type]: value}) + value.showable = panelStore.showableTypes.includes(value.type) + } else { + value = { + name: value, + } + } + return value +} + async function fillInputsAndOutputs() { if (!currentResult.value?.inputs){ fullInputs.value = undefined; additionalAnimationLayers.value = undefined; } else { fullInputs.value = Object.fromEntries( - Object.entries(currentResult.value.inputs).map(([key, value]) => { - const fullValue = analysisStore.currentAnalysisType?.input_options[key]?.find( - (o: any) => o.id == value - ); - if (fullValue) { - fullValue.type = analysisStore.currentAnalysisType?.input_types[key].toLowerCase() - fullValue.visible = panelStore.isVisible({[fullValue.type]: fullValue}) - fullValue.showable = panelStore.showableTypes.includes(fullValue.type) - } - return [key, fullValue || value]; - }) + await Promise.all( + Object.entries(currentResult.value.inputs).map(async ([key, value]) => { + const fullValue = analysisStore.currentAnalysisType?.input_options[key]?.find( + (o: any) => o.id == value + ); + const type = analysisStore.currentAnalysisType?.input_types[key].toLowerCase() + return [key, await getFullObject(type, fullValue || value)]; + }) + ) ); if (fullInputs.value?.flood_simulation && !additionalAnimationLayers.value) { const floodDataset = { @@ -148,22 +188,7 @@ async function fillInputsAndOutputs() { await Promise.all( Object.entries(currentResult.value.outputs).map(async ([key, value]) => { const type = analysisStore.currentAnalysisType?.output_types[key].toLowerCase(); - if (type == 'dataset') { - value = await getDataset(value) - } - if (type == 'chart') { - value = await getChart(value) - } - if (typeof value === 'object') { - value.type = type - value.visible = panelStore.isVisible({[type]: value}) - value.showable = panelStore.showableTypes.includes(value.type) - } else { - value = { - name: value, - } - } - return [key, value]; + return [key, await getFullObject(type, value)]; }) ) ); @@ -206,6 +231,15 @@ watch(() => projectStore.currentProject, createWebSocket) watch(() => analysisStore.currentAnalysisType, () => { fetchResults() + const type = analysisStore.currentAnalysisType + selectedInputs.value = {} + if (type) { + Object.keys(type.input_types).forEach((key) => { + if (inputIsNumeric(key)) { + selectedInputs.value[key] = type.input_options[key][0].min + } + }) + } }) watch(tab, () => { @@ -219,7 +253,6 @@ watch( currentResult, () => layerStore.selectedLayers, () => analysisStore.currentChart, - () => panelStore.panelArrangement ], fillInputsAndOutputs, {deep: true} @@ -264,8 +297,20 @@ watch( Select inputs
+
+ {{ key.replaceAll('_', ' ') }} +
+ +
+
+ > + +
Run Analysis diff --git a/web/src/components/sidebars/ChartsPanel.vue b/web/src/components/sidebars/ChartsPanel.vue index 4b8c492e..49f0a959 100644 --- a/web/src/components/sidebars/ChartsPanel.vue +++ b/web/src/components/sidebars/ChartsPanel.vue @@ -168,16 +168,16 @@ const downloadReady = computed(() => { />
-
+
- - { }); const nodeChanges = computed(() => { - if (props.nodeRecoveries) return props.nodeRecoveries; - else return props.nodeFailures; + const changes = props.nodeRecoveries || props.nodeFailures || {} + return Object.fromEntries( + Object.entries(changes).filter( + ([key]) => !['id', 'name', 'type', 'visible', 'showable'].includes(key) + ) + ) }); +const numTicks = computed(() => { + return Object.keys(nodeChanges.value).length - 1 +}) + function pause() { clearInterval(ticker.value); currentMode.value = undefined; @@ -38,7 +46,7 @@ function play() { pause(); currentMode.value = 'play' ticker.value = setInterval(() => { - if (nodeChanges.value && currentTick.value < Object.keys(nodeChanges.value).length) { + if (nodeChanges.value && currentTick.value < numTicks.value) { currentTick.value += 1; } else { pause(); @@ -90,7 +98,7 @@ watch(currentTick, async () => { track-size="8" min="0" step="1" - :max="Object.keys(nodeChanges).length" + :max="numTicks" hide-details /> {{ currentTick + 1 }} diff --git a/web/src/store/map.ts b/web/src/store/map.ts index f1166038..49068e68 100644 --- a/web/src/store/map.ts +++ b/web/src/store/map.ts @@ -132,6 +132,7 @@ export const useMapStore = defineStore('map', () => { const tooltipOverlay = ref(); const clickedFeature = ref(); const rasterTooltipDataCache = ref>({}); + const rasterSourceTileURLs = ref>({}); const styleStore = useStyleStore(); const layerStore = useLayerStore(); @@ -433,7 +434,7 @@ export const useMapStore = defineStore('map', () => { const { layerId, layerCopyId } = parseSourceString(sourceId); const styleSpec = styleStore.selectedLayerStyles[`${layerId}.${layerCopyId}`].style_spec; let filters: StyleFilter[] = [] - const layer = layerStore.selectedLayers.find((l: Layer) => l.id = layerId) + const layer = layerStore.selectedLayers.find((l: Layer) => l.id === layerId) if (layer) { const frames = layerStore.layerFrames(layer) const frame = frames.find((f: LayerFrame) => f.index === layer.current_frame_index) @@ -452,9 +453,10 @@ export const useMapStore = defineStore('map', () => { if (styleParams) queryParams.style = JSON.stringify(styleParams) } const query = new URLSearchParams(queryParams) + rasterSourceTileURLs.value[sourceId] = `${baseURL}rasters/${raster.id}/tiles/{z}/{x}/{y}.png/?${query}` map.addSource(sourceId, { type: "raster", - tiles: [`${baseURL}rasters/${raster.id}/tiles/{z}/{x}/{y}.png/?${query}`], + tiles: [rasterSourceTileURLs.value[sourceId]], }); const tileSource = map.getSource(sourceId); @@ -502,6 +504,7 @@ export const useMapStore = defineStore('map', () => { tooltipOverlay, clickedFeature, rasterTooltipDataCache, + rasterSourceTileURLs, // Functions handleLayerClick, toggleBaseLayer, diff --git a/web/src/store/panel.ts b/web/src/store/panel.ts index a6c2929a..a5f0fb1e 100644 --- a/web/src/store/panel.ts +++ b/web/src/store/panel.ts @@ -1,7 +1,7 @@ import { FloatingPanelConfig, TaskResult, Chart, Dataset, Layer, Network, RasterData, VectorData } from '@/types'; import { defineStore } from 'pinia'; import { ref } from 'vue'; -import { getDataset } from '@/api/rest'; +import { getChart, getDataset } from '@/api/rest'; import { useAppStore, useLayerStore, useAnalysisStore, useProjectStore } from '.'; @@ -234,11 +234,15 @@ export const usePanelStore = defineStore('panel', () => { } - function show(showable: Showable) { + async function show(showable: Showable) { if (showable.chart) { + let chart = showable.chart + if (!chart.chart_data) { + chart = await getChart(chart.id) + } const chartPanel = panelArrangement.value.find((panel) => panel.id === 'charts') if (chartPanel && !chartPanel?.visible) chartPanel.visible = true - analysisStore.currentChart = showable.chart + analysisStore.currentChart = chart } else if (showable.dataset) { const id = showable.dataset.id layerStore.fetchAvailableLayersForDataset(id).then(() => { diff --git a/web/src/store/style.ts b/web/src/store/style.ts index c960e27b..f33a4f8e 100644 --- a/web/src/store/style.ts +++ b/web/src/store/style.ts @@ -432,13 +432,15 @@ export const useStyleStore = defineStore('style', () => { if (rasterTilesQuery?.bands && !rasterTilesQuery.bands.length) opacity = 0 map.setPaintProperty(mapLayerId, "raster-opacity", opacity) let source = map.getSource(mapLayer.source) as RasterTileSource; - if (source?.tiles?.length) { - const oldQuery = new URLSearchParams(source.tiles[0].split('?')[1]) + const sourceURL = mapStore.rasterSourceTileURLs[mapLayer.source] + if (source && sourceURL) { + const oldQuery = new URLSearchParams(sourceURL.split('?')[1]) const newQueryParams: { projection: string, style?: string } = { projection: 'epsg:3857' } if (rasterTilesQuery) newQueryParams.style = JSON.stringify(rasterTilesQuery) const newQuery = new URLSearchParams(newQueryParams) if (newQuery.toString() !== oldQuery.toString()) { - source.setTiles(source.tiles.map((url) => url.split('?')[0] + '?' + newQuery)) + const newURL = sourceURL.split('?')[0] + '?' + newQuery + source.setTiles([newURL]) } } }