From b81da03f224e0330ae61359eb11d97c204374231 Mon Sep 17 00:00:00 2001 From: Chen Zhang Date: Mon, 9 Dec 2024 13:38:40 -0500 Subject: [PATCH 1/4] add batch Bragg edge fitting notebook with configuration loading and processing --- notebooks/batch_bragg_edge_fitting.py | 130 ++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 notebooks/batch_bragg_edge_fitting.py diff --git a/notebooks/batch_bragg_edge_fitting.py b/notebooks/batch_bragg_edge_fitting.py new file mode 100644 index 0000000..6395d87 --- /dev/null +++ b/notebooks/batch_bragg_edge_fitting.py @@ -0,0 +1,130 @@ +import marimo + +__generated_with = "0.9.15" +app = marimo.App(width="medium") + + +@app.cell +def __(): + import marimo as mo + import json + from ibeatles.core.config import IBeatlesUserConfig + + return IBeatlesUserConfig, json, mo + + +@app.cell +def __(mo): + mo.md(r"""# Batch Bragg Edge Fitting with iBeatles""") + return + + +@app.cell +def __(mo): + mo.md(r"""## Load Configuration from Manual Session""") + return + + +@app.cell +def __(mo): + base_configuration_file = mo.ui.file( + filetypes=[".JSON", ".json"], + label="Select Configuration File", + multiple=False, + kind="area", + ) + + base_configuration_file + return (base_configuration_file,) + + +@app.cell +def __(base_configuration_file, json, mo): + ## display JSON content + base_json_viewer = None + if base_configuration_file.contents() is not None: + base_json_viewer = mo.accordion(json.loads(base_configuration_file.contents())) + return (base_json_viewer,) + + +@app.cell +def __(base_json_viewer, mo): + mo.vstack( + [ + mo.md(r"""Base Configuration JSON"""), + base_json_viewer, + ] + ) + return + + +@app.cell +def __(IBeatlesUserConfig, base_configuration_file, json): + # process the configuration file into a IBeatlesUserConfig object + if base_configuration_file.contents() is not None: + base_ibeatles_config = IBeatlesUserConfig( + **json.loads(base_configuration_file.contents()) + ) + return (base_ibeatles_config,) + + +@app.cell +def __(mo): + mo.md(r"""## Select Folder**s** for Batch Processing""") + return + + +@app.cell +def __(mo): + # select multiple folders for batch processing + folders_selector_sample = mo.ui.file_browser( + initial_path="~/tmp", + multiple=True, + selection_mode="directory", + label="Select SAMPLE folders for batch processing", + restrict_navigation=True, + ) + return (folders_selector_sample,) + + +@app.cell +def __(folders_selector_sample): + folders_selector_sample + return + + +@app.cell +def __(mo): + mo.md(r"""## Verify Processing Parameters""") + return + + +@app.cell +def __(): + return + + +@app.cell +def __(mo): + mo.md(r"""## Running Batch Processing""") + return + + +@app.cell +def __(): + return + + +@app.cell +def __(mo): + mo.md(r"""## Summary of results""") + return + + +@app.cell +def __(): + return + + +if __name__ == "__main__": + app.run() From f3c830fc1db454ebd2800b543392704bce5abfde Mon Sep 17 00:00:00 2001 From: Chen Zhang Date: Tue, 10 Dec 2024 11:39:37 -0500 Subject: [PATCH 2/4] adding cells to create configs --- notebooks/batch_bragg_edge_fitting.py | 42 +++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/notebooks/batch_bragg_edge_fitting.py b/notebooks/batch_bragg_edge_fitting.py index 6395d87..f7d8b59 100644 --- a/notebooks/batch_bragg_edge_fitting.py +++ b/notebooks/batch_bragg_edge_fitting.py @@ -1,6 +1,6 @@ import marimo -__generated_with = "0.9.15" +__generated_with = "0.9.33" app = marimo.App(width="medium") @@ -8,9 +8,10 @@ def __(): import marimo as mo import json + import copy from ibeatles.core.config import IBeatlesUserConfig - return IBeatlesUserConfig, json, mo + return IBeatlesUserConfig, copy, json, mo @app.cell @@ -93,6 +94,43 @@ def __(folders_selector_sample): return +@app.cell +def __(base_ibeatles_config, copy, folders_selector_sample, mo): + # After the users have selected the folders for different sample input, we will create a list of IBeatlesUserConfig objects for each sample + batch_config_list = [] + num_samples = len(folders_selector_sample.value) + + mo.md(f"""Number of samples selected: **{num_samples}**""") + + for _i in range(num_samples): + # duplicate the base configuration + _sample_config = copy.deepcopy(base_ibeatles_config) + # update the raw data path + _sample_config.raw_data.raw_data_dir = folders_selector_sample.value[_i] + # append + batch_config_list.append(_sample_config) + return batch_config_list, num_samples + + +@app.cell +def __(batch_config_list, mo, num_samples): + # visualize as tabs of accordians + _tabs = {} + + for _i in range(num_samples): + _acc = mo.accordion(batch_config_list[_i].dict()) + _tabs[f"Sample {_i}"] = _acc + + # display the tabs + mo.vstack( + [ + mo.md(r"""Samples to be processed"""), + mo.ui.tabs(_tabs), + ] + ) + return + + @app.cell def __(mo): mo.md(r"""## Verify Processing Parameters""") From 92b073a125e5d118a42d0fa409344bc74c1840a8 Mon Sep 17 00:00:00 2001 From: Chen Zhang Date: Thu, 12 Dec 2024 14:46:37 -0500 Subject: [PATCH 3/4] enhance configuration handling and add processing verification --- notebooks/batch_bragg_edge_fitting.py | 90 ++++++++++++++++++++------- 1 file changed, 68 insertions(+), 22 deletions(-) diff --git a/notebooks/batch_bragg_edge_fitting.py b/notebooks/batch_bragg_edge_fitting.py index f7d8b59..4460150 100644 --- a/notebooks/batch_bragg_edge_fitting.py +++ b/notebooks/batch_bragg_edge_fitting.py @@ -1,6 +1,6 @@ import marimo -__generated_with = "0.9.33" +__generated_with = "0.9.34" app = marimo.App(width="medium") @@ -9,9 +9,12 @@ def __(): import marimo as mo import json import copy + import time + from pathlib import Path from ibeatles.core.config import IBeatlesUserConfig + from ibeatles.app.cli import main as ibeatles_main - return IBeatlesUserConfig, copy, json, mo + return IBeatlesUserConfig, Path, copy, ibeatles_main, json, mo, time @app.cell @@ -95,7 +98,7 @@ def __(folders_selector_sample): @app.cell -def __(base_ibeatles_config, copy, folders_selector_sample, mo): +def __(Path, base_ibeatles_config, copy, folders_selector_sample, mo): # After the users have selected the folders for different sample input, we will create a list of IBeatlesUserConfig objects for each sample batch_config_list = [] num_samples = len(folders_selector_sample.value) @@ -106,12 +109,35 @@ def __(base_ibeatles_config, copy, folders_selector_sample, mo): # duplicate the base configuration _sample_config = copy.deepcopy(base_ibeatles_config) # update the raw data path - _sample_config.raw_data.raw_data_dir = folders_selector_sample.value[_i] + _sample_config.raw_data.raw_data_dir = folders_selector_sample.value[_i].path + # Update output file paths + _sample_config.output["normalized_data_dir"] = Path( + str(_sample_config.output["normalized_data_dir"]) + f"_{_i}" + ) + _sample_config.output["analysis_results_dir"] = Path( + str(_sample_config.output["analysis_results_dir"]) + f"_{_i}" + ) + _sample_config.output["strain_results_dir"] = Path( + str(_sample_config.output["strain_results_dir"]) + f"_{_i}" + ) + # append batch_config_list.append(_sample_config) return batch_config_list, num_samples +@app.cell +def __(mo): + mo.md( + r""" + ## Verify Processing Parameters + + Inspect the parameters to ensure all are correct + """ + ) + return + + @app.cell def __(batch_config_list, mo, num_samples): # visualize as tabs of accordians @@ -122,40 +148,60 @@ def __(batch_config_list, mo, num_samples): _tabs[f"Sample {_i}"] = _acc # display the tabs - mo.vstack( - [ - mo.md(r"""Samples to be processed"""), - mo.ui.tabs(_tabs), - ] - ) - return + config_items_viewer = mo.md(r"""No samples to process""") + if num_samples > 0: + config_items_viewer = mo.vstack( + [ + mo.md(r"""Samples to be processed"""), + mo.ui.tabs(_tabs), + ] + ) - -@app.cell -def __(mo): - mo.md(r"""## Verify Processing Parameters""") - return + config_items_viewer + return (config_items_viewer,) @app.cell -def __(): +def __(mo): + mo.md(r"""## Running Batch Processing""") return @app.cell def __(mo): - mo.md(r"""## Running Batch Processing""") - return + exec_button = mo.ui.run_button( + kind="success", + disabled=False, + tooltip="Run batch processing", + label="Run Batch Processing", + ) + return (exec_button,) @app.cell -def __(): +def __(exec_button): + exec_button return @app.cell -def __(mo): - mo.md(r"""## Summary of results""") +def __(batch_config_list, exec_button, ibeatles_main, mo, num_samples): + if exec_button.value: + # disable the button first + exec_button.disabled = True + for _i in mo.status.progress_bar( + range(num_samples), + title="Processing", + subtitle="Please wait...", + show_eta=True, + show_rate=True, + ): + _config = batch_config_list[_i] + with mo.redirect_stdout(), mo.redirect_stderr(): + print(f"Processing sample {_i}") + ibeatles_main(_config) + # re-enable the button + exec_button.disabled = False return From d789430ff37f8dbe8f34c54efe480350fa7edc12 Mon Sep 17 00:00:00 2001 From: Chen Zhang Date: Thu, 12 Dec 2024 15:25:15 -0500 Subject: [PATCH 4/4] adjust layout --- notebooks/batch_bragg_edge_fitting.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/notebooks/batch_bragg_edge_fitting.py b/notebooks/batch_bragg_edge_fitting.py index 4460150..f30cf01 100644 --- a/notebooks/batch_bragg_edge_fitting.py +++ b/notebooks/batch_bragg_edge_fitting.py @@ -178,17 +178,15 @@ def __(mo): return (exec_button,) -@app.cell -def __(exec_button): - exec_button - return - - @app.cell def __(batch_config_list, exec_button, ibeatles_main, mo, num_samples): + # dict to store stderr + dict_stderr = {} + if exec_button.value: # disable the button first exec_button.disabled = True + for _i in mo.status.progress_bar( range(num_samples), title="Processing", @@ -197,12 +195,21 @@ def __(batch_config_list, exec_button, ibeatles_main, mo, num_samples): show_rate=True, ): _config = batch_config_list[_i] - with mo.redirect_stdout(), mo.redirect_stderr(): + with mo.redirect_stdout(), mo.capture_stderr() as buffer_stderr: print(f"Processing sample {_i}") ibeatles_main(_config) + # append the stderr to the output as accordian + dict_stderr[f"stderr::sample_{_i}"] = buffer_stderr.getvalue() # re-enable the button exec_button.disabled = False - return + + mo.vstack( + [ + exec_button, + mo.accordion(dict_stderr), + ] + ) + return buffer_stderr, dict_stderr @app.cell