No magic cells in automatic run_tutorial.py and other related changes

docs/tutorials/run_tutorials.py

@@ -3,6 +3,8 @@
 import logging
 import traceback
 
+from nbclient.exceptions import CellExecutionError
+
 logging.basicConfig(format='%(asctime)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s',
                     datefmt='%Y-%m-%d %H:%M:%S',
                     level=logging.INFO)
@@ -15,7 +17,7 @@
 from pathlib import Path
 
 import nbformat
-from nbconvert.preprocessors import ExecutePreprocessor
+from nbconvert.preprocessors import ExecutePreprocessor, RegexRemovePreprocessor
 from nbconvert import HTMLExporter
 from nbconvert.writers import FilesWriter
 
@@ -213,6 +215,7 @@ def run_tutorial(tutorial):
                         os.symlink(local_copy, wdir/local_copy.name)
 
         # Run
+        failed = False
         if not args.dry:
             for notebook in notebooks:
                 source_nb_path = config.absolute_path(notebook)
@@ -221,13 +224,33 @@ def run_tutorial(tutorial):
                 with (open(nb_path) as nb_file):
                     nb = nbformat.read(nb_file, as_version=nbformat.NO_CONVERT)
 
+                    # Remove magic, which can make a failing notebook look
+                    # like it succeeded.
+                    for cell in nb.cells:
+                        if cell.cell_type == 'code':
+                            source = cell.source.strip("\n").lstrip()
+                            if len(source) >= 2 and source[:2] == "%%":
+                                cell.source = cell.source.replace("%%", "#[magic commented out by run_tutorials.py]%%")
+
                     logger.info(f"Executing notebook {source_nb_path}...")
                     start_time = timeit.default_timer()
                     ep = ExecutePreprocessor(timeout=config['globals:timeout'], kernel_name=config['globals:kernel'])
-                    ep_out = ep.preprocess(nb, {'metadata': {'path': str(wdir)}})
+
+                    try:
+                        ep_out = ep.preprocess(nb, {'metadata': {'path': str(wdir)}})
+                    except CellExecutionError as e:
+                        # Will re-raise after output and cleaning
+                        cell_exception = e
+                        failed = True
+
                     elapsed = timeit.default_timer() - start_time
-                    logger.info(f"Notebook {source_nb_path} took {elapsed} seconds to finish.")
 
+                    if failed:
+                        logger.error(f"Notebook {source_nb_path} failed after {elapsed} seconds")
+                    else:
+                        logger.info(f"Notebook {source_nb_path} took {elapsed} seconds to finish.")
+
+                    # Save output
                     nb_exec_path = nb_path.with_name(nb_path.stem + "_executed" + nb_path.suffix)
                     with open(nb_exec_path, 'w', encoding='utf-8') as exec_nb_file:
                         nbformat.write(nb, exec_nb_file)
@@ -242,6 +265,10 @@ def run_tutorial(tutorial):
         # Remove file logger
         logger.removeHandler(file_handler)
 
+        # Re-raise if failed
+        if failed:
+            raise cell_exception
+
     # Loop through each tutorial
     summary = {}
     for tutorial in tutorials:
@@ -270,7 +297,11 @@ def run_tutorial(tutorial):
         if succeeded:
             logger.info(colorama.Fore.GREEN + "SUCCEEDED " + colorama.Style.RESET_ALL + f"({elapsed:.1f} s) {tutorial}")
         else:
-            logger.info(colorama.Fore.RED   + "FAILED    " + colorama.Style.RESET_ALL + f"({elapsed:.1f} s) {tutorial}")
+            color = colorama.Fore.RED
+            if "test_must_fail" in tutorial:
+                # Failed succesfully!
+                color = colorama.Fore.GREEN
+            logger.info(color               + "FAILED    " + colorama.Style.RESET_ALL + f"({elapsed:.1f} s) {tutorial}")
 
     # Overall summary log
     logger.info(f"cosipy version: {cosipy.__version__}")
@@ -283,7 +314,11 @@ def run_tutorial(tutorial):
         if succeeded:
             logger.info(colorama.Fore.GREEN + "SUCCEEDED " + colorama.Style.RESET_ALL + f"({elapsed:.1f} s) {tutorial}")
         else:
-            logger.info(colorama.Fore.RED   + "FAILED    " + colorama.Style.RESET_ALL + f"({elapsed:.1f} s) {tutorial}")
+            color = colorama.Fore.RED
+            if "test_must_fail" in tutorial:
+                # Failed succesfully!
+                color = colorama.Fore.GREEN
+            logger.info(color               + "FAILED    " + colorama.Style.RESET_ALL + f"({elapsed:.1f} s) {tutorial}")
 
 
 if __name__ == "__main__":

docs/tutorials/run_tutorials.yml

@@ -31,6 +31,8 @@ tutorials:
           unzip: True # Optional. False by default
           #unzip_output: # Optional, if the unzipped file name is different from just removing the .zip or .gz
 
+    test_must_fail_magic:
+       notebook: test/test_must_fail_magic.ipynb
 
     dataIO:
       notebook: DataIO/DataIO_example.ipynb

docs/tutorials/test/test_must_fail_magic.ipynb

@@ -0,0 +1,73 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "b82642d2-a68f-41d2-8a63-3fee53880c71",
+   "metadata": {},
+   "source": [
+    "# Test magic removal"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b42b3af6-b4e6-4809-bbcb-8917202d5c44",
+   "metadata": {},
+   "source": [
+    "Magic cells are run in a subprocess, which catches exceptions and makes it look like the notebook succeeded "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7e9ae042-38d3-4f51-9fd9-0f630bdc1e48",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "# This should fail since \"five\" has not been defined.\n",
+    "5*five"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "eb67f11c-b8cc-42ef-ac81-ac01fd6a88da",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "# It shouldn't make it to this cell\n",
+    "5*5"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ac335794-7781-410e-9b60-d0dfe2ef6ad3",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python [conda env:cosipy]",
+   "language": "python",
+   "name": "conda-env-cosipy-py"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/tutorials/ts_map/Parallel_TS_map_computation.ipynb

@@ -704,7 +704,7 @@
     "\n",
     "GRB_signal_path = data_dir/\"grb_binned_data.hdf5\"\n",
     "# download GRB signal file ~76.90 KB\n",
-    "fetch_wasabi_file(\"COSI-SMEX/cosipy_tutorials/grb_spectral_fit_local_frame/grb_binned_data.hdf5\", GRB_signal_path, checksum = 'fce391a4b45624b25552c7d111945f60')\n",
+    "fetch_wasabi_file(\"COSI-SMEX/cosipy_tutorials/grb_spectral_fit_local_frame/grb_binned_data.hdf5\", GRB_signal_path, checksum = 'fcf7022369b6fb378d67b780fc4b5db8')\n",
     "\n",
     "background_path = data_dir/\"bkg_binned_data_local.hdf5\"\n",
     "# download background file ~255.97 MB\n",