WIP: Use modified climkern to stream kernel data

environment.yml

@@ -25,4 +25,4 @@ dependencies:
   - h5netcdf
   - pip
   - pip:
-    - climkern
+    - git+https://github.com/brian-rose/climkern.git@stream # a modified version that streams the kernel data

myst.yml

@@ -24,6 +24,7 @@ project:
         - file: notebooks/foundations/energy-balance-model.ipynb
         - file: notebooks/foundations/manual-calc.ipynb
         - file: notebooks/foundations/climkern-calc.ipynb
+        - file: notebooks/foundations/multi-model.ipynb
     - title: Appendix
       children:
         - file: references.md

notebooks/foundations/climkern-calc.ipynb

@@ -109,17 +109,26 @@
     "Future versions of ClimKern will be able to pull the kernel from Jetstream2 without storing locally.\n",
     ":::\n",
     "\n",
-    "First, set the URL and path to point to ClimKern. Also, specify which kernel you want."
+    "First, set the URL and path to point to ClimKern. Also, specify which kernel you want.\n",
+    "\n",
+    "ACTUALLY THIS VERSION USES THE MODIFIED CLIMKERN THAT HANDLES DATA STREAMING AUTOMATICALLY."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "URL = \"https://js2.jetstream-cloud.org:8001/\" # URL for jetstream access\n",
-    "path = f\"pythia/ClimKern\" # Location of ClimKern\n",
+    "# URL = \"https://js2.jetstream-cloud.org:8001/\" # URL for jetstream access\n",
+    "# path = f\"pythia/ClimKern\" # Location of ClimKern\n",
     "kernel = \"ERA5\""
    ]
   },
@@ -136,20 +145,20 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Read in data\n",
-    "# Set up access to jetstream2\n",
-    "fs = fsspec.filesystem(\"s3\", anon=True, client_kwargs=dict(endpoint_url=URL))\n",
-    "pattern = f\"s3://{path}/kernels/\"+kernel+f\"/TOA*.nc\"\n",
+    "# # Read in data\n",
+    "# # Set up access to jetstream2\n",
+    "# fs = fsspec.filesystem(\"s3\", anon=True, client_kwargs=dict(endpoint_url=URL))\n",
+    "# pattern = f\"s3://{path}/kernels/\"+kernel+f\"/TOA*.nc\"\n",
     "\n",
-    "# Grab the data\n",
-    "files = sorted(fs.glob(pattern))\n",
-    "fs.invalidate_cache() # This is necessary to deal with peculiarities of objects served from jetstream2\n",
+    "# # Grab the data\n",
+    "# files = sorted(fs.glob(pattern))\n",
+    "# fs.invalidate_cache() # This is necessary to deal with peculiarities of objects served from jetstream2\n",
     "\n",
-    "# Open file and make it Xarray Dataset\n",
-    "kern = xr.open_dataset(fs.open(files[0]))\n",
+    "# # Open file and make it Xarray Dataset\n",
+    "# kern = xr.open_dataset(fs.open(files[0]))\n",
     "\n",
-    "# Save path for later\n",
-    "path_out = files[0].split(kernel+\"/\",1)[1]"
+    "# # Save path for later\n",
+    "# path_out = files[0].split(kernel+\"/\",1)[1]"
    ]
   },
   {
@@ -167,20 +176,20 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Get the package location\n",
-    "spec = importlib.util.find_spec(\"climkern\")\n",
-    "package_dir = os.path.dirname(spec.origin)\n",
+    "# # Get the package location\n",
+    "# spec = importlib.util.find_spec(\"climkern\")\n",
+    "# package_dir = os.path.dirname(spec.origin)\n",
     "\n",
-    "# print(f\"The package directory is: {package_dir}\")\n",
+    "# # print(f\"The package directory is: {package_dir}\")\n",
     "\n",
-    "# Define the path where you want to save the netCDF file within the package directory\n",
-    "netcdf_path = os.path.join(package_dir,\"data/kernels\",kernel,path_out)\n",
+    "# # Define the path where you want to save the netCDF file within the package directory\n",
+    "# netcdf_path = os.path.join(package_dir,\"data/kernels\",kernel,path_out)\n",
     "\n",
-    "# Ensure the directory exists\n",
-    "os.makedirs(os.path.dirname(netcdf_path), exist_ok=True)\n",
+    "# # Ensure the directory exists\n",
+    "# os.makedirs(os.path.dirname(netcdf_path), exist_ok=True)\n",
     "\n",
-    "# Save the dataset as a netCDF file\n",
-    "kern.to_netcdf(netcdf_path)"
+    "# # Save the dataset as a netCDF file\n",
+    "# kern.to_netcdf(netcdf_path)"
    ]
   },
   {
@@ -874,7 +883,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.13.5"
+   "version": "3.13.7"
   },
   "nbdime-conflicts": {
    "local_diff": [