From 85a8e826ac3f86497df762a641cec005bb1b11d0 Mon Sep 17 00:00:00 2001 From: Luis Lopez Date: Tue, 5 May 2026 12:46:30 -0500 Subject: [PATCH 01/24] Add open_virtual() top-level API with force_external, load, and codec homogenization - open_virtual(): open kerchunk/parquet/json/icechunk stores from URI or DataCollection with load=True (kerchunk engine) or load=False (VirtualiZarr) - force_external: download refs, rewrite s3:// to https:// for external access - DataCollection.virtual_collection_url(), get_s3_credentials(), get_s3_filesystem() - homogenize_dataset_codec_level(): patch Zlib codec levels on ManifestArrays - virtualize() default access changed from direct to indirect - build_obstore_registry: remove bearer token from HTTPStore (proxies return 303 redirects) - Update deps: virtualizarr latest, icechunk, matplotlib, hvplot; remove pandas pin --- .codespellignore | 1 + earthaccess/__init__.py | 4 +- earthaccess/results.py | 48 ++ earthaccess/virtual/__init__.py | 16 +- earthaccess/virtual/_parser.py | 68 +++ earthaccess/virtual/core.py | 454 ++++++++++++++++++- pyproject.toml | 5 +- tests/integration/test_virtualizarr.py | 53 +++ tests/unit/test_virtual.py | 584 ++++++++++++++++++------- uv.lock | 211 ++++++++- 10 files changed, 1283 insertions(+), 161 deletions(-) diff --git a/.codespellignore b/.codespellignore index 9dfa5ce6d..aef7d4351 100644 --- a/.codespellignore +++ b/.codespellignore @@ -1,2 +1,3 @@ ges cach +fo diff --git a/earthaccess/__init__.py b/earthaccess/__init__.py index 34da7e41d..55877714d 100644 --- a/earthaccess/__init__.py +++ b/earthaccess/__init__.py @@ -25,7 +25,7 @@ from .services import DataServices from .store import Store from .system import PROD, UAT -from .virtual import virtualize +from .virtual import open_virtual, virtualize logger = logging.getLogger(__name__) @@ -58,6 +58,7 @@ # store.py "Store", # virtual + "open_virtual", "virtualize", # system.py "PROD", @@ -73,7 +74,6 @@ _auth = Auth() _store: Store | None = None -_lock = threading.Lock() def __getattr__(name): # type: ignore diff --git a/earthaccess/results.py b/earthaccess/results.py index bf3cb2038..a36f86400 100644 --- a/earthaccess/results.py +++ b/earthaccess/results.py @@ -5,6 +5,7 @@ from typing import Any import requests +import s3fs import earthaccess @@ -291,6 +292,53 @@ def s3_bucket(self) -> dict[str, Any]: return self["umm"].get("DirectDistributionInformation", {}) + def virtual_collection_url(self) -> str | None: + """Return the VIRTUAL COLLECTION URL from the collection's RelatedUrls. + + Returns: + The URL of the virtual store reference file (e.g. a .json or + .icechunk file), or ``None`` if no such link exists. + """ + for link in self["umm"].get("RelatedUrls", []): + if ( + link.get("Type") == "GET DATA" + and link.get("Subtype") == "VIRTUAL COLLECTION" + ): + return link["URL"] + return None + + def get_s3_credentials(self) -> dict[str, str]: + """Return temporary S3 credentials for this collection. + + Returns: + A dictionary with ``accessKeyId``, ``secretAccessKey``, + and ``sessionToken``. + + Raises: + ValueError: If the collection has no ``S3CredentialsAPIEndpoint``. + """ + dd = self["umm"].get("DirectDistributionInformation", {}) + endpoint = dd.get("S3CredentialsAPIEndpoint") + if not endpoint: + raise ValueError( + "This collection does not provide an S3CredentialsAPIEndpoint.", + ) + return earthaccess.__auth__.get_s3_credentials(endpoint=endpoint) + + def get_s3_filesystem(self) -> s3fs.S3FileSystem: + """Return an authenticated ``s3fs.S3FileSystem`` for this collection. + + Returns: + An ``s3fs.S3FileSystem`` configured with temporary S3 credentials + from the collection's credentials endpoint. + """ + creds = self.get_s3_credentials() + return s3fs.S3FileSystem( + key=creds["accessKeyId"], + secret=creds["secretAccessKey"], + token=creds["sessionToken"], + ) + def services(self) -> dict[Any, list[dict[str, Any]]]: """Return list of services available for this collection.""" warnings.warn( diff --git a/earthaccess/virtual/__init__.py b/earthaccess/virtual/__init__.py index 10c69492c..6a0002d16 100644 --- a/earthaccess/virtual/__init__.py +++ b/earthaccess/virtual/__init__.py @@ -1,11 +1,16 @@ """Virtual dataset utilities for cloud-native access to NASA Earthdata granules. This subpackage provides tools for creating virtual xarray Datasets from -NASA Earthdata granules without downloading data, using VirtualiZarr parsers. +NASA Earthdata granules without downloading data, using VirtualiZarr parsers, +and for opening existing virtual stores (Icechunk / kerchunk references). Public API ---------- - ``virtualize`` — create a virtual (or loaded) xarray Dataset from granules. +- ``open_virtual`` — open an existing virtual store (Icechunk, kerchunk .parquet + or .json references) from a URI. +- ``homogenize_dataset_codec_level`` — patch Zlib codec levels on all + ``ManifestArray`` objects in a virtual dataset (power-users). - ``SUPPORTED_PARSERS`` — frozenset of recognised parser name strings. - ``get_granule_credentials_endpoint_and_region`` — resolve S3 credentials for a granule (useful for advanced direct-access workflows). @@ -18,11 +23,16 @@ from earthaccess.virtual._credentials import ( get_granule_credentials_endpoint_and_region, ) -from earthaccess.virtual._parser import SUPPORTED_PARSERS -from earthaccess.virtual.core import virtualize +from earthaccess.virtual._parser import ( + SUPPORTED_PARSERS, + homogenize_dataset_codec_level, +) +from earthaccess.virtual.core import open_virtual, virtualize __all__ = [ "SUPPORTED_PARSERS", "get_granule_credentials_endpoint_and_region", + "homogenize_dataset_codec_level", + "open_virtual", "virtualize", ] diff --git a/earthaccess/virtual/_parser.py b/earthaccess/virtual/_parser.py index 3351fb9da..b76271cab 100644 --- a/earthaccess/virtual/_parser.py +++ b/earthaccess/virtual/_parser.py @@ -12,6 +12,8 @@ from earthaccess.virtual._types import AccessType, ParserType if TYPE_CHECKING: + import xarray as xr + import earthaccess from earthaccess.virtual._types import AccessType, ParserType @@ -131,3 +133,69 @@ def get_urls_for_parser( url = url + ".dmrpp" urls.append(url) return urls + + +def homogenize_dataset_codec_level(ds: xr.Dataset, target_level: int = 7) -> xr.Dataset: + """Patch Zlib codec levels on all ManifestArrays in *ds* to *target_level*.""" + try: + import xarray as xr + from virtualizarr.manifests import ManifestArray + except ImportError: + raise ImportError( + "earthaccess.virtualize() requires `pip install earthaccess[virtualizarr]`", + ) from None + + def _patch_ma(ma: ManifestArray, level: int) -> ManifestArray: + if not isinstance(ma, ManifestArray): + return ma + + meta = ma.metadata + codecs = list(meta.codecs) + modified = False + + for i, codec in enumerate(codecs): + if not hasattr(codec, "to_dict"): + continue + cd = codec.to_dict() + if cd.get("name") != "numcodecs.zlib": + continue + cfg = cd.get("configuration", {}) + if cfg.get("level") == level: + continue + cd["configuration"] = {**cfg, "level": level} + codecs[i] = type(codec).from_dict(cd) + modified = True + + if not modified: + return ma + + import zarr + + new_meta = zarr.core.metadata.v3.ArrayV3Metadata( + shape=meta.shape, + data_type=meta.data_type, + chunk_grid=meta.chunk_grid, + chunk_key_encoding=meta.chunk_key_encoding, + fill_value=meta.fill_value, + codecs=tuple(codecs), + attributes=meta.attributes, + dimension_names=meta.dimension_names, + storage_transformers=meta.storage_transformers, + ) + return ManifestArray(new_meta, ma.manifest) + + new_vars = { + name: ( + xr.DataArray( + _patch_ma(var.data, target_level), + dims=var.dims, + attrs=var.attrs, + name=name, + ) + if isinstance(var.data, ManifestArray) + else var + ) + for name, var in ds.data_vars.items() + } + + return xr.Dataset(new_vars, coords=ds.coords, attrs=ds.attrs) diff --git a/earthaccess/virtual/core.py b/earthaccess/virtual/core.py index 8e5620531..5f2615363 100644 --- a/earthaccess/virtual/core.py +++ b/earthaccess/virtual/core.py @@ -1,11 +1,12 @@ -"""Core implementation of ``earthaccess.virtualize()``. +"""Core implementation of ``earthaccess.virtualize()`` and ``earthaccess.open_virtual()``. -This module contains the single public entry point for creating virtual -xarray Datasets from NASA Earthdata granules. +This module contains public entry points for creating virtual xarray +Datasets from NASA Earthdata granules and opening existing virtual stores. """ from __future__ import annotations +import json import logging import tempfile import warnings @@ -38,7 +39,7 @@ def virtualize( granules: list[earthaccess.DataGranule], *, - access: AccessType = "direct", + access: AccessType = "indirect", load: bool = False, group: str = "/", concat_dim: str | None = None, @@ -68,8 +69,9 @@ def virtualize( Parameters: granules: One or more ``DataGranule`` objects from ``earthaccess.search_data()``. - access: Cloud access mode. ``"direct"`` uses S3 (fastest inside AWS - us-west-2); ``"indirect"`` uses HTTPS (works anywhere). + access: Cloud access mode. ``"indirect"`` (default) uses HTTPS + (works anywhere); ``"direct"`` uses S3 (fastest inside AWS + us-west-2 but requires S3 credentials). load: When ``False`` (default) returns a virtual dataset with ``ManifestArray`` variables. When ``True`` materialises the references via a kerchunk round-trip and returns a concrete, @@ -296,3 +298,443 @@ def _load_via_kerchunk( engine="kerchunk", storage_options=storage_options, ) + + +# --------------------------------------------------------------------------- +# open_virtual — open existing virtual stores (Icechunk / VirtualiZarr) +# --------------------------------------------------------------------------- + + +def _is_icechunk_uri(uri: str) -> bool: + return uri.startswith("icechunk://") or uri.endswith(".icechunk") + + +def _is_kerchunk_uri(uri: str) -> bool: + return uri.endswith((".parquet", ".json")) + + +def _open_icechunk( + uri: str, + storage_options: dict[str, Any] | None = None, + **kwargs: Any, +) -> xr.Dataset: + try: + import icechunk + import xarray as xr + except ImportError as exc: + raise ImportError( + "earthaccess.open_virtual() with an Icechunk store requires " + "`pip install earthaccess[virtualizarr]`", + ) from exc + + if storage_options: + storage = icechunk.Storage(**storage_options) + else: + storage = icechunk.Storage.filesystem(uri) + + store = icechunk.open(storage=storage, mode="r") + return xr.open_zarr(store, **kwargs) + + +def _open_kerchunk( + uri: str, + storage_options: dict[str, Any] | None = None, + **kwargs: Any, +) -> xr.Dataset: + try: + import xarray as xr + except ImportError as exc: + raise ImportError( + "earthaccess.open_virtual() requires `pip install earthaccess[virtualizarr]`", + ) from exc + + store_opts = storage_options or {} + return xr.open_dataset(uri, engine="kerchunk", storage_options=store_opts, **kwargs) + + +# --------------------------------------------------------------------------- +# force_external — download kerchunk refs and rewrite s3:// URLs to https:// +# --------------------------------------------------------------------------- + + +def _transform_refs(obj: Any, https_base: str) -> None: + """Recursively walk a kerchunk refs dict/list and rewrite s3:// URLs in-place.""" + prefix = "s3://" + if isinstance(obj, dict): + for k, v in obj.items(): + if isinstance(v, str) and v.startswith(prefix): + obj[k] = https_base + v[len(prefix) :] + elif isinstance(v, (dict, list)): + _transform_refs(v, https_base) + elif isinstance(obj, list): + for i, v in enumerate(obj): + if isinstance(v, str) and v.startswith(prefix): + obj[i] = https_base + v[len(prefix) :] + elif isinstance(v, (dict, list)): + _transform_refs(v, https_base) + + +def _sanitize_references_for_external(url: str) -> str: + """Download a kerchunk reference file, rewrite ``s3://`` URLs to ``https://``. + + The HTTPS base is inferred from the reference file URL's host, which works + for all NASA Earthdata Cloud DAACs (PODAAC, NSIDC, GES DISC, LP DAAC, …). + + Returns the local path to the sanitized file. + """ + from urllib.parse import urlparse + + parsed = urlparse(url) + if not parsed.scheme: + return url + + local = Path(tempfile.gettempdir()) / f"external_{Path(url).name}" + + if local.exists(): + return str(local) + + host = parsed.netloc + https_base = f"https://{host}/" + fs = earthaccess.get_fsspec_https_session() + + if url.endswith(".json"): + with fs.open(url, cache="first") as f: + refs: dict[str, Any] = json.load(f) + _transform_refs(refs, https_base) + local.write_text(json.dumps(refs)) + + elif url.endswith(".parquet"): + import pandas as pd + + with fs.open(url, cache="first") as f: + refs_df = pd.read_parquet(f) + if "path" in refs_df.columns: + mask = refs_df["path"].str.startswith("s3://", na=False) + refs_df.loc[mask, "path"] = ( + https_base + refs_df.loc[mask, "path"].str[len("s3://") :] + ) + refs_df.to_parquet(local) + + return str(local) + + +def _open_kerchunk_from_collection( + collection: earthaccess.DataCollection, + url: str, + access: str = "indirect", + **kwargs: Any, +) -> xr.Dataset: + try: + import fsspec + import xarray as xr + import zarr + except ImportError as exc: + raise ImportError( + "earthaccess.open_virtual() requires `pip install earthaccess[virtualizarr]`", + ) from exc + + if access == "direct": + daac_fs = collection.get_s3_filesystem() + remote_protocol = "s3" + else: + daac_fs = earthaccess.get_fsspec_https_session() + remote_protocol = "https" + + remote_options = {"asynchronous": True, **daac_fs.storage_options} + fs = fsspec.filesystem( + "reference", + fo=url, + remote_protocol=remote_protocol, + asynchronous=True, + remote_options=remote_options, + ) + store = zarr.storage.FsspecStore(fs, read_only=True) + return xr.open_zarr(store, consolidated=False, **kwargs) + + +def _open_icechunk_from_collection( + collection: earthaccess.DataCollection, + url: str, + access: str = "indirect", + **kwargs: Any, +) -> xr.Dataset: + try: + from datetime import UTC, datetime, timedelta + + import icechunk + import xarray as xr + except ImportError as exc: + raise ImportError( + "earthaccess.open_virtual() with an Icechunk store requires " + "`pip install earthaccess[virtualizarr]`", + ) from exc + + if access == "direct": + creds = collection.get_s3_credentials() + ice_creds = icechunk.S3StaticCredentials( + access_key_id=creds["accessKeyId"], + secret_access_key=creds["secretAccessKey"], + session_token=creds["sessionToken"], + expires_after=datetime.now(UTC) + timedelta(hours=1), + ) + storage = icechunk.S3Storage(url, get_credentials=lambda: ice_creds) + else: + storage = icechunk.Storage.filesystem(url) + + repo = icechunk.Repository.open(storage=storage) + session = repo.readonly_session("main") + store = session.store + return xr.open_zarr(store, **kwargs) + + +# --------------------------------------------------------------------------- +# open_virtual via VirtualiZarr (load=False) +# --------------------------------------------------------------------------- + + +def _is_nasa_url(url: str) -> bool: + """Return ``True`` if the URL belongs to a NASA Earthdata host.""" + return "nasa.gov" in url.lower() + + +def _build_registry_for_url(url: str) -> Any: + """Build an ``ObjectStoreRegistry`` for the given reference file *url*. + + A ``LocalStore`` for ``file://`` is always registered so that local + reference files can be read. For remote URLs an authenticated + ``HTTPStore`` is also registered so that referenced data files can + be resolved with the user's EDL credentials. + """ + from urllib.parse import urlparse + + try: + from obspec_utils.registry import ObjectStoreRegistry + from obstore.store import HTTPStore, LocalStore + except ImportError: + try: + from virtualizarr.registry import ObjectStoreRegistry + except ImportError: + raise ImportError( + "earthaccess.open_virtual(load=False) requires " + "`pip install earthaccess[virtualizarr]`", + ) from None + + stores: dict[str, Any] = {"file://": LocalStore.from_url("file:///")} + + parsed = urlparse(url) + if parsed.scheme == "https": + try: + token = earthaccess.__auth__.token["access_token"] + except (AttributeError, TypeError, KeyError): + pass + else: + http_store = HTTPStore.from_url( + f"https://{parsed.netloc}", + client_options={ + "default_headers": {"Authorization": f"Bearer {token}"}, + }, + ) + stores[f"https://{parsed.netloc}"] = http_store + + if not parsed.scheme or parsed.scheme == "file": + path = Path(parsed.path).resolve() + parent = path.parent if path.suffix else path + file_prefix = parent.as_uri() + stores[file_prefix] = LocalStore.from_url(file_prefix) + + return ObjectStoreRegistry(stores) + + +def _download_reference_file(url: str) -> str: + """Download a remote reference file to a local cache (``/tmp/cached_*``). + + Returns the local path. Already-local files are returned as-is. + """ + from urllib.parse import urlparse + + parsed = urlparse(url) + if not parsed.scheme: + return url + + local = Path(tempfile.gettempdir()) / f"cached_{Path(url).name}" + if local.exists(): + return str(local) + + if _is_nasa_url(url): + fs = earthaccess.get_fsspec_https_session() + else: + import fsspec + + fs = fsspec.filesystem("https") + + with fs.open(url) as src: + local.write_bytes(src.read()) + return str(local) + + +def _open_virtual_via_virtualizarr( + url: str, + *, + registry_url: str | None = None, + **kwargs: Any, +) -> xr.Dataset: + """Open a kerchunk reference file using VirtualiZarr (``load=False``). + + First the reference file is opened via an fsspec reference filesystem + to load inline coordinate values, then VirtualiZarr virtualises the + remaining data variables and the two results are merged. + + An ``ObjectStoreRegistry`` is automatically configured from + *registry_url* (defaults to *url*) so that referenced data files can + be resolved with the user's EDL credentials. + + For JSON files the reference file is first downloaded to a local cache; + parquet files are read directly from their URL. + """ + try: + import fsspec + import virtualizarr as vz + import xarray as xr + import zarr + from virtualizarr.parsers import KerchunkJSONParser, KerchunkParquetParser + except ImportError as exc: + raise ImportError( + "earthaccess.open_virtual(load=False) requires " + "`pip install earthaccess[virtualizarr]`", + ) from exc + + auth_url = registry_url or url + registry = _build_registry_for_url(auth_url) + + ref_path = _download_reference_file(url) if url.endswith(".json") else url + + daac_fs = ( + earthaccess.get_fsspec_https_session() + if _is_nasa_url(auth_url) + else fsspec.filesystem("https") + ) + remote_options = {"asynchronous": True, **daac_fs.storage_options} + fs = fsspec.filesystem( + "reference", + fo=ref_path, + remote_protocol="https", + asynchronous=True, + remote_options=remote_options, + ) + store = zarr.storage.FsspecStore(fs, read_only=True) + kds = xr.open_zarr(store, consolidated=False) + + parser: KerchunkJSONParser | KerchunkParquetParser + if url.endswith(".json"): + parser = KerchunkJSONParser(skip_variables=list(kds.coords)) + elif url.endswith(".parquet"): + parser = KerchunkParquetParser(skip_variables=list(kds.coords)) + else: + raise ValueError( + f"Unsupported virtual store format: {url}. " + "Expected a .json or .parquet file.", + ) + + vds = vz.open_virtual_dataset(ref_path, parser=parser, registry=registry, **kwargs) + + for k in kds.coords: + vds.coords[k] = kds[k] + return vds + + +def open_virtual( # noqa: PLR0911 + uri: str | Path | earthaccess.DataCollection, + *, + access: str = "indirect", + storage_options: dict[str, Any] | None = None, + force_external: bool = False, + load: bool = True, + **kwargs: Any, +) -> xr.Dataset: + """Open a URI or collection as a virtual xarray Dataset. + + Supports two kinds of virtual stores: + + - **Icechunk** — a versioned Zarr store (``.icechunk`` file/URI). + - **VirtualiZarr / kerchunk** — reference-file-backed datasets + (``.parquet`` or ``.json`` files). + + When given a ``DataCollection``, the virtual store URL is extracted from + its metadata (``GET DATA`` + ``VIRTUAL COLLECTION`` subtype). + + Parameters: + uri: A ``DataCollection``, or a path/URI to the virtual store + (``.icechunk``, ``.parquet``, or ``.json``). + access: ``"indirect"`` (HTTPS, default) or ``"direct"`` (S3). + storage_options: Additional options forwarded to the storage backend. + Ignored when ``uri`` is a ``DataCollection``. + force_external: When ``True``, download kerchunk reference files and + rewrite ``s3://`` URLs to ``https://``, so the dataset can be + opened without direct S3 access. Only applies to ``.json`` and + ``.parquet`` reference files. Requires authentication. + load: When ``True`` (default), returns a concrete lazily-loaded dataset + via the kerchunk engine. When ``False``, returns a virtual dataset + backed by ``ManifestArray`` objects via VirtualiZarr's + ``open_virtual_dataset``. + **kwargs: Additional keyword arguments forwarded to the opener. + + Returns: + An ``xr.Dataset`` backed by the virtual store. + + Raises: + ValueError: If the URI is not recognised, or the collection has no + virtual collection URL. + ImportError: If the required optional dependency is not installed. + AttributeError: If the user is not authenticated. + + Examples: + >>> import earthaccess + >>> ds = earthaccess.open_virtual("s3://bucket/refs.parquet") + >>> ds = earthaccess.open_virtual("/local/store.icechunk") + >>> ds = earthaccess.open_virtual(collection) + >>> ds = earthaccess.open_virtual(collection, force_external=True) + """ + if isinstance(uri, earthaccess.DataCollection): + url = uri.virtual_collection_url() + if url is None: + raise ValueError( + f"Collection {uri.get('meta', {}).get('concept-id', '')} " + "does not have a virtual store (no VIRTUAL COLLECTION " + "URL found in its RelatedUrls).", + ) + if not _is_icechunk_uri(url) and not _is_kerchunk_uri(url): + raise ValueError( + f"Unrecognised virtual store URL in collection: {url}. " + "Expected a .icechunk, .parquet, or .json file.", + ) + else: + url = str(uri) + + if not _is_icechunk_uri(url) and not _is_kerchunk_uri(url): + raise ValueError( + f"Unrecognised virtual store URI: {uri}. " + "Expected a .icechunk, .parquet, or .json file/URI.", + ) + + if _is_icechunk_uri(url): + if isinstance(uri, earthaccess.DataCollection): + return _open_icechunk_from_collection(uri, url, access=access, **kwargs) + return _open_icechunk(url, storage_options=storage_options, **kwargs) + + if not load: + if force_external: + sanitized = _sanitize_references_for_external(url) + return _open_virtual_via_virtualizarr(sanitized, registry_url=url, **kwargs) + return _open_virtual_via_virtualizarr(url, **kwargs) + + if force_external: + sanitized = _sanitize_references_for_external(url) + sopts = { + "remote_protocol": "https", + "remote_options": earthaccess.get_fsspec_https_session().storage_options, + } + return _open_kerchunk(sanitized, storage_options=sopts, **kwargs) + + if isinstance(uri, earthaccess.DataCollection): + return _open_kerchunk_from_collection(uri, url, access=access, **kwargs) + return _open_kerchunk(url, storage_options=storage_options, **kwargs) diff --git a/pyproject.toml b/pyproject.toml index 4e6b47ba1..9b221b66c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,8 @@ kerchunk = [ virtualizarr = [ "numpy >=1.26.4", "zarr >=3.1.1", - "virtualizarr >=2.1.2", + "virtualizarr", + "icechunk", "kerchunk>=0.2.9", "dask", "h5py >=3.14.0", @@ -86,6 +87,8 @@ virtualizarr = [ "fastparquet >= 2023.4.0", "numcodecs >= 0.16.2", "xarray", + "matplotlib", + "hvplot", ] [dependency-groups] diff --git a/tests/integration/test_virtualizarr.py b/tests/integration/test_virtualizarr.py index aa7f66b26..c5b55d52b 100644 --- a/tests/integration/test_virtualizarr.py +++ b/tests/integration/test_virtualizarr.py @@ -59,3 +59,56 @@ def test_virtualize_non_materialize(granules): # we are not materializing the data for name in vds.data_vars: assert isinstance(vds[name].variable.data, ManifestArray) + + +MUR_COLLECTION_CONCEPT_ID = "C1996881146-POCLOUD" +MUR_VIRTUAL_URL = ( + "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-docs/ghrsst/open/" + "docs/MUR-JPL-L4-GLOB-v4.1_combined-ref.json" +) + + +def test_open_virtual_from_collection(): + """open_virtual(DataCollection) opens the MUR virtual store via HTTPS.""" + collection = earthaccess.search_datasets( + count=1, concept_id=MUR_COLLECTION_CONCEPT_ID + )[0] + vds = earthaccess.open_virtual(collection, access="indirect") + assert vds is not None + assert len(vds.dims) > 0 + + +def test_open_virtual_from_url(): + """open_virtual(str) opens the MUR virtual store URL via kerchunk engine.""" + vds = earthaccess.open_virtual(MUR_VIRTUAL_URL) + assert vds is not None + assert len(vds.dims) > 0 + + +def test_open_virtual_load_false_from_collection(): + """open_virtual(collection, load=False) returns a virtual dataset with ManifestArrays.""" + from virtualizarr.manifests.array import ManifestArray + + collection = earthaccess.search_datasets( + count=1, concept_id=MUR_COLLECTION_CONCEPT_ID + )[0] + vds = earthaccess.open_virtual(collection, load=False, access="indirect") + assert vds is not None + assert len(vds.dims) > 0 + for name in vds.data_vars: + assert isinstance(vds[name].variable.data, ManifestArray) + + +def test_open_virtual_load_false_external_url(): + """open_virtual(external URL, load=False) reads a public kerchunk reference.""" + from virtualizarr.manifests.array import ManifestArray + + external_url = ( + "https://its-live-data.s3-us-west-2.amazonaws.com/test-space/vds/" + "SPL4SMGP.parquet" + ) + vds = earthaccess.open_virtual(external_url, load=False) + assert vds is not None + assert len(vds.dims) > 0 + for name in vds.data_vars: + assert isinstance(vds[name].variable.data, ManifestArray) diff --git a/tests/unit/test_virtual.py b/tests/unit/test_virtual.py index b5a09f8f7..1bf321ca2 100644 --- a/tests/unit/test_virtual.py +++ b/tests/unit/test_virtual.py @@ -1,9 +1,9 @@ """Unit tests for earthaccess.virtual. -Covers the three public-facing modules: - - core (virtualize / _load_via_kerchunk) - - _parser (SUPPORTED_PARSERS / resolve_parser / get_urls_for_parser) - - _credentials (get_granule_credentials_endpoint_and_region) +Covers the high-level public API: + - virtualize() + - open_virtual() (str, DataCollection, load, force_external paths) + - DataCollection.virtual_collection_url() / get_s3_credentials() All external I/O is mocked so the suite runs without network access or optional heavy dependencies. @@ -56,12 +56,11 @@ def _patch_internals(mock_vds: MagicMock | None = None): # --------------------------------------------------------------------------- -# core — input validation +# virtualize # --------------------------------------------------------------------------- def test_virtualize_empty_granules_raises() -> None: - """virtualize() raises ValueError when granules list is empty.""" from earthaccess.virtual.core import virtualize with pytest.raises(ValueError, match=r"[Nn]o granules"): @@ -69,7 +68,6 @@ def test_virtualize_empty_granules_raises() -> None: def test_virtualize_multi_granule_no_concat_dim_raises() -> None: - """virtualize() raises ValueError for >1 granule without concat_dim.""" from earthaccess.virtual.core import virtualize with ( @@ -83,20 +81,13 @@ def test_virtualize_multi_granule_no_concat_dim_raises() -> None: def test_virtualize_invalid_parser_string_raises() -> None: - """virtualize() raises ValueError for an unrecognised parser string.""" from earthaccess.virtual.core import virtualize with pytest.raises(ValueError, match="BadParser"): virtualize(_make_granules(1), parser="BadParser") -# --------------------------------------------------------------------------- -# core — happy paths -# --------------------------------------------------------------------------- - - def test_virtualize_load_false_returns_virtual_dataset() -> None: - """virtualize(load=False) returns the raw virtual dataset without calling kerchunk.""" from earthaccess.virtual.core import virtualize mock_vds = MagicMock() @@ -113,7 +104,6 @@ def test_virtualize_load_false_returns_virtual_dataset() -> None: def test_virtualize_load_true_delegates_to_kerchunk(tmp_path) -> None: - """virtualize(load=True) calls _load_via_kerchunk and returns its result.""" from earthaccess.virtual.core import virtualize expected_ds = MagicMock() @@ -136,13 +126,7 @@ def test_virtualize_load_true_delegates_to_kerchunk(tmp_path) -> None: assert result is expected_ds -# --------------------------------------------------------------------------- -# core — DMR++ fallback behaviour -# --------------------------------------------------------------------------- - - def test_virtualize_dmrpp_fallback_emits_user_warning() -> None: - """When DMR++ sidecars are missing, virtualize() warns and retries with HDFParser.""" from earthaccess.virtual.core import virtualize mock_vds_hdf = MagicMock() @@ -172,193 +156,499 @@ def side_effect(*args, **kwargs): # --------------------------------------------------------------------------- -# _parser — SUPPORTED_PARSERS +# open_virtual — str URL routing # --------------------------------------------------------------------------- -def test_supported_parsers_contains_canonical_names() -> None: - """SUPPORTED_PARSERS includes the three canonical parser names.""" - from earthaccess.virtual._parser import SUPPORTED_PARSERS +def test_open_virtual_unrecognised_uri_raises() -> None: + from earthaccess.virtual.core import open_virtual - assert isinstance(SUPPORTED_PARSERS, frozenset) - assert {"DMRPPParser", "HDFParser", "NetCDF3Parser"} <= SUPPORTED_PARSERS + with pytest.raises(ValueError, match="Unrecognised"): + open_virtual("data.nc") -# --------------------------------------------------------------------------- -# _parser — resolve_parser -# --------------------------------------------------------------------------- +def test_open_virtual_icechunk_delegates(tmp_path) -> None: + from earthaccess.virtual.core import open_virtual + mock_ds = MagicMock() + icechunk_path = tmp_path / "store.icechunk" + icechunk_path.write_text("") -def test_resolve_parser_dmrpp_returns_instance() -> None: - """resolve_parser('DMRPPParser') returns a DMRPPParser instance.""" - from earthaccess.virtual._parser import resolve_parser + with patch( + "earthaccess.virtual.core._open_icechunk", + return_value=mock_ds, + ) as mock_open: + result = open_virtual(str(icechunk_path)) - assert type(resolve_parser("DMRPPParser")).__name__ == "DMRPPParser" + mock_open.assert_called_once_with(str(icechunk_path), storage_options=None) + assert result is mock_ds -def test_resolve_parser_invalid_string_raises() -> None: - """resolve_parser raises ValueError and lists valid names in the message.""" - from earthaccess.virtual._parser import resolve_parser +def test_open_virtual_kerchunk_json_delegates() -> None: + from earthaccess.virtual.core import open_virtual - with pytest.raises(ValueError, match="DMRPPParser"): - resolve_parser("UnknownParser") + mock_ds = MagicMock() + with patch( + "earthaccess.virtual.core._open_kerchunk", + return_value=mock_ds, + ) as mock_open: + result = open_virtual("refs.json") -# --------------------------------------------------------------------------- -# _parser — get_urls_for_parser -# --------------------------------------------------------------------------- + mock_open.assert_called_once_with("refs.json", storage_options=None) + assert result is mock_ds + + +def test_open_virtual_kerchunk_parquet_delegates() -> None: + from earthaccess.virtual.core import open_virtual + mock_ds = MagicMock() -def test_get_urls_dmrpp_appends_dmrpp_suffix() -> None: - """get_urls_for_parser with DMRPPParser appends '.dmrpp' to each URL.""" - from earthaccess.virtual._parser import get_urls_for_parser, resolve_parser + with patch( + "earthaccess.virtual.core._open_kerchunk", + return_value=mock_ds, + ) as mock_open: + result = open_virtual("refs.parquet", storage_options={"key": "val"}) - granule = cast("DataGranule", MagicMock()) - granule.data_links.return_value = ["s3://bucket/file.nc"] # type: ignore[attr-defined] - urls = get_urls_for_parser( - [granule], - resolve_parser("DMRPPParser"), - access="direct", + mock_open.assert_called_once_with( + "refs.parquet", + storage_options={"key": "val"}, ) - assert urls == ["s3://bucket/file.nc.dmrpp"] + assert result is mock_ds -def test_get_urls_passes_access_to_data_links() -> None: - """get_urls_for_parser forwards the access argument to granule.data_links.""" - from earthaccess.virtual._parser import get_urls_for_parser, resolve_parser +def test_open_virtual_forwards_storage_options_to_icechunk(tmp_path) -> None: + from earthaccess.virtual.core import open_virtual + + mock_ds = MagicMock() + icechunk_path = tmp_path / "store.icechunk" + icechunk_path.write_text("") + opts = {"some": "option"} + + with patch( + "earthaccess.virtual.core._open_icechunk", + return_value=mock_ds, + ) as mock_open: + result = open_virtual( + str(icechunk_path), + storage_options=opts, + ) - mock = MagicMock() - mock.data_links.return_value = ["https://example.com/file.h5"] - granule = cast("DataGranule", mock) - get_urls_for_parser([granule], resolve_parser("HDFParser"), access="indirect") - mock.data_links.assert_called_once_with(access="indirect") + mock_open.assert_called_once_with(str(icechunk_path), storage_options=opts) + assert result is mock_ds # --------------------------------------------------------------------------- -# _credentials — get_granule_credentials_endpoint_and_region -# Uses real DataGranule / DataCollection objects (no MagicMock stand-ins). +# DataCollection — virtual_collection_url / get_s3_credentials # --------------------------------------------------------------------------- -_granule_no_endpoint = DataGranule( - { - "meta": {"collection-concept-id": "C1234-PROV"}, - "umm": { - "RelatedUrls": [ - {"URL": "https://data.earthdata.nasa.gov/data.h5", "Type": "GET DATA"}, - ], - }, - }, - cloud_hosted=True, -) - -@patch("earthaccess.search_datasets") -def test_credentials_endpoint_from_granule(mock_search_datasets) -> None: - """Endpoint embedded in the granule UMM-G record is used directly.""" - from earthaccess.virtual._credentials import ( - get_granule_credentials_endpoint_and_region, +def test_collection_virtual_url_found() -> None: + collection = DataCollection( + { + "meta": {"concept-id": "C1234-PROV"}, + "umm": { + "RelatedUrls": [ + { + "URL": "https://example.com/refs.json", + "Type": "GET DATA", + "Subtype": "VIRTUAL COLLECTION", + }, + { + "URL": "https://example.com/data.nc", + "Type": "GET DATA", + }, + ], + }, + }, ) + assert collection.virtual_collection_url() == "https://example.com/refs.json" - endpoint_url = "https://archive.daac.earthdata.nasa.gov/s3credentials" - granule = DataGranule( + +def test_collection_virtual_url_not_found() -> None: + collection = DataCollection( { - "meta": {"collection-concept-id": "C1234-PROV"}, + "meta": {"concept-id": "C1234-PROV"}, "umm": { "RelatedUrls": [ { - "URL": "https://data.earthdata.nasa.gov/data.h5", + "URL": "https://example.com/data.nc", "Type": "GET DATA", }, + ], + }, + }, + ) + assert collection.virtual_collection_url() is None + + +def test_collection_virtual_url_no_related_urls() -> None: + collection = DataCollection( + { + "meta": {"concept-id": "C1234-PROV"}, + "umm": {}, + }, + ) + assert collection.virtual_collection_url() is None + + +def test_collection_get_s3_credentials() -> None: + collection = DataCollection( + { + "meta": {"concept-id": "C1234-PROV"}, + "umm": { + "DirectDistributionInformation": { + "S3CredentialsAPIEndpoint": "https://example.com/s3credentials", + "Region": "us-west-2", + }, + }, + }, + ) + mock_creds = { + "accessKeyId": "AKIA...", + "secretAccessKey": "secret...", + "sessionToken": "token...", + } + with patch( + "earthaccess.results.earthaccess.__auth__", + autospec=True, + ) as mock_auth: + mock_auth.get_s3_credentials.return_value = mock_creds + result = collection.get_s3_credentials() + + mock_auth.get_s3_credentials.assert_called_once_with( + endpoint="https://example.com/s3credentials", + ) + assert result == mock_creds + + +def test_collection_get_s3_credentials_no_endpoint() -> None: + collection = DataCollection( + { + "meta": {"concept-id": "C1234-PROV"}, + "umm": { + "DirectDistributionInformation": { + "Region": "us-west-2", + }, + }, + }, + ) + with pytest.raises(ValueError, match="S3CredentialsAPIEndpoint"): + collection.get_s3_credentials() + + +# --------------------------------------------------------------------------- +# open_virtual — DataCollection +# --------------------------------------------------------------------------- + + +def test_open_virtual_with_collection_kerchunk() -> None: + from earthaccess.virtual.core import open_virtual + + collection = DataCollection( + { + "meta": {"concept-id": "C1234-PROV"}, + "umm": { + "RelatedUrls": [ { - "URL": "s3://bucket/data.h5", - "Type": "GET DATA VIA DIRECT ACCESS", + "URL": "https://example.com/refs.json", + "Type": "GET DATA", + "Subtype": "VIRTUAL COLLECTION", }, - {"URL": endpoint_url, "Type": "VIEW RELATED INFORMATION"}, ], }, }, - cloud_hosted=True, ) + mock_ds = MagicMock() - assert get_granule_credentials_endpoint_and_region(granule) == ( - endpoint_url, - "us-west-2", + with patch( + "earthaccess.virtual.core._open_kerchunk_from_collection", + return_value=mock_ds, + ) as mock_open: + result = open_virtual(collection) + + mock_open.assert_called_once_with( + collection, "https://example.com/refs.json", access="indirect" ) - mock_search_datasets.assert_not_called() + assert result is mock_ds -@patch("earthaccess.search_datasets") -def test_credentials_endpoint_from_collection(mock_search_datasets) -> None: - """Falls back to the collection record when the granule has no endpoint.""" - from earthaccess.virtual._credentials import ( - get_granule_credentials_endpoint_and_region, - ) +def test_open_virtual_with_collection_icechunk() -> None: + from earthaccess.virtual.core import open_virtual - coll_endpoint = "https://archive.other-daac.earthdata.nasa.gov/s3credentials" - coll_region = "us-east-1" - mock_search_datasets.return_value = [ - DataCollection( - { - "meta": {"concept-id": "C1234-PROV"}, - "umm": { - "DirectDistributionInformation": { - "Region": coll_region, - "S3CredentialsAPIEndpoint": coll_endpoint, + collection = DataCollection( + { + "meta": {"concept-id": "C1234-PROV"}, + "umm": { + "RelatedUrls": [ + { + "URL": "s3://bucket/store.icechunk", + "Type": "GET DATA", + "Subtype": "VIRTUAL COLLECTION", }, - }, + ], }, - ), - ] + }, + ) + mock_ds = MagicMock() + + with patch( + "earthaccess.virtual.core._open_icechunk_from_collection", + return_value=mock_ds, + ) as mock_open: + result = open_virtual(collection, access="direct") - assert get_granule_credentials_endpoint_and_region(_granule_no_endpoint) == ( - coll_endpoint, - coll_region, + mock_open.assert_called_once_with( + collection, "s3://bucket/store.icechunk", access="direct" ) - mock_search_datasets.assert_called_once_with(count=1, concept_id="C1234-PROV") + assert result is mock_ds -@patch("earthaccess.search_datasets") -def test_credentials_collection_missing_region_defaults_to_us_west_2( - mock_search_datasets, -) -> None: - """Region defaults to us-west-2 when the collection record omits it.""" - from earthaccess.virtual._credentials import ( - get_granule_credentials_endpoint_and_region, +def test_open_virtual_with_collection_no_virtual_url() -> None: + from earthaccess.virtual.core import open_virtual + + collection = DataCollection( + { + "meta": {"concept-id": "C1234-PROV"}, + "umm": {}, + }, ) + with pytest.raises(ValueError, match="VIRTUAL COLLECTION"): + open_virtual(collection) + + +# --------------------------------------------------------------------------- +# open_virtual — str path still works +# --------------------------------------------------------------------------- + + +def test_open_virtual_str_path_still_works_after_refactor() -> None: + from earthaccess.virtual.core import open_virtual + + mock_ds = MagicMock() + + with patch( + "earthaccess.virtual.core._open_kerchunk", + return_value=mock_ds, + ) as mock_open: + result = open_virtual("refs.parquet") + + mock_open.assert_called_once_with("refs.parquet", storage_options=None) + assert result is mock_ds + + +# --------------------------------------------------------------------------- +# open_virtual — load=False (VirtualiZarr path) +# --------------------------------------------------------------------------- - coll_endpoint = "https://archive.other-daac.earthdata.nasa.gov/s3credentials" - mock_search_datasets.return_value = [ - DataCollection( - { - "meta": {"concept-id": "C1234-PROV"}, - "umm": { - "DirectDistributionInformation": { - "S3CredentialsAPIEndpoint": coll_endpoint, + +def test_open_virtual_load_false_url() -> None: + from earthaccess.virtual.core import open_virtual + + mock_ds = MagicMock() + + with patch( + "earthaccess.virtual.core._open_virtual_via_virtualizarr", + return_value=mock_ds, + ) as mock_vz: + result = open_virtual("https://example.com/refs.json", load=False) + + assert mock_vz.called + assert result is mock_ds + + +def test_open_virtual_load_false_collection() -> None: + from earthaccess.virtual.core import open_virtual + + collection = DataCollection( + { + "meta": {"concept-id": "C1234-PROV"}, + "umm": { + "RelatedUrls": [ + { + "URL": "https://example.com/refs.json", + "Type": "GET DATA", + "Subtype": "VIRTUAL COLLECTION", }, - }, + ], }, - ), - ] + }, + ) + mock_ds = MagicMock() + + with patch( + "earthaccess.virtual.core._open_virtual_via_virtualizarr", + return_value=mock_ds, + ) as mock_vz: + result = open_virtual(collection, load=False) + + assert mock_vz.called + assert result is mock_ds + + +def test_open_virtual_load_false_external_url() -> None: + from earthaccess.virtual.core import open_virtual + + mock_ds = MagicMock() - _, region = get_granule_credentials_endpoint_and_region(_granule_no_endpoint) - assert region == "us-west-2" + with patch( + "earthaccess.virtual.core._open_virtual_via_virtualizarr", + return_value=mock_ds, + ) as mock_vz: + result = open_virtual( + "https://its-live-data.s3-us-west-2.amazonaws.com/test-space/vds/SPL4SMGP.parquet", + load=False, + ) + + assert mock_vz.called + assert result is mock_ds + + +def test_open_virtual_load_true_default_still_works() -> None: + from earthaccess.virtual.core import open_virtual + + mock_ds = MagicMock() + + with patch( + "earthaccess.virtual.core._open_kerchunk", + return_value=mock_ds, + ) as mock_open: + result = open_virtual("refs.parquet") + + mock_open.assert_called_once_with("refs.parquet", storage_options=None) + assert result is mock_ds + + +# --------------------------------------------------------------------------- +# open_virtual — force_external +# --------------------------------------------------------------------------- -@patch("earthaccess.search_datasets") -def test_credentials_raises_when_no_endpoint_anywhere(mock_search_datasets) -> None: - """ValueError raised when neither granule nor collection has an endpoint.""" - from earthaccess.virtual._credentials import ( - get_granule_credentials_endpoint_and_region, +def test_open_virtual_force_external_url() -> None: + from earthaccess.virtual.core import open_virtual + + remote_url = "https://example.com/data/refs.json" + mock_session = MagicMock() + mock_session.storage_options = { + "client_kwargs": {"headers": {"Authorization": "Bearer x"}} + } + mock_ds = MagicMock() + + with ( + patch( + "earthaccess.virtual.core._sanitize_references_for_external", + return_value="/tmp/external_refs.json", + ) as mock_sanitize, + patch( + "earthaccess.virtual.core._open_kerchunk", + return_value=mock_ds, + ) as mock_open, + patch( + "earthaccess.get_fsspec_https_session", + return_value=mock_session, + ), + ): + result = open_virtual(remote_url, force_external=True) + + mock_sanitize.assert_called_once_with(remote_url) + mock_open.assert_called_once_with( + "/tmp/external_refs.json", + storage_options={ + "remote_protocol": "https", + "remote_options": mock_session.storage_options, + }, ) + assert result is mock_ds + + +def test_open_virtual_force_external_with_collection() -> None: + from earthaccess.virtual.core import open_virtual - mock_search_datasets.return_value = [ - DataCollection( - { - "meta": {"concept-id": "C1234-PROV"}, - "umm": {"DirectDistributionInformation": {"Region": "us-east-1"}}, + collection = DataCollection( + { + "meta": {"concept-id": "C1234-PROV"}, + "umm": { + "RelatedUrls": [ + { + "URL": "https://example.com/data/refs.json", + "Type": "GET DATA", + "Subtype": "VIRTUAL COLLECTION", + }, + ], }, + }, + ) + mock_session = MagicMock() + mock_session.storage_options = { + "client_kwargs": {"headers": {"Authorization": "Bearer x"}} + } + mock_ds = MagicMock() + + with ( + patch( + "earthaccess.virtual.core._sanitize_references_for_external", + return_value="/tmp/external_refs.json", + ) as mock_sanitize, + patch( + "earthaccess.virtual.core._open_kerchunk", + return_value=mock_ds, + ) as mock_open, + patch( + "earthaccess.get_fsspec_https_session", + return_value=mock_session, ), - ] + ): + result = open_virtual(collection, force_external=True) + + mock_sanitize.assert_called_once_with("https://example.com/data/refs.json") + mock_open.assert_called_once_with( + "/tmp/external_refs.json", + storage_options={ + "remote_protocol": "https", + "remote_options": mock_session.storage_options, + }, + ) + assert result is mock_ds + + +def test_open_virtual_force_external_does_not_affect_plain_string() -> None: + from earthaccess.virtual.core import open_virtual + + mock_ds = MagicMock() + + with patch( + "earthaccess.virtual.core._open_kerchunk", + return_value=mock_ds, + ) as mock_open: + result = open_virtual("refs.parquet") + + mock_open.assert_called_once_with("refs.parquet", storage_options=None) + assert result is mock_ds - with pytest.raises(ValueError, match="did not provide an S3CredentialsAPIEndpoint"): - get_granule_credentials_endpoint_and_region(_granule_no_endpoint) + +# --------------------------------------------------------------------------- +# homogenize_dataset_codec_level — data integrity after patching +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("level", [1, 6, 7, 9]) +def test_zlib_decompression_is_level_independent(level: int) -> None: + """Zlib decompression produces the same result regardless of the level param. + + Data compressed at level 6 must decompress correctly with any level in + the codec config. This validates that patching Zlib levels in + ManifestArray metadata is safe. + """ + import numcodecs + import numpy as np + + original = np.array([1.0, np.nan, 3.0, -999.0, 42.5], dtype="f8") + compressed = numcodecs.Zlib(level=6).encode(original.tobytes()) + + result = np.frombuffer( + numcodecs.Zlib(level=level).decode(compressed), + dtype="f8", + ) + assert np.allclose(original, result, equal_nan=True), ( + f"Mismatch at Zlib level {level}" + ) diff --git a/uv.lock b/uv.lock index 8e98bb717..ad510dc57 100644 --- a/uv.lock +++ b/uv.lock @@ -319,6 +319,26 @@ css = [ { name = "tinycss2" }, ] +[[package]] +name = "bokeh" +version = "3.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "contourpy" }, + { name = "jinja2" }, + { name = "narwhals" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "pyyaml" }, + { name = "tornado", marker = "sys_platform != 'emscripten'" }, + { name = "xyzservices" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bf/0d/fabb70707646217e4b0e3943e05730eab8c1f7b7e7485145f8594b52e606/bokeh-3.9.0.tar.gz", hash = "sha256:775219714a8496973ddbae16b1861606ba19fe670a421e4d43267b41148e07a3", size = 5740345, upload-time = "2026-03-11T17:58:34.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/0b/bdf449df87be3f07b23091ceafee8c3ef569cf6d2fb7edec6e3b12b3faa4/bokeh-3.9.0-py3-none-any.whl", hash = "sha256:b252bfb16a505f0e0c57d532d0df308ae1667235bafc622aa9441fe9e7c5ce4a", size = 6396068, upload-time = "2026-03-11T17:58:31.645Z" }, +] + [[package]] name = "botocore" version = "1.39.11" @@ -541,6 +561,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "colorcet" +version = "3.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3f/af/b969f541242b84cbbacabdf20862e487689e352bf0f02f90df2795d29da5/colorcet-3.2.1.tar.gz", hash = "sha256:48d9a67e6e59dc5c0a965aa1b46fe5d59cdc95cc36a95949f29313f950ac59f7", size = 2202958, upload-time = "2026-04-28T16:25:37.43Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/24/e95471ae93c08d3606c9c7343cf65d490f154daa88b50581957a0aa780f4/colorcet-3.2.1-py3-none-any.whl", hash = "sha256:3f6fde13cef2169222dd5fe2a2bf847c02d644470fdf167ed566f6421df470f7", size = 262291, upload-time = "2026-04-28T16:25:35.365Z" }, +] + [[package]] name = "colorlog" version = "6.9.0" @@ -955,7 +984,10 @@ all = [ { name = "fastparquet" }, { name = "h5netcdf" }, { name = "h5py" }, + { name = "hvplot" }, + { name = "icechunk" }, { name = "kerchunk" }, + { name = "matplotlib" }, { name = "numcodecs" }, { name = "numpy" }, { name = "obstore" }, @@ -975,7 +1007,10 @@ virtualizarr = [ { name = "dask" }, { name = "fastparquet" }, { name = "h5py" }, + { name = "hvplot" }, + { name = "icechunk" }, { name = "kerchunk" }, + { name = "matplotlib" }, { name = "numcodecs" }, { name = "numpy" }, { name = "obstore" }, @@ -1051,10 +1086,16 @@ requires-dist = [ { name = "h5py", marker = "extra == 'all'", specifier = ">=3.14.0" }, { name = "h5py", marker = "extra == 'kerchunk'", specifier = ">=3.14.0" }, { name = "h5py", marker = "extra == 'virtualizarr'", specifier = ">=3.14.0" }, + { name = "hvplot", marker = "extra == 'all'" }, + { name = "hvplot", marker = "extra == 'virtualizarr'" }, + { name = "icechunk", marker = "extra == 'all'" }, + { name = "icechunk", marker = "extra == 'virtualizarr'" }, { name = "importlib-resources", specifier = ">=6.3.2" }, { name = "kerchunk", marker = "extra == 'all'", specifier = ">=0.2.9" }, { name = "kerchunk", marker = "extra == 'kerchunk'", specifier = ">=0.2.9" }, { name = "kerchunk", marker = "extra == 'virtualizarr'", specifier = ">=0.2.9" }, + { name = "matplotlib", marker = "extra == 'all'" }, + { name = "matplotlib", marker = "extra == 'virtualizarr'" }, { name = "multimethod", specifier = ">=1.8" }, { name = "numcodecs", marker = "extra == 'all'", specifier = ">=0.16.2" }, { name = "numcodecs", marker = "extra == 'virtualizarr'", specifier = ">=0.16.2" }, @@ -1069,8 +1110,8 @@ requires-dist = [ { name = "tenacity", specifier = ">=8.0" }, { name = "tinynetrc", specifier = ">=1.3.1" }, { name = "typing-extensions", specifier = ">=4.10.0" }, - { name = "virtualizarr", marker = "extra == 'all'", specifier = ">=2.1.2" }, - { name = "virtualizarr", marker = "extra == 'virtualizarr'", specifier = ">=2.1.2" }, + { name = "virtualizarr", marker = "extra == 'all'" }, + { name = "virtualizarr", marker = "extra == 'virtualizarr'" }, { name = "xarray", marker = "extra == 'all'" }, { name = "xarray", marker = "extra == 'all'", specifier = ">=2025.4.0" }, { name = "xarray", marker = "extra == 'kerchunk'", specifier = ">=2025.4.0" }, @@ -1447,6 +1488,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/96/91/9fad90cfc5f9b2489c7c26ad897157bce82f0e9534a986a221b99760b23b/h5py-3.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:faca8fb4e4319c09d83337adc80b2ca7d5c5a343c2d6f1b6388f32cfecca13c1", size = 2740706, upload-time = "2026-03-06T13:49:06.347Z" }, ] +[[package]] +name = "holoviews" +version = "1.22.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "bokeh" }, + { name = "colorcet" }, + { name = "narwhals" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "panel" }, + { name = "param" }, + { name = "python-dateutil" }, + { name = "pyviz-comms" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/e4/40c001bae370f9e89bbcda71c6fb75bd5baa4ec8d5004369a591cfaa0d5d/holoviews-1.22.1.tar.gz", hash = "sha256:7dde3a10bb77aff0982e21786b4f5249c6a9d70c463f5604a49fcf30c0247756", size = 5509590, upload-time = "2025-12-05T14:54:20.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/ab/a6aa43d45ceb88adc0e8c1358fa6935c6e6a5895537431dec67524ca2ccd/holoviews-1.22.1-py3-none-any.whl", hash = "sha256:6f4f0656336035cde1d8103ac6461d7c8ac9a60c4a6d883785cc81f2cc5b8702", size = 5946246, upload-time = "2025-12-05T14:54:19.191Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -1475,6 +1537,66 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "hvplot" +version = "0.12.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "bokeh" }, + { name = "colorcet" }, + { name = "holoviews" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "panel" }, + { name = "param" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/b4/ce73a568fc263efe84cab8de04673949c8f6596f6aeb1531ca2b75b80903/hvplot-0.12.2.tar.gz", hash = "sha256:9e29bbe65f94937d4eeaeccf33566540df936c8f9aeadfa12ea9f306d5237938", size = 7084556, upload-time = "2025-12-18T11:15:37.736Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/cd/ec193e471780dfad60e44ded5526c123695c1354910edd537d7aa1d22094/hvplot-0.12.2-py3-none-any.whl", hash = "sha256:0687e2e4d2eeb035c437af0011922abff856054299c121914d903a02b1bb1b22", size = 180626, upload-time = "2025-12-18T11:15:35.091Z" }, +] + +[[package]] +name = "icechunk" +version = "2.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zarr" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/dd/c38e89707affc0a279d97b3ec4226647a42da2f651a34ab60f262ee4cdb6/icechunk-2.0.4.tar.gz", hash = "sha256:21d789be2cf4612cd8a95cf83ededac3265eb181bc837d9eaf86c50f3e10b7f5", size = 3316217, upload-time = "2026-04-29T22:01:55.715Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/02/5a91854d3fe39f3519e6824d2555c8e571d4021f7852d13311c2455e4261/icechunk-2.0.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:45b9cf2031dcf612eccf840a8709d76616b784d2e2421b4e0dc82df95cbed86e", size = 16797229, upload-time = "2026-04-29T22:02:25.869Z" }, + { url = "https://files.pythonhosted.org/packages/72/51/0c4b8ec694b5a7c3af483ab896fefa2177db5b0a1fdf5695e6b32bc47770/icechunk-2.0.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27e17b3449212ab25624ea1d191271619b8923c95f496634bebc3bff1d208076", size = 15495651, upload-time = "2026-04-29T22:02:16.972Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5a/f2c010d09dd1b6021ba64701990d613a861cc6b80f8bedf01e77e09f3312/icechunk-2.0.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e9a41d0555d03b2e9d1298c6345a9b199b5a84ca6420ab7d2236b5cd60c2808", size = 17180204, upload-time = "2026-04-29T22:02:07.68Z" }, + { url = "https://files.pythonhosted.org/packages/f4/92/4e274304c43391523680b1e03dcff407eb313aa5fc54665c983f7195fd38/icechunk-2.0.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e666631e350dc364cb88835c155b5657ae50aabd4c9fabef3aecaf68cd58060a", size = 16830767, upload-time = "2026-04-29T22:01:46.267Z" }, + { url = "https://files.pythonhosted.org/packages/64/eb/15bdcd61f5f1d871363fe8428047ca0ad3fe4d1d9a44cb0b7742d1a5e0c0/icechunk-2.0.4-cp312-cp312-manylinux_2_28_armv7l.whl", hash = "sha256:8cce3691fa28a25926560f588b75644991438bf64840ac0190b735558db87577", size = 16655187, upload-time = "2026-04-29T22:01:57.854Z" }, + { url = "https://files.pythonhosted.org/packages/f8/6e/4a9da3ecf3435b61b9bca1ed52dd3c3b242bb9fb126302d43677ac3bd3b8/icechunk-2.0.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:abcd43253883abfe95c35e831a67ecb491a014bf908692e4baa5c1e50263ed0c", size = 17041769, upload-time = "2026-04-29T22:02:34.888Z" }, + { url = "https://files.pythonhosted.org/packages/da/4a/1369fd815205dea73fae86e9dc426b600871578590cdb75d3389fd9af9bf/icechunk-2.0.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:aa9bb4e8f198bd69327ea04c7ef6f9ecb6508a2f0e42a50352e52275ccfdb257", size = 16821544, upload-time = "2026-04-29T22:02:44.261Z" }, + { url = "https://files.pythonhosted.org/packages/af/00/06a4dc871350542b4d460c53270f98e621cc46aaef8ade19d601b6c0eda7/icechunk-2.0.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:11b5e68930f2c444f9e84b2b6ed5cc4d1ae2f0c560bd5a919a6ab262d0fe954f", size = 16902987, upload-time = "2026-04-29T22:02:53.363Z" }, + { url = "https://files.pythonhosted.org/packages/e8/00/15b3f47b3a8a0ee5ba00f766bad289e8c9837bacfce671d4d8f3db54be71/icechunk-2.0.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:796bda2ce27e2329bf483eeab94a2c3ae9c80d87771b24a8c95654cbc5e2b9f8", size = 17583401, upload-time = "2026-04-29T22:03:01.902Z" }, + { url = "https://files.pythonhosted.org/packages/00/75/e3ef1fbfeecc97b59061dd833c1df6796881567fd1e132e7d72fa687d565/icechunk-2.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:b055ea9cfd6eb0f71afed9fb7df81eba85bfd4b1cc52b3615f9c7b056f5a9165", size = 15897330, upload-time = "2026-04-29T22:03:12.14Z" }, + { url = "https://files.pythonhosted.org/packages/75/9f/dba7e96f90b38961a875cc100f79be0d63274d1fa72d54a3f45ec67ace18/icechunk-2.0.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:20543ae093a4a34d54f36ceee0eea73185c3679b9c1ad541ee056baae31394ec", size = 16796797, upload-time = "2026-04-29T22:02:28.92Z" }, + { url = "https://files.pythonhosted.org/packages/89/50/393d04f9e59bfc7577fdbff1cfff9c2564ba34dc8403e107dc2f81c11ed0/icechunk-2.0.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:321077a01b842dfc86bd3c14d9abe46f4e50112e4eb5d1f00dc82e19893822e4", size = 15494880, upload-time = "2026-04-29T22:02:19.915Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/ed7147dac5f0768a3dbb72be399f3346553252c7627c6560ae40655f31cf/icechunk-2.0.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fbdae59b438dea1da1caaa1860c26fca696afd1547a89b1bb66d90a21f4f8337", size = 17179693, upload-time = "2026-04-29T22:02:10.594Z" }, + { url = "https://files.pythonhosted.org/packages/4d/ac/c49b8c2f611f4eae9eecf1893c0f6c3f4111572d27b79ed307b89c67714d/icechunk-2.0.4-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:5690ffea3f5966e18273a4fe64041038cd43afe01d40067d36640ef24e07192a", size = 16830182, upload-time = "2026-04-29T22:01:49.787Z" }, + { url = "https://files.pythonhosted.org/packages/e9/15/082f9c9ced590fc2daa63ffe2b34810959c34c434e7877f1dbe20da8beef/icechunk-2.0.4-cp313-cp313-manylinux_2_28_armv7l.whl", hash = "sha256:07718b38150ef0c35b094658822d0307993d6cdcde5215ce0332e58587fdebac", size = 16653774, upload-time = "2026-04-29T22:02:01.087Z" }, + { url = "https://files.pythonhosted.org/packages/25/28/258b7f468b2418af086ce397c785ee972523a8df1ded7baeb34636068a34/icechunk-2.0.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:887fd0366ed54f281a7791c35b1afc4865264a7aeaba6436f9221744127f19b6", size = 17042662, upload-time = "2026-04-29T22:02:38.091Z" }, + { url = "https://files.pythonhosted.org/packages/46/5d/f2bfd70c279cf3aa601dd483cffebec294c31d3909db2bcefcc560c44094/icechunk-2.0.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:4c9fe9b3494041a3f291d22e3ed7b6157924a0a95476c501f2924297cb070283", size = 16819304, upload-time = "2026-04-29T22:02:47.18Z" }, + { url = "https://files.pythonhosted.org/packages/54/4e/6d97e198e863dd71b0fc82fcb34b3a9eabe46bbac42a9bf5cd0a0f9b11d7/icechunk-2.0.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ea9201f87653950f1a6a5ed9d0b0c9e74d9828f8657822c1e2b2906aadc96614", size = 16901546, upload-time = "2026-04-29T22:02:55.932Z" }, + { url = "https://files.pythonhosted.org/packages/fa/fb/5229deed47429d944b1065ad4fa404778ff07e2f29a078680f515b15bf2a/icechunk-2.0.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:36d25f1d4e753c783913c5b5a0ffeee235e103b6ad73fac0f46aa9ac21a95be2", size = 17582653, upload-time = "2026-04-29T22:03:05.283Z" }, + { url = "https://files.pythonhosted.org/packages/71/a5/339d62b471faeb27172e75063d481bc0af3f0ba9ee7bf35988a8617943e8/icechunk-2.0.4-cp313-cp313-win_amd64.whl", hash = "sha256:42d9561c933c3a7a36650aa382eca04a25709a54cfe9378b7bbaf84af3da8484", size = 15895555, upload-time = "2026-04-29T22:03:15.257Z" }, + { url = "https://files.pythonhosted.org/packages/5d/25/a1f99c187a01de546f55508c804e464a25e1a5c98d3367139a4915e681d4/icechunk-2.0.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:597dbde7a780fb44d66883b5dcae7ba6350a9e4e1f101839f1eafdd52d64d6f6", size = 16802237, upload-time = "2026-04-29T22:02:32.098Z" }, + { url = "https://files.pythonhosted.org/packages/6b/dd/d8ee1be3f3e84c231557549cbfeb9f26eafa4ff0d056dff5182cb7de161f/icechunk-2.0.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5ce7569f2d2a1af322448c74c2dda3277db1a7927ce151323043ea578b4dc61d", size = 15504080, upload-time = "2026-04-29T22:02:23.026Z" }, + { url = "https://files.pythonhosted.org/packages/39/b3/8b66d93447a4515b0d6afce9f0fa600cb527532ca271e7846acbc4aff18e/icechunk-2.0.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e5d3d60038424bc9f04da4abcbb225c2a0741ea28ca2c851b5fdb02e4dcd17d", size = 17189150, upload-time = "2026-04-29T22:02:13.705Z" }, + { url = "https://files.pythonhosted.org/packages/f7/7f/9e6b4b3ad9d5e1cecd62d2453b997c726ed00e853e9d7561c6ebad419ac9/icechunk-2.0.4-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:c6f73aef9e4bc26bd1889823dba8d4ae14488520efdd599dfa92ec46f4bdfe2c", size = 16843481, upload-time = "2026-04-29T22:01:53.031Z" }, + { url = "https://files.pythonhosted.org/packages/92/10/4698e602237be3d66c0a33764e1bf2b911927b5c8f4b5db854d6ec8b79af/icechunk-2.0.4-cp314-cp314-manylinux_2_28_armv7l.whl", hash = "sha256:ba41c0bf14a2b509dff2142d746b0a285f473994b3cb5d36d2d91e148640f7dc", size = 16663026, upload-time = "2026-04-29T22:02:04.114Z" }, + { url = "https://files.pythonhosted.org/packages/a8/e9/e25c0cb72923cb0bbcd589d3bdd91c4584cf20d0c056de78dcb4d481fb97/icechunk-2.0.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5ae8f59807cecd87a11abc06ea6a92724adcbb49ac08f3a4e45b9ca975d6ad6c", size = 17058642, upload-time = "2026-04-29T22:02:41.124Z" }, + { url = "https://files.pythonhosted.org/packages/3f/c6/0452f30e0407b6aea64089c4c4beb5a98b213467887c7acf8872de064ced/icechunk-2.0.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:17b6da024950b20d5126bb903d4717430c0b9164f1c450fd257f7a719744488a", size = 16831074, upload-time = "2026-04-29T22:02:49.928Z" }, + { url = "https://files.pythonhosted.org/packages/90/8e/045b247bf6691b66ee0bca03bba2784a3813fa42c445189f687fada53e28/icechunk-2.0.4-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:5a5cbc9e5ea0d8ba1e42c73f7decd72cec389b436de347d28ce286b7c33174d9", size = 16912935, upload-time = "2026-04-29T22:02:59.129Z" }, + { url = "https://files.pythonhosted.org/packages/bc/47/7190655c250e1e247b7952cc63ac74dfc71321f119c5445971a1c21832f5/icechunk-2.0.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:12c7adfba219d1c9cb24379a875f72f5991bf88b0a8e49bedaf5c5cb821288cd", size = 17595683, upload-time = "2026-04-29T22:03:08.255Z" }, + { url = "https://files.pythonhosted.org/packages/a9/2b/0dd74453d0c4f5b68a66682051f37e7d88908a27e85c843eba41402262ca/icechunk-2.0.4-cp314-cp314-win_amd64.whl", hash = "sha256:339d8f8584940497c7b69b17613620b3aeb64b394a85b6ff69192709e1eaddeb", size = 15907915, upload-time = "2026-04-29T22:03:18.286Z" }, +] + [[package]] name = "identify" version = "2.6.13" @@ -1976,6 +2098,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2d/00/d90b10b962b4277f5e64a78b6609968859ff86889f5b898c1a778c06ec00/lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c", size = 111036, upload-time = "2024-08-13T19:48:58.603Z" }, ] +[[package]] +name = "linkify-it-py" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "uc-micro-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/c9/06ea13676ef354f0af6169587ae292d3e2406e212876a413bf9eece4eb23/linkify_it_py-2.1.0.tar.gz", hash = "sha256:43360231720999c10e9328dc3691160e27a718e280673d444c38d7d3aaa3b98b", size = 29158, upload-time = "2026-03-01T07:48:47.683Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/de/88b3be5c31b22333b3ca2f6ff1de4e863d8fe45aaea7485f591970ec1d3e/linkify_it_py-2.1.0-py3-none-any.whl", hash = "sha256:0d252c1594ecba2ecedc444053db5d3a9b7ec1b0dd929c8f1d74dce89f86c05e", size = 19878, upload-time = "2026-03-01T07:48:46.098Z" }, +] + [[package]] name = "locket" version = "1.0.0" @@ -2578,6 +2712,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "narwhals" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e9/f3/257adc69a71011b4c8cda321b00f02c5bf1980ae38ffd05a58d9632d4de8/narwhals-2.20.0.tar.gz", hash = "sha256:c10994975fa7dc5a68c2cffcddbd5908fc8ebb2d463c5bab085309c0ee1f551e", size = 627848, upload-time = "2026-04-20T12:11:45.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/69/f24d3d1c38ad69e256138b4ec2452a8c7cf66be49dc214771ae99dd4f0a0/narwhals-2.20.0-py3-none-any.whl", hash = "sha256:16e750ea5507d4ba6e8d03455b5f93a535e0405976561baea235bca5dc9f475d", size = 449373, upload-time = "2026-04-20T12:11:43.596Z" }, +] + [[package]] name = "natsort" version = "8.4.0" @@ -2924,6 +3067,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/af/4fbc8cab944db5d21b7e2a5b8e9211a03a79852b1157e2c102fcc61ac440/pandocfilters-1.5.1-py2.py3-none-any.whl", hash = "sha256:93be382804a9cdb0a7267585f157e5d1731bbe5545a85b268d6f5fe6232de2bc", size = 8663, upload-time = "2024-01-18T20:08:11.28Z" }, ] +[[package]] +name = "panel" +version = "1.8.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "bleach" }, + { name = "bokeh" }, + { name = "linkify-it-py" }, + { name = "markdown" }, + { name = "markdown-it-py" }, + { name = "mdit-py-plugins" }, + { name = "narwhals" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "param" }, + { name = "pyviz-comms" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/c6/9e0de5cab6eac1416a76bf94e0fcec229423526eeb25558e44ba7bdc506c/panel-1.8.10.tar.gz", hash = "sha256:762881b06efb99f3d201749a3634c6b5f18f46cfca520970c2c1bb31bed42c90", size = 32186528, upload-time = "2026-03-16T12:59:00.448Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/85/3290cc84bb35503293ea23d2a0b39a78cf02c560ae1455502b042975c951/panel-1.8.10-py3-none-any.whl", hash = "sha256:b1de9304e729b87fdeee59a5be5b0d3ce7786dc301626cb728f69fbc4c136693", size = 30270089, upload-time = "2026-03-16T12:58:57.03Z" }, +] + +[[package]] +name = "param" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/64/30f7fa7fd5ba7337ba6e746fbf7a963800229430c663ab82b09821d9a8c3/param-2.3.3.tar.gz", hash = "sha256:43c24345d287d4abda15eac65b5b15d55f178a393421f0bfca02a77dd6c0d1dd", size = 202196, upload-time = "2026-03-31T16:09:47.982Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/2b/4bc14b04ae260e68c4db7d1e5d2fdda214fb4381236fc8e2d720ed14e2d9/param-2.3.3-py3-none-any.whl", hash = "sha256:ae25afbc372c1a5e4ee72935cc76ff77b1db1c4eb092bc46715c906ef3cc2e69", size = 139996, upload-time = "2026-03-31T16:09:45.845Z" }, +] + [[package]] name = "parso" version = "0.8.5" @@ -3566,6 +3743,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, ] +[[package]] +name = "pyviz-comms" +version = "3.0.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "param" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/ee/2b5367b911bab506662abffe6f342101a9b3edacee91ff9afe62db5fe9a7/pyviz_comms-3.0.6.tar.gz", hash = "sha256:73d66b620390d97959b2c4d8a2c0778d41fe20581be4717f01e46b8fae8c5695", size = 197772, upload-time = "2025-06-20T16:50:30.97Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/5a/f8c0868199bbb231a02616286ce8a4ccb85f5387b9215510297dcfedd214/pyviz_comms-3.0.6-py3-none-any.whl", hash = "sha256:4eba6238cd4a7f4add2d11879ce55411785b7d38a7c5dba42c7a0826ca53e6c2", size = 84275, upload-time = "2025-06-20T16:50:28.826Z" }, +] + [[package]] name = "pywin32" version = "311" @@ -4211,6 +4400,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, ] +[[package]] +name = "uc-micro-py" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/78/67/9a363818028526e2d4579334460df777115bdec1bb77c08f9db88f6389f2/uc_micro_py-2.0.0.tar.gz", hash = "sha256:c53691e495c8db60e16ffc4861a35469b0ba0821fe409a8a7a0a71864d33a811", size = 6611, upload-time = "2026-03-01T06:31:27.526Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/73/d21edf5b204d1467e06500080a50f79d49ef2b997c79123a536d4a17d97c/uc_micro_py-2.0.0-py3-none-any.whl", hash = "sha256:3603a3859af53e5a39bc7677713c78ea6589ff188d70f4fee165db88e22b242c", size = 6383, upload-time = "2026-03-01T06:31:26.257Z" }, +] + [[package]] name = "ujson" version = "5.12.0" @@ -4515,6 +4713,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8d/f0/73c24457c941b8b08f7d090853e40f4b2cdde88b5da721f3f28e98df77c9/xarray-2025.9.0-py3-none-any.whl", hash = "sha256:79f0e25fb39571f612526ee998ee5404d8725a1db3951aabffdb287388885df0", size = 1349595, upload-time = "2025-09-04T04:20:24.36Z" }, ] +[[package]] +name = "xyzservices" +version = "2026.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/08/3cb9f67a8d48021aca2a02292cc26eecd71d949ae70ad66420a8730cc302/xyzservices-2026.3.0.tar.gz", hash = "sha256:d226866a5d8e9fef337034d8da37a8298f0a1d9d1489b4018e69579eb321fea4", size = 1135736, upload-time = "2026-03-30T14:42:25.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a9/d23012099dc88ec69a29c6407b41d89681cb674c2043cd5b467c7e299c08/xyzservices-2026.3.0-py3-none-any.whl", hash = "sha256:503183d4b322bfebc3c50cdd21192aa3e81e36c5efbf9133d54ae82143e0576b", size = 94101, upload-time = "2026-03-30T14:42:24.608Z" }, +] + [[package]] name = "yarl" version = "1.20.1" From cba74a3de3f35559641f91cfb1aa7074f97cccc4 Mon Sep 17 00:00:00 2001 From: Luis Lopez Date: Tue, 5 May 2026 12:50:42 -0500 Subject: [PATCH 02/24] Add virtual API docs page, link virtual_data.ipynb tutorial in nav --- docs/.nav.yml | 3 + docs/api/virtual/virtual.md | 16 + docs/tutorials/virtual_data.ipynb | 3732 +++++++++++++++++++++++++++++ mkdocs.yml | 1 + 4 files changed, 3752 insertions(+) create mode 100644 docs/api/virtual/virtual.md create mode 100644 docs/tutorials/virtual_data.ipynb diff --git a/docs/.nav.yml b/docs/.nav.yml index 9af3b61ce..511bef2bc 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -22,6 +22,8 @@ nav: - "Store": "api/store/store.md" - Auth: - "Auth": "api/auth/auth.md" + - Virtual: + - "Virtual": "api/virtual/virtual.md" - User guide: - "user/index.md" @@ -48,6 +50,7 @@ nav: # - "Reproducing NASA sea level rise infographic": "tutorials/SSL.ipynb" TODO: make this faster - "Accessing and visualizing EMIT data with a few lines of code": "user/tutorials/emit-earthaccess.ipynb" - "Cloud optimized access to TEMPO air quality data with earthaccess and virtualizarr": "user/tutorials/virtual_dataset_tutorial_with_TEMPO_Level3.ipynb" + - "Virtual datasets with earthaccess": "tutorials/virtual_data.ipynb" - Reference: - Glossary: - "NASA Glossary": "user/reference/glossary/nasa-glossary.md" diff --git a/docs/api/virtual/virtual.md b/docs/api/virtual/virtual.md new file mode 100644 index 000000000..e6781e2d5 --- /dev/null +++ b/docs/api/virtual/virtual.md @@ -0,0 +1,16 @@ +# Documentation for `earthaccess.virtual` + +::: earthaccess.virtual + options: + inherited_members: true + show_root_heading: true + show_source: false + +--- + +## Internal helpers + +::: earthaccess.virtual._parser.homogenize_dataset_codec_level + options: + show_root_heading: true + show_source: false diff --git a/docs/tutorials/virtual_data.ipynb b/docs/tutorials/virtual_data.ipynb new file mode 100644 index 000000000..0d2ae5eb7 --- /dev/null +++ b/docs/tutorials/virtual_data.ipynb @@ -0,0 +1,3732 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "63bfc86a-20e8-4b7e-ace1-892dff3f6b68", + "metadata": {}, + "source": [ + "# Virtual Datasets with `earthaccess`\n", + "\n", + "earthaccess can help build and use virtual data stores for NASA archives by providing a unified gateway to search and stream cloud-resident granules without needing to manage complex authentication or local storage. \n", + "\n", + "Virtual datasets themselves function as a metadata-only representation of this distributed data, acting as a lightweight \"map\" that makes thousands of separate files appear as a single, contiguous array. Instead of moving or duplicating massive physical files, a virtual dataset stores the specific byte-range coordinates of where data chunks live. This allows us to perform high-performance, cloud-native analysis—like slicing through time or space—without the heavy overhead of traditional file-opening processes or the cost of data conversion.\n", + "\n", + "### Unified virtual access API\n", + "\n", + "`earthaccess` offers a unified virtual data API. On one hand, `virtualize(granules)` allows us to generate virtual data cubes from CMR granule searches. These cubes can be saved as [kerchunk](https://fsspec.github.io/kerchunk/) or [icechunk](https://icechunk.io/en/latest/understanding/concepts/) references that later can be opened with the new `open_virtual()` method. \n", + "\n", + "#### Key Workflow Components:\n", + "\n", + "* **`virtualize(granules)`**: Automates the metadata extraction (using dmrpp or native parsers) to create a VirtualiZarr object directly from your search results.\n", + " * Persistence: You can save these references to avoid re-parsing thousands of files in future sessions.\n", + "* **`open_virtual(path)`**: Instantly restores the session, providing an Xarray-compatible object that behaves like a local Zarr store but reads directly from NASA's S3 buckets or HTTP distributed files.\n", + "\n", + "| Format | Best For | Key Characteristic |\n", + "| :--- | :--- | :--- |\n", + "| **Kerchunk** | Static Archives | A stable, json or parquet map of existing archival files. |\n", + "| **Icechunk** | Dynamic Workflows | Transactional storage for Zarr that supports versioning and snapshots. |\n", + "\n", + "\n", + "## Opening virtual datasets with `earthaccess`" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "3e3d9c3e-890a-4be9-a107-f1e5784d0e31", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "(function(root) {\n", + " function now() {\n", + " return new Date();\n", + " }\n", + "\n", + " const force = true;\n", + " const version = '3.9.0'.replace('rc', '-rc.').replace('.dev', '-dev.');\n", + " const reloading = false;\n", + " const Bokeh = root.Bokeh;\n", + " const BK_RE = /^https:\\/\\/cdn\\.bokeh\\.org\\/bokeh\\/(release|dev)\\/bokeh-/;\n", + " const PN_RE = /^https:\\/\\/cdn\\.holoviz\\.org\\/panel\\/[^/]+\\/dist\\/panel/i;\n", + "\n", + " // Set a timeout for this load but only if we are not already initializing\n", + " if (typeof (root._bokeh_timeout) === \"undefined\" || (force || !root._bokeh_is_initializing)) {\n", + " root._bokeh_timeout = Date.now() + 5000;\n", + " root._bokeh_failed_load = false;\n", + " }\n", + "\n", + " function run_callbacks() {\n", + " try {\n", + " root._bokeh_onload_callbacks.forEach(function(callback) {\n", + " if (callback != null)\n", + " callback();\n", + " });\n", + " } finally {\n", + " delete root._bokeh_onload_callbacks;\n", + " }\n", + " console.debug(\"Bokeh: all callbacks have finished\");\n", + " }\n", + "\n", + " function load_libs(css_urls, js_urls, js_modules, js_exports, Bokeh, callback) {\n", + " if (css_urls == null) css_urls = [];\n", + " if (js_urls == null) js_urls = [];\n", + " if (js_modules == null) js_modules = [];\n", + " if (js_exports == null) js_exports = {};\n", + "\n", + " root._bokeh_onload_callbacks.push(callback);\n", + "\n", + " if (root._bokeh_is_loading > 0) {\n", + " // Don't load bokeh if it is still initializing\n", + " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", + " return null;\n", + " } else if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n", + " // There is nothing to load\n", + " run_callbacks();\n", + " return null;\n", + " }\n", + "\n", + " function on_load() {\n", + " root._bokeh_is_loading--;\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", + " run_callbacks()\n", + " }\n", + " }\n", + " window._bokeh_on_load = on_load\n", + "\n", + " function on_error(e) {\n", + " const src_el = e.srcElement\n", + " console.error(\"failed to load \" + (src_el.href || src_el.src));\n", + " }\n", + "\n", + " const skip = [];\n", + " if (window.requirejs) {\n", + " window.requirejs.config({'packages': {}, 'paths': {}, 'shim': {}});\n", + " root._bokeh_is_loading = css_urls.length + 0;\n", + " } else {\n", + " root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n", + " }\n", + "\n", + " const existing_stylesheets = []\n", + " const links = document.getElementsByTagName('link')\n", + " for (let i = 0; i < links.length; i++) {\n", + " const link = links[i]\n", + " if (link.href != null) {\n", + " existing_stylesheets.push(link.href)\n", + " }\n", + " }\n", + " for (let i = 0; i < css_urls.length; i++) {\n", + " const url = css_urls[i];\n", + " const escaped = encodeURI(url)\n", + " if (existing_stylesheets.indexOf(escaped) !== -1) {\n", + " on_load()\n", + " continue;\n", + " }\n", + " const element = document.createElement(\"link\");\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.rel = \"stylesheet\";\n", + " element.type = \"text/css\";\n", + " element.href = url;\n", + " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", + " document.body.appendChild(element);\n", + " } var existing_scripts = []\n", + " const scripts = document.getElementsByTagName('script')\n", + " for (let i = 0; i < scripts.length; i++) {\n", + " var script = scripts[i]\n", + " if (script.src != null) {\n", + " existing_scripts.push(script.src)\n", + " }\n", + " }\n", + " for (let i = 0; i < js_urls.length; i++) {\n", + " const url = js_urls[i];\n", + " const escaped = encodeURI(url)\n", + " const shouldSkip = skip.includes(escaped) || existing_scripts.includes(escaped)\n", + " const isBokehOrPanel = BK_RE.test(escaped) || PN_RE.test(escaped)\n", + " const missingOrBroken = Bokeh == null || Bokeh.Panel == null || (Bokeh.version != version && !Bokeh.versions?.has(version)) || Bokeh.versions?.get(version)?.Panel == null;\n", + " if (shouldSkip && !(isBokehOrPanel && missingOrBroken)) {\n", + " if (!window.requirejs) {\n", + " on_load();\n", + " }\n", + " continue;\n", + " }\n", + " const element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.src = url;\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " for (let i = 0; i < js_modules.length; i++) {\n", + " const url = js_modules[i];\n", + " const escaped = encodeURI(url)\n", + " if (skip.indexOf(escaped) !== -1 || existing_scripts.indexOf(escaped) !== -1) {\n", + " if (!window.requirejs) {\n", + " on_load();\n", + " }\n", + " continue;\n", + " }\n", + " var element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.src = url;\n", + " element.type = \"module\";\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " for (const name in js_exports) {\n", + " const url = js_exports[name];\n", + " const escaped = encodeURI(url)\n", + " if (skip.indexOf(escaped) >= 0 || root[name] != null) {\n", + " if (!window.requirejs) {\n", + " on_load();\n", + " }\n", + " continue;\n", + " }\n", + " var element = document.createElement('script');\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.type = \"module\";\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " element.textContent = `\n", + " import ${name} from \"${url}\"\n", + " window.${name} = ${name}\n", + " window._bokeh_on_load()\n", + " `\n", + " document.head.appendChild(element);\n", + " }\n", + " if (!js_urls.length && !js_modules.length) {\n", + " on_load()\n", + " }\n", + " };\n", + "\n", + " function inject_raw_css(css) {\n", + " const element = document.createElement(\"style\");\n", + " element.appendChild(document.createTextNode(css));\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " const js_urls = [\"https://cdn.holoviz.org/panel/1.8.10/dist/bundled/reactiveesm/es-module-shims@^1.10.0/dist/es-module-shims.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.9.0.min.js\", \"https://cdn.holoviz.org/panel/1.8.10/dist/panel.min.js\"];\n", + " const js_modules = [];\n", + " const js_exports = {};\n", + " const css_urls = [];\n", + " const inline_js = [ function(Bokeh) {\n", + " Bokeh.set_log_level(\"info\");\n", + " },\n", + "function(Bokeh) {} // ensure no trailing comma for IE\n", + " ];\n", + "\n", + " function run_inline_js() {\n", + " if ((root.Bokeh !== undefined) || (force === true)) {\n", + " for (let i = 0; i < inline_js.length; i++) {\n", + " try {\n", + " inline_js[i].call(root, root.Bokeh);\n", + " } catch(e) {\n", + " if (!reloading) {\n", + " throw e;\n", + " }\n", + " }\n", + " }\n", + " } else if (Date.now() < root._bokeh_timeout) {\n", + " setTimeout(run_inline_js, 100);\n", + " } else if (!root._bokeh_failed_load) {\n", + " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", + " root._bokeh_failed_load = true;\n", + " }\n", + " root._bokeh_is_initializing = false;\n", + " }\n", + "\n", + " function load_or_wait() {\n", + " // Implement a backoff loop that tries to ensure we do not load multiple\n", + " // versions of Bokeh and its dependencies at the same time.\n", + " // In recent versions we use the root._bokeh_is_initializing flag\n", + " // to determine whether there is an ongoing attempt to initialize\n", + " // bokeh, however for backward compatibility we also try to ensure\n", + " // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n", + " // before older versions are fully initialized.\n", + " if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n", + " // If the timeout and bokeh was not successfully loaded we reset\n", + " // everything and try loading again\n", + " root._bokeh_timeout = Date.now() + 5000;\n", + " root._bokeh_is_initializing = false;\n", + " root._bokeh_onload_callbacks = undefined;\n", + " root._bokeh_is_loading = 0;\n", + " console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n", + " load_or_wait();\n", + " } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n", + " setTimeout(load_or_wait, 100);\n", + " } else {\n", + " root._bokeh_is_initializing = true;\n", + " root._bokeh_onload_callbacks = [];\n", + " const bokeh_loaded = Bokeh != null && ((Bokeh.version === version && Bokeh.Panel) || (Bokeh.versions?.has(version) && Bokeh.versions.get(version)?.Panel));\n", + " if (!reloading && !bokeh_loaded) {\n", + " if (root.Bokeh) {\n", + " root.Bokeh = undefined;\n", + " }\n", + " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", + " }\n", + " load_libs(css_urls, js_urls, js_modules, js_exports, Bokeh, function() {\n", + " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", + " run_inline_js();\n", + " if (Bokeh != undefined && !reloading) {\n", + " const NewBokeh = root.Bokeh;\n", + " if (Bokeh.versions === undefined) {\n", + " Bokeh.versions = new Map();\n", + " }\n", + " if (NewBokeh.version !== Bokeh.version) {\n", + " Bokeh[NewBokeh.version] = NewBokeh;\n", + " Bokeh.versions.set(NewBokeh.version, NewBokeh);\n", + " }\n", + " root.Bokeh = Bokeh;\n", + " }\n", + " });\n", + " }\n", + " }\n", + " // Give older versions of the autoload script a head-start to ensure\n", + " // they initialize before we start loading newer version.\n", + " setTimeout(load_or_wait, 100)\n", + "}(window));" + ], + "application/vnd.holoviews_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n const version = '3.9.0'.replace('rc', '-rc.').replace('.dev', '-dev.');\n const reloading = false;\n const Bokeh = root.Bokeh;\n const BK_RE = /^https:\\/\\/cdn\\.bokeh\\.org\\/bokeh\\/(release|dev)\\/bokeh-/;\n const PN_RE = /^https:\\/\\/cdn\\.holoviz\\.org\\/panel\\/[^/]+\\/dist\\/panel/i;\n\n // Set a timeout for this load but only if we are not already initializing\n if (typeof (root._bokeh_timeout) === \"undefined\" || (force || !root._bokeh_is_initializing)) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks;\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, js_modules, js_exports, Bokeh, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n if (js_modules == null) js_modules = [];\n if (js_exports == null) js_exports = {};\n\n root._bokeh_onload_callbacks.push(callback);\n\n if (root._bokeh_is_loading > 0) {\n // Don't load bokeh if it is still initializing\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n } else if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n // There is nothing to load\n run_callbacks();\n return null;\n }\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n window._bokeh_on_load = on_load\n\n function on_error(e) {\n const src_el = e.srcElement\n console.error(\"failed to load \" + (src_el.href || src_el.src));\n }\n\n const skip = [];\n if (window.requirejs) {\n window.requirejs.config({'packages': {}, 'paths': {}, 'shim': {}});\n root._bokeh_is_loading = css_urls.length + 0;\n } else {\n root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n }\n\n const existing_stylesheets = []\n const links = document.getElementsByTagName('link')\n for (let i = 0; i < links.length; i++) {\n const link = links[i]\n if (link.href != null) {\n existing_stylesheets.push(link.href)\n }\n }\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const escaped = encodeURI(url)\n if (existing_stylesheets.indexOf(escaped) !== -1) {\n on_load()\n continue;\n }\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n } var existing_scripts = []\n const scripts = document.getElementsByTagName('script')\n for (let i = 0; i < scripts.length; i++) {\n var script = scripts[i]\n if (script.src != null) {\n existing_scripts.push(script.src)\n }\n }\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const escaped = encodeURI(url)\n const shouldSkip = skip.includes(escaped) || existing_scripts.includes(escaped)\n const isBokehOrPanel = BK_RE.test(escaped) || PN_RE.test(escaped)\n const missingOrBroken = Bokeh == null || Bokeh.Panel == null || (Bokeh.version != version && !Bokeh.versions?.has(version)) || Bokeh.versions?.get(version)?.Panel == null;\n if (shouldSkip && !(isBokehOrPanel && missingOrBroken)) {\n if (!window.requirejs) {\n on_load();\n }\n continue;\n }\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (let i = 0; i < js_modules.length; i++) {\n const url = js_modules[i];\n const escaped = encodeURI(url)\n if (skip.indexOf(escaped) !== -1 || existing_scripts.indexOf(escaped) !== -1) {\n if (!window.requirejs) {\n on_load();\n }\n continue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (const name in js_exports) {\n const url = js_exports[name];\n const escaped = encodeURI(url)\n if (skip.indexOf(escaped) >= 0 || root[name] != null) {\n if (!window.requirejs) {\n on_load();\n }\n continue;\n }\n var element = document.createElement('script');\n element.onerror = on_error;\n element.async = false;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n element.textContent = `\n import ${name} from \"${url}\"\n window.${name} = ${name}\n window._bokeh_on_load()\n `\n document.head.appendChild(element);\n }\n if (!js_urls.length && !js_modules.length) {\n on_load()\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.holoviz.org/panel/1.8.10/dist/bundled/reactiveesm/es-module-shims@^1.10.0/dist/es-module-shims.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.9.0.min.js\", \"https://cdn.holoviz.org/panel/1.8.10/dist/panel.min.js\"];\n const js_modules = [];\n const js_exports = {};\n const css_urls = [];\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n ];\n\n function run_inline_js() {\n if ((root.Bokeh !== undefined) || (force === true)) {\n for (let i = 0; i < inline_js.length; i++) {\n try {\n inline_js[i].call(root, root.Bokeh);\n } catch(e) {\n if (!reloading) {\n throw e;\n }\n }\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n }\n root._bokeh_is_initializing = false;\n }\n\n function load_or_wait() {\n // Implement a backoff loop that tries to ensure we do not load multiple\n // versions of Bokeh and its dependencies at the same time.\n // In recent versions we use the root._bokeh_is_initializing flag\n // to determine whether there is an ongoing attempt to initialize\n // bokeh, however for backward compatibility we also try to ensure\n // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n // before older versions are fully initialized.\n if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n // If the timeout and bokeh was not successfully loaded we reset\n // everything and try loading again\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_is_initializing = false;\n root._bokeh_onload_callbacks = undefined;\n root._bokeh_is_loading = 0;\n console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n load_or_wait();\n } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n setTimeout(load_or_wait, 100);\n } else {\n root._bokeh_is_initializing = true;\n root._bokeh_onload_callbacks = [];\n const bokeh_loaded = Bokeh != null && ((Bokeh.version === version && Bokeh.Panel) || (Bokeh.versions?.has(version) && Bokeh.versions.get(version)?.Panel));\n if (!reloading && !bokeh_loaded) {\n if (root.Bokeh) {\n root.Bokeh = undefined;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n }\n load_libs(css_urls, js_urls, js_modules, js_exports, Bokeh, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n if (Bokeh != undefined && !reloading) {\n const NewBokeh = root.Bokeh;\n if (Bokeh.versions === undefined) {\n Bokeh.versions = new Map();\n }\n if (NewBokeh.version !== Bokeh.version) {\n Bokeh[NewBokeh.version] = NewBokeh;\n Bokeh.versions.set(NewBokeh.version, NewBokeh);\n }\n root.Bokeh = Bokeh;\n }\n });\n }\n }\n // Give older versions of the autoload script a head-start to ensure\n // they initialize before we start loading newer version.\n setTimeout(load_or_wait, 100)\n}(window));" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "\n", + "if ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n", + " window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n", + "}\n", + "\n", + "\n", + " function JupyterCommManager() {\n", + " }\n", + "\n", + " JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n", + " if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", + " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", + " comm_manager.register_target(comm_id, function(comm) {\n", + " comm.on_msg(msg_handler);\n", + " });\n", + " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", + " window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n", + " comm.onMsg = msg_handler;\n", + " });\n", + " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", + " google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n", + " var messages = comm.messages[Symbol.asyncIterator]();\n", + " function processIteratorResult(result) {\n", + " var message = result.value;\n", + " var content = {data: message.data, comm_id};\n", + " var buffers = []\n", + " for (var buffer of message.buffers || []) {\n", + " buffers.push(new DataView(buffer))\n", + " }\n", + " var metadata = message.metadata || {};\n", + " var msg = {content, buffers, metadata}\n", + " msg_handler(msg);\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " return messages.next().then(processIteratorResult);\n", + " })\n", + " }\n", + " }\n", + "\n", + " JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n", + " if (comm_id in window.PyViz.comms) {\n", + " return window.PyViz.comms[comm_id];\n", + " } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", + " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", + " var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n", + " if (msg_handler) {\n", + " comm.on_msg(msg_handler);\n", + " }\n", + " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", + " var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n", + " let retries = 0;\n", + " const open = () => {\n", + " if (comm.active) {\n", + " comm.open();\n", + " } else if (retries > 3) {\n", + " console.warn('Comm target never activated')\n", + " } else {\n", + " retries += 1\n", + " setTimeout(open, 500)\n", + " }\n", + " }\n", + " if (comm.active) {\n", + " comm.open();\n", + " } else {\n", + " setTimeout(open, 500)\n", + " }\n", + " if (msg_handler) {\n", + " comm.onMsg = msg_handler;\n", + " }\n", + " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", + " var comm_promise = google.colab.kernel.comms.open(comm_id)\n", + " comm_promise.then((comm) => {\n", + " window.PyViz.comms[comm_id] = comm;\n", + " if (msg_handler) {\n", + " var messages = comm.messages[Symbol.asyncIterator]();\n", + " function processIteratorResult(result) {\n", + " var message = result.value;\n", + " var content = {data: message.data};\n", + " var metadata = message.metadata || {comm_id};\n", + " var msg = {content, metadata}\n", + " msg_handler(msg);\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " })\n", + " var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n", + " return comm_promise.then((comm) => {\n", + " comm.send(data, metadata, buffers, disposeOnDone);\n", + " });\n", + " };\n", + " var comm = {\n", + " send: sendClosure\n", + " };\n", + " }\n", + " window.PyViz.comms[comm_id] = comm;\n", + " return comm;\n", + " }\n", + " window.PyViz.comm_manager = new JupyterCommManager();\n", + " \n", + "\n", + "\n", + "var JS_MIME_TYPE = 'application/javascript';\n", + "var HTML_MIME_TYPE = 'text/html';\n", + "var EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\n", + "var CLASS_NAME = 'output';\n", + "\n", + "/**\n", + " * Render data to the DOM node\n", + " */\n", + "function render(props, node) {\n", + " var div = document.createElement(\"div\");\n", + " var script = document.createElement(\"script\");\n", + " node.appendChild(div);\n", + " node.appendChild(script);\n", + "}\n", + "\n", + "/**\n", + " * Handle when a new output is added\n", + " */\n", + "function handle_add_output(event, handle) {\n", + " var output_area = handle.output_area;\n", + " var output = handle.output;\n", + " if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n", + " return\n", + " }\n", + " var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", + " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", + " if (id !== undefined) {\n", + " var nchildren = toinsert.length;\n", + " var html_node = toinsert[nchildren-1].children[0];\n", + " html_node.innerHTML = output.data[HTML_MIME_TYPE];\n", + " var scripts = [];\n", + " var nodelist = html_node.querySelectorAll(\"script\");\n", + " for (var i in nodelist) {\n", + " if (nodelist.hasOwnProperty(i)) {\n", + " scripts.push(nodelist[i])\n", + " }\n", + " }\n", + "\n", + " scripts.forEach( function (oldScript) {\n", + " var newScript = document.createElement(\"script\");\n", + " var attrs = [];\n", + " var nodemap = oldScript.attributes;\n", + " for (var j in nodemap) {\n", + " if (nodemap.hasOwnProperty(j)) {\n", + " attrs.push(nodemap[j])\n", + " }\n", + " }\n", + " attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n", + " newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n", + " oldScript.parentNode.replaceChild(newScript, oldScript);\n", + " });\n", + " if (JS_MIME_TYPE in output.data) {\n", + " toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n", + " }\n", + " output_area._hv_plot_id = id;\n", + " if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n", + " window.PyViz.plot_index[id] = Bokeh.index[id];\n", + " } else {\n", + " window.PyViz.plot_index[id] = null;\n", + " }\n", + " } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", + " var bk_div = document.createElement(\"div\");\n", + " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", + " var script_attrs = bk_div.children[0].attributes;\n", + " for (var i = 0; i < script_attrs.length; i++) {\n", + " toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n", + " }\n", + " // store reference to server id on output_area\n", + " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", + " }\n", + "}\n", + "\n", + "/**\n", + " * Handle when an output is cleared or removed\n", + " */\n", + "function handle_clear_output(event, handle) {\n", + " var id = handle.cell.output_area._hv_plot_id;\n", + " var server_id = handle.cell.output_area._bokeh_server_id;\n", + " if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n", + " var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n", + " if (server_id !== null) {\n", + " comm.send({event_type: 'server_delete', 'id': server_id});\n", + " return;\n", + " } else if (comm !== null) {\n", + " comm.send({event_type: 'delete', 'id': id});\n", + " }\n", + " delete PyViz.plot_index[id];\n", + " if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n", + " var doc = window.Bokeh.index[id].model.document\n", + " doc.clear();\n", + " const i = window.Bokeh.documents.indexOf(doc);\n", + " if (i > -1) {\n", + " window.Bokeh.documents.splice(i, 1);\n", + " }\n", + " }\n", + "}\n", + "\n", + "/**\n", + " * Handle kernel restart event\n", + " */\n", + "function handle_kernel_cleanup(event, handle) {\n", + " delete PyViz.comms[\"hv-extension-comm\"];\n", + " window.PyViz.plot_index = {}\n", + "}\n", + "\n", + "/**\n", + " * Handle update_display_data messages\n", + " */\n", + "function handle_update_output(event, handle) {\n", + " handle_clear_output(event, {cell: {output_area: handle.output_area}})\n", + " handle_add_output(event, handle)\n", + "}\n", + "\n", + "function register_renderer(events, OutputArea) {\n", + " function append_mime(data, metadata, element) {\n", + " // create a DOM node to render to\n", + " var toinsert = this.create_output_subarea(\n", + " metadata,\n", + " CLASS_NAME,\n", + " EXEC_MIME_TYPE\n", + " );\n", + " this.keyboard_manager.register_events(toinsert);\n", + " // Render to node\n", + " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", + " render(props, toinsert[0]);\n", + " element.append(toinsert);\n", + " return toinsert\n", + " }\n", + "\n", + " events.on('output_added.OutputArea', handle_add_output);\n", + " events.on('output_updated.OutputArea', handle_update_output);\n", + " events.on('clear_output.CodeCell', handle_clear_output);\n", + " events.on('delete.Cell', handle_clear_output);\n", + " events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n", + "\n", + " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", + " safe: true,\n", + " index: 0\n", + " });\n", + "}\n", + "\n", + "if (window.Jupyter !== undefined) {\n", + " try {\n", + " var events = require('base/js/events');\n", + " var OutputArea = require('notebook/js/outputarea').OutputArea;\n", + " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", + " register_renderer(events, OutputArea);\n", + " }\n", + " } catch(err) {\n", + " }\n", + "}\n" + ], + "application/vnd.holoviews_load.v0+json": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n function JupyterCommManager() {\n }\n\n JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n comm_manager.register_target(comm_id, function(comm) {\n comm.on_msg(msg_handler);\n });\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n comm.onMsg = msg_handler;\n });\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data, comm_id};\n var buffers = []\n for (var buffer of message.buffers || []) {\n buffers.push(new DataView(buffer))\n }\n var metadata = message.metadata || {};\n var msg = {content, buffers, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n })\n }\n }\n\n JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n if (comm_id in window.PyViz.comms) {\n return window.PyViz.comms[comm_id];\n } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n if (msg_handler) {\n comm.on_msg(msg_handler);\n }\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n let retries = 0;\n const open = () => {\n if (comm.active) {\n comm.open();\n } else if (retries > 3) {\n console.warn('Comm target never activated')\n } else {\n retries += 1\n setTimeout(open, 500)\n }\n }\n if (comm.active) {\n comm.open();\n } else {\n setTimeout(open, 500)\n }\n if (msg_handler) {\n comm.onMsg = msg_handler;\n }\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n var comm_promise = google.colab.kernel.comms.open(comm_id)\n comm_promise.then((comm) => {\n window.PyViz.comms[comm_id] = comm;\n if (msg_handler) {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data};\n var metadata = message.metadata || {comm_id};\n var msg = {content, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n }\n })\n var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n return comm_promise.then((comm) => {\n comm.send(data, metadata, buffers, disposeOnDone);\n });\n };\n var comm = {\n send: sendClosure\n };\n }\n window.PyViz.comms[comm_id] = comm;\n return comm;\n }\n window.PyViz.comm_manager = new JupyterCommManager();\n \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n var div = document.createElement(\"div\");\n var script = document.createElement(\"script\");\n node.appendChild(div);\n node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n var output_area = handle.output_area;\n var output = handle.output;\n if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n return\n }\n var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n if (id !== undefined) {\n var nchildren = toinsert.length;\n var html_node = toinsert[nchildren-1].children[0];\n html_node.innerHTML = output.data[HTML_MIME_TYPE];\n var scripts = [];\n var nodelist = html_node.querySelectorAll(\"script\");\n for (var i in nodelist) {\n if (nodelist.hasOwnProperty(i)) {\n scripts.push(nodelist[i])\n }\n }\n\n scripts.forEach( function (oldScript) {\n var newScript = document.createElement(\"script\");\n var attrs = [];\n var nodemap = oldScript.attributes;\n for (var j in nodemap) {\n if (nodemap.hasOwnProperty(j)) {\n attrs.push(nodemap[j])\n }\n }\n attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n oldScript.parentNode.replaceChild(newScript, oldScript);\n });\n if (JS_MIME_TYPE in output.data) {\n toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n }\n output_area._hv_plot_id = id;\n if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n window.PyViz.plot_index[id] = Bokeh.index[id];\n } else {\n window.PyViz.plot_index[id] = null;\n }\n } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n var bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n var script_attrs = bk_div.children[0].attributes;\n for (var i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n var id = handle.cell.output_area._hv_plot_id;\n var server_id = handle.cell.output_area._bokeh_server_id;\n if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n if (server_id !== null) {\n comm.send({event_type: 'server_delete', 'id': server_id});\n return;\n } else if (comm !== null) {\n comm.send({event_type: 'delete', 'id': id});\n }\n delete PyViz.plot_index[id];\n if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n var doc = window.Bokeh.index[id].model.document\n doc.clear();\n const i = window.Bokeh.documents.indexOf(doc);\n if (i > -1) {\n window.Bokeh.documents.splice(i, 1);\n }\n }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n delete PyViz.comms[\"hv-extension-comm\"];\n window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n handle_clear_output(event, {cell: {output_area: handle.output_area}})\n handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n var toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[0]);\n element.append(toinsert);\n return toinsert\n }\n\n events.on('output_added.OutputArea', handle_add_output);\n events.on('output_updated.OutputArea', handle_update_output);\n events.on('clear_output.CodeCell', handle_clear_output);\n events.on('delete.Cell', handle_clear_output);\n events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n safe: true,\n index: 0\n });\n}\n\nif (window.Jupyter !== undefined) {\n try {\n var events = require('base/js/events');\n var OutputArea = require('notebook/js/outputarea').OutputArea;\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n } catch(err) {\n }\n}\n" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.holoviews_exec.v0+json": "", + "text/html": [ + "
\n", + "
\n", + "
\n", + "" + ] + }, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "939e9a61-60de-4bc9-beda-22d8e87278ab" + } + }, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 17.4 s, sys: 3.44 s, total: 20.9 s\n", + "Wall time: 24.7 s\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 70TB\n",
+       "Dimensions:           (time: 3868, lat: 17999, lon: 36000)\n",
+       "Coordinates:\n",
+       "  * time              (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n",
+       "  * lat               (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n",
+       "  * lon               (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n",
+       "Data variables:\n",
+       "    analysed_sst      (time, lat, lon) float64 20TB ...\n",
+       "    analysis_error    (time, lat, lon) float64 20TB ...\n",
+       "    mask              (time, lat, lon) float32 10TB ...\n",
+       "    sea_ice_fraction  (time, lat, lon) float64 20TB ...\n",
+       "Attributes: (12/47)\n",
+       "    Conventions:                CF-1.5\n",
+       "    Metadata_Conventions:       Unidata Observation Dataset v1.0\n",
+       "    acknowledgment:             Please acknowledge the use of these data with...\n",
+       "    cdm_data_type:              grid\n",
+       "    comment:                    MUR = "Multi-scale Ultra-high Reolution"\n",
+       "    creator_email:              ghrsst@podaac.jpl.nasa.gov\n",
+       "    ...                         ...\n",
+       "    summary:                    A merged, multi-sensor L4 Foundation SST anal...\n",
+       "    time_coverage_end:          20020601T210000Z\n",
+       "    time_coverage_start:        20020531T210000Z\n",
+       "    title:                      Daily MUR SST, Final product\n",
+       "    uuid:                       27665bc0-d5fc-11e1-9b23-0800200c9a66\n",
+       "    westernmost_longitude:      -180.0
" + ], + "text/plain": [ + " Size: 70TB\n", + "Dimensions: (time: 3868, lat: 17999, lon: 36000)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n", + " * lat (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n", + " * lon (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n", + "Data variables:\n", + " analysed_sst (time, lat, lon) float64 20TB ...\n", + " analysis_error (time, lat, lon) float64 20TB ...\n", + " mask (time, lat, lon) float32 10TB ...\n", + " sea_ice_fraction (time, lat, lon) float64 20TB ...\n", + "Attributes: (12/47)\n", + " Conventions: CF-1.5\n", + " Metadata_Conventions: Unidata Observation Dataset v1.0\n", + " acknowledgment: Please acknowledge the use of these data with...\n", + " cdm_data_type: grid\n", + " comment: MUR = \"Multi-scale Ultra-high Reolution\"\n", + " creator_email: ghrsst@podaac.jpl.nasa.gov\n", + " ... ...\n", + " summary: A merged, multi-sensor L4 Foundation SST anal...\n", + " time_coverage_end: 20020601T210000Z\n", + " time_coverage_start: 20020531T210000Z\n", + " title: Daily MUR SST, Final product\n", + " uuid: 27665bc0-d5fc-11e1-9b23-0800200c9a66\n", + " westernmost_longitude: -180.0" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "\n", + "import warnings\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "import earthaccess as ea\n", + "import hvplot.xarray\n", + "import xarray as xr\n", + "\n", + "auth = ea.login()\n", + "\n", + "collection = ea.search_datasets(concept_id=\"C1996881146-POCLOUD\")[\n", + " 0\n", + "] # <- note that we are using the first item\n", + "vds = ea.open_virtual(collection, load=True, force_external=True)\n", + "vds" + ] + }, + { + "cell_type": "markdown", + "id": "08b7fcea-bd3b-4d4d-bd86-ee2e190c79ba", + "metadata": {}, + "source": [ + "### open_virtual()\n", + "\n", + "`open_virtual()` can open virtual collections listed at the collection level metadata, fow now following the convention adopted by JPL of using the following URL in `RelatedUrls` \n", + "\n", + "```json\n", + "{\n", + " \"Description\": \"Virtual data set reference \",\n", + " \"URLContentType\": \"DistributionURL\",\n", + " \"URL\": \"https://archive.podaac.earthdata.nasa.gov/../MUR-JPL-L4-GLOB-v4.1_combined-ref.json\",\n", + " \"Type\": \"GET DATA\",\n", + " \"Subtype\": \"VIRTUAL COLLECTION\"\n", + "}\n", + "```\n", + "\n", + "> Note: We may have multiple virtual stores for any given collection so we anticipate this convention may change in the near future, for now the assumption is that each collection has at most one virtual representation. \n", + "\n", + "`open_virtual()` has the following signature that we explain in short:\n", + "\n", + "* **uri**: this can be a collection level result from `collection = earthaccess.search_datasets()[0]` a local path, or a publicly accessible reference URL. If we use a collection from earthaccess, we don't need to deal with the cumbersome auth scafolding for the underlaying libraries; in all cases we'll get an xarray dataset instance pointing to our virtual dataset.This can be kerchunk or Icechunk stores \n", + "* **access**: this one informs earthaccess if the virtual store points to S3 buckets or HTTP, without inspecting the references we need to be explicit for now. values are `direct` or `external` (same as `indirect`)\n", + "* **storage_options**: if we need custom authentication for the virtual store this works the same way xarray uses storage options, not needed for NASA data as it's automatic.\n", + "* **force_external**: This is a handy flag to force reference translation, this is if the only available store points to S3 but we are accessing the data from outside AWS `us-west-2` setting this to `True` will translate the URLs on the fly for out of region access.\n", + "* **load**: if we should load the indexes in our dataset, if we do we can address the dataset as any other xarray dataset and perform operations like subsetting and aggregations, if not we get a `virtualizarr` dataset backed by `ManifestArray` instances, this is only useful if we plan to add more virtual references and update the cube.\n", + "\n", + "The method signature is as follows:\n", + "\n", + "```python\n", + "ea.open_virtual(\n", + " uri: 'str | Path | earthaccess.DataCollection',\n", + " *,\n", + " access: 'str' = 'indirect',\n", + " storage_options: 'dict[str, Any] | None' = None,\n", + " force_external: 'bool' = False,\n", + " load: 'bool' = True,\n", + " **kwargs: 'Any',\n", + ") -> 'xr.Dataset'\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "0e2366e8-aab1-49ce-a66a-bfc019df2a3b", + "metadata": {}, + "source": [ + "If we try to open a virtual dataset on a collection that has no virtual distribution will result in an exception." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "8ab91cc0-b12b-410c-88af-37ce03153c34", + "metadata": {}, + "outputs": [ + { + "ename": "ValueError", + "evalue": "Collection C3564876127-NSIDC_CPRD does not have a virtual store (no VIRTUAL COLLECTION URL found in its RelatedUrls).", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mValueError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 5\u001b[39m\n\u001b[32m 1\u001b[39m collection = ea.search_datasets(\n\u001b[32m 2\u001b[39m short_name=\u001b[33m\"\u001b[39m\u001b[33mATL06\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 3\u001b[39m )[\u001b[32m0\u001b[39m]\n\u001b[32m----> \u001b[39m\u001b[32m5\u001b[39m vds = \u001b[43mea\u001b[49m\u001b[43m.\u001b[49m\u001b[43mopen_virtual\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcollection\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 6\u001b[39m vds\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/github/betolink/earthaccess/earthaccess/virtual/core.py:700\u001b[39m, in \u001b[36mopen_virtual\u001b[39m\u001b[34m(uri, access, storage_options, force_external, load, **kwargs)\u001b[39m\n\u001b[32m 698\u001b[39m url = uri.virtual_collection_url()\n\u001b[32m 699\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m url \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m700\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m 701\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mCollection \u001b[39m\u001b[38;5;132;01m{\u001b[39;00muri.get(\u001b[33m'\u001b[39m\u001b[33mmeta\u001b[39m\u001b[33m'\u001b[39m,\u001b[38;5;250m \u001b[39m{}).get(\u001b[33m'\u001b[39m\u001b[33mconcept-id\u001b[39m\u001b[33m'\u001b[39m,\u001b[38;5;250m \u001b[39m\u001b[33m'\u001b[39m\u001b[33m'\u001b[39m)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 702\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mdoes not have a virtual store (no VIRTUAL COLLECTION \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 703\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mURL found in its RelatedUrls).\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 704\u001b[39m )\n\u001b[32m 705\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m _is_icechunk_uri(url) \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m _is_kerchunk_uri(url):\n\u001b[32m 706\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m 707\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mUnrecognised virtual store URL in collection: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00murl\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m. \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 708\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mExpected a .icechunk, .parquet, or .json file.\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 709\u001b[39m )\n", + "\u001b[31mValueError\u001b[39m: Collection C3564876127-NSIDC_CPRD does not have a virtual store (no VIRTUAL COLLECTION URL found in its RelatedUrls)." + ] + } + ], + "source": [ + "collection = ea.search_datasets(short_name=\"ATL06\")[0]\n", + "\n", + "vds = ea.open_virtual(collection)\n", + "vds" + ] + }, + { + "cell_type": "markdown", + "id": "54bf4cd2-3650-49f7-bdd8-24ba11da80f7", + "metadata": {}, + "source": [ + "#### `load = False`\n", + "\n", + "When we do not load indexes into our VDS variables, we cannot access byte ranges using slicing, this prevents us from normal xarray operations but it also allows us to treat the dataset as appendable, we can search for new granules, virtualize them with `earthaccess.virtualize()` and concatenate both virtual datasets to persist the updated cube! \n", + "\n", + "Something not intuitive is that because of embeding values in virtual arrays, is better to reconstruct the coordinates than interpreting native byte types and this roundtrip takes more time than just loading the indexes themselves. Thus, not loading takes longer specially if the original virtual store uses JSON serialization. `earthaccess` will cache it but a reference file that's 100+ MB still needs to be fetched and parsed. " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "3b9e877f-f910-4c0d-901d-6a93f6be5237", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 53.4 s, sys: 5.46 s, total: 58.9 s\n", + "Wall time: 1min 2s\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 15TB\n",
+       "Dimensions:           (time: 3868, lat: 17999, lon: 36000)\n",
+       "Coordinates:\n",
+       "  * time              (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n",
+       "  * lat               (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n",
+       "  * lon               (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n",
+       "Data variables:\n",
+       "    analysed_sst      (time, lat, lon) int16 5TB ManifestArray<shape=(3868, 1...\n",
+       "    analysis_error    (time, lat, lon) int16 5TB ManifestArray<shape=(3868, 1...\n",
+       "    mask              (time, lat, lon) int8 3TB ManifestArray<shape=(3868, 17...\n",
+       "    sea_ice_fraction  (time, lat, lon) int8 3TB ManifestArray<shape=(3868, 17...\n",
+       "Attributes: (12/47)\n",
+       "    Conventions:                CF-1.5\n",
+       "    Metadata_Conventions:       Unidata Observation Dataset v1.0\n",
+       "    acknowledgment:             Please acknowledge the use of these data with...\n",
+       "    cdm_data_type:              grid\n",
+       "    comment:                    MUR = "Multi-scale Ultra-high Reolution"\n",
+       "    creator_email:              ghrsst@podaac.jpl.nasa.gov\n",
+       "    ...                         ...\n",
+       "    summary:                    A merged, multi-sensor L4 Foundation SST anal...\n",
+       "    time_coverage_end:          20020601T210000Z\n",
+       "    time_coverage_start:        20020531T210000Z\n",
+       "    title:                      Daily MUR SST, Final product\n",
+       "    uuid:                       27665bc0-d5fc-11e1-9b23-0800200c9a66\n",
+       "    westernmost_longitude:      -180.0
" + ], + "text/plain": [ + " Size: 15TB\n", + "Dimensions: (time: 3868, lat: 17999, lon: 36000)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n", + " * lat (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n", + " * lon (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n", + "Data variables:\n", + " analysed_sst (time, lat, lon) int16 5TB ManifestArray Note how the variables when we use `load=False` are backed by `ManifestArray` instances, they do not load indexes.\n", + "\n", + "Now we can search for new granules with `search_data()` using the last date from our virtual cube so we get granules in the following time steps. " + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "e263f631-358d-4131-b147-f1ab3ae6987e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "31" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sst_granules = ea.search_data(\n", + " concept_id=\"C1996881146-POCLOUD\", temporal=(\"2013-01-02\", \"2013-01-31\")\n", + ")\n", + "len(sst_granules)" + ] + }, + { + "cell_type": "markdown", + "id": "e3a8acf1-7b63-46e5-941e-f2a2b638763d", + "metadata": {}, + "source": [ + "### Virtualizing with `virtualize()`\n", + "\n", + "`earthaccess` can virtualize data cubes from granules, this is meant for data distributors (NASA) or power users that want to try this beta feature for faster data access. Is very important to note that virtualize won't work for most data as they need to be:\n", + "\n", + "* *Homogeneous*: The underlying files must share the exact same variables, data types, and internal schema across all granules. If one file is missing a variable, or if data types differ (e.g., float32 in one file and float64 in another), the virtual concatenation will fail.\n", + "* *CF compliant*: The files' metadata must strictly follow Climate and Forecast (CF) conventions. This is required so the analysis engine (like Xarray) can automatically decode time arrays, correctly identify coordinate variables, and handle scale factors or fill values without manual intervention.\n", + "* *Unified grid*: The granules must share a consistent spatial resolution and coordinate reference system. They need to stitch together seamlessly (usually along a time dimension) without any spatial overlapping, gaps, or need for regridding and interpolation.\n", + "\n", + "For NASA Earth datasets this falls into the level 4 processing datasets, they have global coverage and are distributed in NetCDF or HDF5 format with these characteristics. This is not a requirement but many metadata pitfalls will be encountered if we want to virtualize data from lower processing levels." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f78c350c-e10f-435f-8675-3674b631f150", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 121GB\n",
+       "Dimensions:           (time: 31, lat: 17999, lon: 36000)\n",
+       "Coordinates:\n",
+       "  * time              (time) datetime64[ns] 248B 2013-01-02T09:00:00 ... 2013...\n",
+       "  * lat               (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n",
+       "  * lon               (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n",
+       "Data variables:\n",
+       "    mask              (time, lat, lon) int8 20GB ManifestArray<shape=(31, 179...\n",
+       "    sea_ice_fraction  (time, lat, lon) int8 20GB ManifestArray<shape=(31, 179...\n",
+       "    analysed_sst      (time, lat, lon) int16 40GB ManifestArray<shape=(31, 17...\n",
+       "    analysis_error    (time, lat, lon) int16 40GB ManifestArray<shape=(31, 17...\n",
+       "Attributes: (12/42)\n",
+       "    Conventions:                CF-1.5\n",
+       "    title:                      Daily MUR SST, Final product\n",
+       "    summary:                    A merged, multi-sensor L4 Foundation SST anal...\n",
+       "    references:                 http://podaac.jpl.nasa.gov/Multi-scale_Ultra-...\n",
+       "    institution:                Jet Propulsion Laboratory\n",
+       "    history:                    created at nominal 4-day latency; replaced nr...\n",
+       "    ...                         ...\n",
+       "    project:                    NASA Making Earth Science Data Records for Us...\n",
+       "    publisher_name:             GHRSST Project Office\n",
+       "    publisher_url:              http://www.ghrsst.org\n",
+       "    publisher_email:            ghrsst-po@nceo.ac.uk\n",
+       "    processing_level:           L4\n",
+       "    cdm_data_type:              grid
" + ], + "text/plain": [ + " Size: 121GB\n", + "Dimensions: (time: 31, lat: 17999, lon: 36000)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 248B 2013-01-02T09:00:00 ... 2013...\n", + " * lat (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n", + " * lon (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n", + "Data variables:\n", + " mask (time, lat, lon) int8 20GB ManifestArray NOTE: Do not use this in production, this is to show the workflow of what it would be to concatenate homogeneous datasets" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "161ea6a2-7b26-42bf-a695-3a45b8ae7053", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 121GB\n",
+       "Dimensions:           (time: 31, lat: 17999, lon: 36000)\n",
+       "Coordinates:\n",
+       "  * time              (time) datetime64[ns] 248B 2013-01-02T09:00:00 ... 2013...\n",
+       "  * lat               (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n",
+       "  * lon               (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n",
+       "Data variables:\n",
+       "    mask              (time, lat, lon) int8 20GB ManifestArray<shape=(31, 179...\n",
+       "    sea_ice_fraction  (time, lat, lon) int8 20GB ManifestArray<shape=(31, 179...\n",
+       "    analysed_sst      (time, lat, lon) int16 40GB ManifestArray<shape=(31, 17...\n",
+       "    analysis_error    (time, lat, lon) int16 40GB ManifestArray<shape=(31, 17...\n",
+       "Attributes: (12/42)\n",
+       "    Conventions:                CF-1.5\n",
+       "    title:                      Daily MUR SST, Final product\n",
+       "    summary:                    A merged, multi-sensor L4 Foundation SST anal...\n",
+       "    references:                 http://podaac.jpl.nasa.gov/Multi-scale_Ultra-...\n",
+       "    institution:                Jet Propulsion Laboratory\n",
+       "    history:                    created at nominal 4-day latency; replaced nr...\n",
+       "    ...                         ...\n",
+       "    project:                    NASA Making Earth Science Data Records for Us...\n",
+       "    publisher_name:             GHRSST Project Office\n",
+       "    publisher_url:              http://www.ghrsst.org\n",
+       "    publisher_email:            ghrsst-po@nceo.ac.uk\n",
+       "    processing_level:           L4\n",
+       "    cdm_data_type:              grid
" + ], + "text/plain": [ + " Size: 121GB\n", + "Dimensions: (time: 31, lat: 17999, lon: 36000)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 248B 2013-01-02T09:00:00 ... 2013...\n", + " * lat (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n", + " * lon (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n", + "Data variables:\n", + " mask (time, lat, lon) int8 20GB ManifestArray\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 15TB\n",
+       "Dimensions:           (time: 3899, lat: 17999, lon: 36000)\n",
+       "Coordinates:\n",
+       "  * time              (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n",
+       "  * lat               (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n",
+       "  * lon               (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n",
+       "Data variables:\n",
+       "    analysed_sst      (time, lat, lon) int16 5TB ManifestArray<shape=(3899, 1...\n",
+       "    analysis_error    (time, lat, lon) int16 5TB ManifestArray<shape=(3899, 1...\n",
+       "    mask              (time, lat, lon) int8 3TB ManifestArray<shape=(3899, 17...\n",
+       "    sea_ice_fraction  (time, lat, lon) int8 3TB ManifestArray<shape=(3899, 17...\n",
+       "Attributes: (12/47)\n",
+       "    Conventions:                CF-1.5\n",
+       "    Metadata_Conventions:       Unidata Observation Dataset v1.0\n",
+       "    acknowledgment:             Please acknowledge the use of these data with...\n",
+       "    cdm_data_type:              grid\n",
+       "    comment:                    MUR = "Multi-scale Ultra-high Reolution"\n",
+       "    creator_email:              ghrsst@podaac.jpl.nasa.gov\n",
+       "    ...                         ...\n",
+       "    summary:                    A merged, multi-sensor L4 Foundation SST anal...\n",
+       "    time_coverage_end:          20020601T210000Z\n",
+       "    time_coverage_start:        20020531T210000Z\n",
+       "    title:                      Daily MUR SST, Final product\n",
+       "    uuid:                       27665bc0-d5fc-11e1-9b23-0800200c9a66\n",
+       "    westernmost_longitude:      -180.0
" + ], + "text/plain": [ + " Size: 15TB\n", + "Dimensions: (time: 3899, lat: 17999, lon: 36000)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n", + " * lat (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n", + " * lon (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n", + "Data variables:\n", + " analysed_sst (time, lat, lon) int16 5TB ManifestArray Date: Tue, 5 May 2026 17:00:16 -0500 Subject: [PATCH 03/24] update wording --- docs/tutorials/virtual_data.ipynb | 144 +++++++++++++++++------------- 1 file changed, 83 insertions(+), 61 deletions(-) diff --git a/docs/tutorials/virtual_data.ipynb b/docs/tutorials/virtual_data.ipynb index 0d2ae5eb7..334215b70 100644 --- a/docs/tutorials/virtual_data.ipynb +++ b/docs/tutorials/virtual_data.ipynb @@ -7,9 +7,9 @@ "source": [ "# Virtual Datasets with `earthaccess`\n", "\n", - "earthaccess can help build and use virtual data stores for NASA archives by providing a unified gateway to search and stream cloud-resident granules without needing to manage complex authentication or local storage. \n", + "earthaccess can help build and use virtual data stores for NASA archives by providing a unified API to virtual data technologies.\n", "\n", - "Virtual datasets themselves function as a metadata-only representation of this distributed data, acting as a lightweight \"map\" that makes thousands of separate files appear as a single, contiguous array. Instead of moving or duplicating massive physical files, a virtual dataset stores the specific byte-range coordinates of where data chunks live. This allows us to perform high-performance, cloud-native analysis—like slicing through time or space—without the heavy overhead of traditional file-opening processes or the cost of data conversion.\n", + "Virtual datasets function as a metadata-only representation of this distributed data, acting as a lightweight \"map\" that makes thousands of separate files appear as a single, contiguous array. Instead of moving or duplicating massive physical files, a virtual dataset stores the specific byte-range coordinates of where data chunks live. This allows us to perform high-performance, cloud-native analysis like slicing through time or space without the heavy overhead of traditional file-opening processes or the cost of data conversion.\n", "\n", "### Unified virtual access API\n", "\n", @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "3e3d9c3e-890a-4be9-a107-f1e5784d0e31", "metadata": {}, "outputs": [ @@ -586,12 +586,12 @@ "data": { "application/vnd.holoviews_exec.v0+json": "", "text/html": [ - "
\n", - "
\n", + "
\n", + "
\n", "
\n", "" + "cell_type": "markdown", + "id": "63bfc86a-20e8-4b7e-ace1-892dff3f6b68", + "metadata": {}, + "source": [ + "# Virtual Datasets with `earthaccess`\n", + "\n", + "earthaccess can help build and use virtual data stores for NASA archives by providing a unified API to virtual data technologies.\n", + "\n", + "Virtual datasets function as a metadata-only representation of this distributed data, acting as a lightweight \"map\" that makes thousands of separate files appear as a single, contiguous array. Instead of moving or duplicating massive physical files, a virtual dataset stores the specific byte-range coordinates of where data chunks live. This allows us to perform high-performance, cloud-native analysis like slicing through time or space without the heavy overhead of traditional file-opening processes or the cost of data conversion.\n", + "\n", + "### Unified virtual access API\n", + "\n", + "`earthaccess` offers a unified virtual data API. On one hand, `virtualize(granules)` allows us to generate virtual data cubes from CMR granule searches. These cubes can be saved as [kerchunk](https://fsspec.github.io/kerchunk/) or [icechunk](https://icechunk.io/en/latest/understanding/concepts/) references that later can be opened with the new `open_virtual()` method. \n", + "\n", + "#### Key Workflow Components:\n", + "\n", + "* **`virtualize(granules)`**: Automates the metadata extraction (using dmrpp or native parsers) to create a VirtualiZarr object directly from your search results.\n", + " * Persistence: You can save these references to avoid re-parsing thousands of files in future sessions.\n", + "* **`open_virtual(path)`**: Instantly restores the session, providing an Xarray-compatible object that behaves like a local Zarr store but reads directly from NASA's S3 buckets or HTTP distributed files.\n", + "\n", + "| Format | Best For | Key Characteristic |\n", + "| :--- | :--- | :--- |\n", + "| **Kerchunk** | Static Archives | A stable, json or parquet map of existing archival files. |\n", + "| **Icechunk** | Dynamic Workflows | Transactional storage for Zarr that supports versioning and snapshots. |\n", + "\n", + "\n", + "## Opening virtual datasets with `earthaccess`" ] - }, - "metadata": {}, - "output_type": "display_data" }, { - "data": { - "application/javascript": [ - "(function(root) {\n", - " function now() {\n", - " return new Date();\n", - " }\n", - "\n", - " const force = true;\n", - " const version = '3.9.0'.replace('rc', '-rc.').replace('.dev', '-dev.');\n", - " const reloading = false;\n", - " const Bokeh = root.Bokeh;\n", - " const BK_RE = /^https:\\/\\/cdn\\.bokeh\\.org\\/bokeh\\/(release|dev)\\/bokeh-/;\n", - " const PN_RE = /^https:\\/\\/cdn\\.holoviz\\.org\\/panel\\/[^/]+\\/dist\\/panel/i;\n", - "\n", - " // Set a timeout for this load but only if we are not already initializing\n", - " if (typeof (root._bokeh_timeout) === \"undefined\" || (force || !root._bokeh_is_initializing)) {\n", - " root._bokeh_timeout = Date.now() + 5000;\n", - " root._bokeh_failed_load = false;\n", - " }\n", - "\n", - " function run_callbacks() {\n", - " try {\n", - " root._bokeh_onload_callbacks.forEach(function(callback) {\n", - " if (callback != null)\n", - " callback();\n", - " });\n", - " } finally {\n", - " delete root._bokeh_onload_callbacks;\n", - " }\n", - " console.debug(\"Bokeh: all callbacks have finished\");\n", - " }\n", - "\n", - " function load_libs(css_urls, js_urls, js_modules, js_exports, Bokeh, callback) {\n", - " if (css_urls == null) css_urls = [];\n", - " if (js_urls == null) js_urls = [];\n", - " if (js_modules == null) js_modules = [];\n", - " if (js_exports == null) js_exports = {};\n", - "\n", - " root._bokeh_onload_callbacks.push(callback);\n", - "\n", - " if (root._bokeh_is_loading > 0) {\n", - " // Don't load bokeh if it is still initializing\n", - " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", - " return null;\n", - " } else if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n", - " // There is nothing to load\n", - " run_callbacks();\n", - " return null;\n", - " }\n", - "\n", - " function on_load() {\n", - " root._bokeh_is_loading--;\n", - " if (root._bokeh_is_loading === 0) {\n", - " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", - " run_callbacks()\n", - " }\n", - " }\n", - " window._bokeh_on_load = on_load\n", - "\n", - " function on_error(e) {\n", - " const src_el = e.srcElement\n", - " console.error(\"failed to load \" + (src_el.href || src_el.src));\n", - " }\n", - "\n", - " const skip = [];\n", - " if (window.requirejs) {\n", - " window.requirejs.config({'packages': {}, 'paths': {}, 'shim': {}});\n", - " root._bokeh_is_loading = css_urls.length + 0;\n", - " } else {\n", - " root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n", - " }\n", - "\n", - " const existing_stylesheets = []\n", - " const links = document.getElementsByTagName('link')\n", - " for (let i = 0; i < links.length; i++) {\n", - " const link = links[i]\n", - " if (link.href != null) {\n", - " existing_stylesheets.push(link.href)\n", - " }\n", - " }\n", - " for (let i = 0; i < css_urls.length; i++) {\n", - " const url = css_urls[i];\n", - " const escaped = encodeURI(url)\n", - " if (existing_stylesheets.indexOf(escaped) !== -1) {\n", - " on_load()\n", - " continue;\n", - " }\n", - " const element = document.createElement(\"link\");\n", - " element.onload = on_load;\n", - " element.onerror = on_error;\n", - " element.rel = \"stylesheet\";\n", - " element.type = \"text/css\";\n", - " element.href = url;\n", - " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", - " document.body.appendChild(element);\n", - " } var existing_scripts = []\n", - " const scripts = document.getElementsByTagName('script')\n", - " for (let i = 0; i < scripts.length; i++) {\n", - " var script = scripts[i]\n", - " if (script.src != null) {\n", - " existing_scripts.push(script.src)\n", - " }\n", - " }\n", - " for (let i = 0; i < js_urls.length; i++) {\n", - " const url = js_urls[i];\n", - " const escaped = encodeURI(url)\n", - " const shouldSkip = skip.includes(escaped) || existing_scripts.includes(escaped)\n", - " const isBokehOrPanel = BK_RE.test(escaped) || PN_RE.test(escaped)\n", - " const missingOrBroken = Bokeh == null || Bokeh.Panel == null || (Bokeh.version != version && !Bokeh.versions?.has(version)) || Bokeh.versions?.get(version)?.Panel == null;\n", - " if (shouldSkip && !(isBokehOrPanel && missingOrBroken)) {\n", - " if (!window.requirejs) {\n", - " on_load();\n", - " }\n", - " continue;\n", - " }\n", - " const element = document.createElement('script');\n", - " element.onload = on_load;\n", - " element.onerror = on_error;\n", - " element.async = false;\n", - " element.src = url;\n", - " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " document.head.appendChild(element);\n", - " }\n", - " for (let i = 0; i < js_modules.length; i++) {\n", - " const url = js_modules[i];\n", - " const escaped = encodeURI(url)\n", - " if (skip.indexOf(escaped) !== -1 || existing_scripts.indexOf(escaped) !== -1) {\n", - " if (!window.requirejs) {\n", - " on_load();\n", - " }\n", - " continue;\n", - " }\n", - " var element = document.createElement('script');\n", - " element.onload = on_load;\n", - " element.onerror = on_error;\n", - " element.async = false;\n", - " element.src = url;\n", - " element.type = \"module\";\n", - " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " document.head.appendChild(element);\n", - " }\n", - " for (const name in js_exports) {\n", - " const url = js_exports[name];\n", - " const escaped = encodeURI(url)\n", - " if (skip.indexOf(escaped) >= 0 || root[name] != null) {\n", - " if (!window.requirejs) {\n", - " on_load();\n", - " }\n", - " continue;\n", - " }\n", - " var element = document.createElement('script');\n", - " element.onerror = on_error;\n", - " element.async = false;\n", - " element.type = \"module\";\n", - " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " element.textContent = `\n", - " import ${name} from \"${url}\"\n", - " window.${name} = ${name}\n", - " window._bokeh_on_load()\n", - " `\n", - " document.head.appendChild(element);\n", - " }\n", - " if (!js_urls.length && !js_modules.length) {\n", - " on_load()\n", - " }\n", - " };\n", - "\n", - " function inject_raw_css(css) {\n", - " const element = document.createElement(\"style\");\n", - " element.appendChild(document.createTextNode(css));\n", - " document.body.appendChild(element);\n", - " }\n", - "\n", - " const js_urls = [\"https://cdn.holoviz.org/panel/1.8.10/dist/bundled/reactiveesm/es-module-shims@^1.10.0/dist/es-module-shims.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.9.0.min.js\", \"https://cdn.holoviz.org/panel/1.8.10/dist/panel.min.js\"];\n", - " const js_modules = [];\n", - " const js_exports = {};\n", - " const css_urls = [];\n", - " const inline_js = [ function(Bokeh) {\n", - " Bokeh.set_log_level(\"info\");\n", - " },\n", - "function(Bokeh) {} // ensure no trailing comma for IE\n", - " ];\n", - "\n", - " function run_inline_js() {\n", - " if ((root.Bokeh !== undefined) || (force === true)) {\n", - " for (let i = 0; i < inline_js.length; i++) {\n", - " try {\n", - " inline_js[i].call(root, root.Bokeh);\n", - " } catch(e) {\n", - " if (!reloading) {\n", - " throw e;\n", - " }\n", - " }\n", - " }\n", - " } else if (Date.now() < root._bokeh_timeout) {\n", - " setTimeout(run_inline_js, 100);\n", - " } else if (!root._bokeh_failed_load) {\n", - " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", - " root._bokeh_failed_load = true;\n", - " }\n", - " root._bokeh_is_initializing = false;\n", - " }\n", - "\n", - " function load_or_wait() {\n", - " // Implement a backoff loop that tries to ensure we do not load multiple\n", - " // versions of Bokeh and its dependencies at the same time.\n", - " // In recent versions we use the root._bokeh_is_initializing flag\n", - " // to determine whether there is an ongoing attempt to initialize\n", - " // bokeh, however for backward compatibility we also try to ensure\n", - " // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n", - " // before older versions are fully initialized.\n", - " if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n", - " // If the timeout and bokeh was not successfully loaded we reset\n", - " // everything and try loading again\n", - " root._bokeh_timeout = Date.now() + 5000;\n", - " root._bokeh_is_initializing = false;\n", - " root._bokeh_onload_callbacks = undefined;\n", - " root._bokeh_is_loading = 0;\n", - " console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n", - " load_or_wait();\n", - " } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n", - " setTimeout(load_or_wait, 100);\n", - " } else {\n", - " root._bokeh_is_initializing = true;\n", - " root._bokeh_onload_callbacks = [];\n", - " const bokeh_loaded = Bokeh != null && ((Bokeh.version === version && Bokeh.Panel) || (Bokeh.versions?.has(version) && Bokeh.versions.get(version)?.Panel));\n", - " if (!reloading && !bokeh_loaded) {\n", - " if (root.Bokeh) {\n", - " root.Bokeh = undefined;\n", - " }\n", - " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", - " }\n", - " load_libs(css_urls, js_urls, js_modules, js_exports, Bokeh, function() {\n", - " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", - " run_inline_js();\n", - " if (Bokeh != undefined && !reloading) {\n", - " const NewBokeh = root.Bokeh;\n", - " if (Bokeh.versions === undefined) {\n", - " Bokeh.versions = new Map();\n", - " }\n", - " if (NewBokeh.version !== Bokeh.version) {\n", - " Bokeh[NewBokeh.version] = NewBokeh;\n", - " Bokeh.versions.set(NewBokeh.version, NewBokeh);\n", - " }\n", - " root.Bokeh = Bokeh;\n", - " }\n", - " });\n", - " }\n", - " }\n", - " // Give older versions of the autoload script a head-start to ensure\n", - " // they initialize before we start loading newer version.\n", - " setTimeout(load_or_wait, 100)\n", - "}(window));" + "cell_type": "code", + "execution_count": 2, + "id": "3e3d9c3e-890a-4be9-a107-f1e5784d0e31", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "(function(root) {\n", + " function now() {\n", + " return new Date();\n", + " }\n", + "\n", + " const force = true;\n", + " const version = '3.9.0'.replace('rc', '-rc.').replace('.dev', '-dev.');\n", + " const reloading = false;\n", + " const Bokeh = root.Bokeh;\n", + " const BK_RE = /^https:\\/\\/cdn\\.bokeh\\.org\\/bokeh\\/(release|dev)\\/bokeh-/;\n", + " const PN_RE = /^https:\\/\\/cdn\\.holoviz\\.org\\/panel\\/[^/]+\\/dist\\/panel/i;\n", + "\n", + " // Set a timeout for this load but only if we are not already initializing\n", + " if (typeof (root._bokeh_timeout) === \"undefined\" || (force || !root._bokeh_is_initializing)) {\n", + " root._bokeh_timeout = Date.now() + 5000;\n", + " root._bokeh_failed_load = false;\n", + " }\n", + "\n", + " function run_callbacks() {\n", + " try {\n", + " root._bokeh_onload_callbacks.forEach(function(callback) {\n", + " if (callback != null)\n", + " callback();\n", + " });\n", + " } finally {\n", + " delete root._bokeh_onload_callbacks;\n", + " }\n", + " console.debug(\"Bokeh: all callbacks have finished\");\n", + " }\n", + "\n", + " function load_libs(css_urls, js_urls, js_modules, js_exports, Bokeh, callback) {\n", + " if (css_urls == null) css_urls = [];\n", + " if (js_urls == null) js_urls = [];\n", + " if (js_modules == null) js_modules = [];\n", + " if (js_exports == null) js_exports = {};\n", + "\n", + " root._bokeh_onload_callbacks.push(callback);\n", + "\n", + " if (root._bokeh_is_loading > 0) {\n", + " // Don't load bokeh if it is still initializing\n", + " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", + " return null;\n", + " } else if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n", + " // There is nothing to load\n", + " run_callbacks();\n", + " return null;\n", + " }\n", + "\n", + " function on_load() {\n", + " root._bokeh_is_loading--;\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", + " run_callbacks()\n", + " }\n", + " }\n", + " window._bokeh_on_load = on_load\n", + "\n", + " function on_error(e) {\n", + " const src_el = e.srcElement\n", + " console.error(\"failed to load \" + (src_el.href || src_el.src));\n", + " }\n", + "\n", + " const skip = [];\n", + " if (window.requirejs) {\n", + " window.requirejs.config({'packages': {}, 'paths': {}, 'shim': {}});\n", + " root._bokeh_is_loading = css_urls.length + 0;\n", + " } else {\n", + " root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n", + " }\n", + "\n", + " const existing_stylesheets = []\n", + " const links = document.getElementsByTagName('link')\n", + " for (let i = 0; i < links.length; i++) {\n", + " const link = links[i]\n", + " if (link.href != null) {\n", + " existing_stylesheets.push(link.href)\n", + " }\n", + " }\n", + " for (let i = 0; i < css_urls.length; i++) {\n", + " const url = css_urls[i];\n", + " const escaped = encodeURI(url)\n", + " if (existing_stylesheets.indexOf(escaped) !== -1) {\n", + " on_load()\n", + " continue;\n", + " }\n", + " const element = document.createElement(\"link\");\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.rel = \"stylesheet\";\n", + " element.type = \"text/css\";\n", + " element.href = url;\n", + " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", + " document.body.appendChild(element);\n", + " } var existing_scripts = []\n", + " const scripts = document.getElementsByTagName('script')\n", + " for (let i = 0; i < scripts.length; i++) {\n", + " var script = scripts[i]\n", + " if (script.src != null) {\n", + " existing_scripts.push(script.src)\n", + " }\n", + " }\n", + " for (let i = 0; i < js_urls.length; i++) {\n", + " const url = js_urls[i];\n", + " const escaped = encodeURI(url)\n", + " const shouldSkip = skip.includes(escaped) || existing_scripts.includes(escaped)\n", + " const isBokehOrPanel = BK_RE.test(escaped) || PN_RE.test(escaped)\n", + " const missingOrBroken = Bokeh == null || Bokeh.Panel == null || (Bokeh.version != version && !Bokeh.versions?.has(version)) || Bokeh.versions?.get(version)?.Panel == null;\n", + " if (shouldSkip && !(isBokehOrPanel && missingOrBroken)) {\n", + " if (!window.requirejs) {\n", + " on_load();\n", + " }\n", + " continue;\n", + " }\n", + " const element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.src = url;\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " for (let i = 0; i < js_modules.length; i++) {\n", + " const url = js_modules[i];\n", + " const escaped = encodeURI(url)\n", + " if (skip.indexOf(escaped) !== -1 || existing_scripts.indexOf(escaped) !== -1) {\n", + " if (!window.requirejs) {\n", + " on_load();\n", + " }\n", + " continue;\n", + " }\n", + " var element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.src = url;\n", + " element.type = \"module\";\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " for (const name in js_exports) {\n", + " const url = js_exports[name];\n", + " const escaped = encodeURI(url)\n", + " if (skip.indexOf(escaped) >= 0 || root[name] != null) {\n", + " if (!window.requirejs) {\n", + " on_load();\n", + " }\n", + " continue;\n", + " }\n", + " var element = document.createElement('script');\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.type = \"module\";\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " element.textContent = `\n", + " import ${name} from \"${url}\"\n", + " window.${name} = ${name}\n", + " window._bokeh_on_load()\n", + " `\n", + " document.head.appendChild(element);\n", + " }\n", + " if (!js_urls.length && !js_modules.length) {\n", + " on_load()\n", + " }\n", + " };\n", + "\n", + " function inject_raw_css(css) {\n", + " const element = document.createElement(\"style\");\n", + " element.appendChild(document.createTextNode(css));\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " const js_urls = [\"https://cdn.holoviz.org/panel/1.8.10/dist/bundled/reactiveesm/es-module-shims@^1.10.0/dist/es-module-shims.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.9.0.min.js\", \"https://cdn.holoviz.org/panel/1.8.10/dist/panel.min.js\"];\n", + " const js_modules = [];\n", + " const js_exports = {};\n", + " const css_urls = [];\n", + " const inline_js = [ function(Bokeh) {\n", + " Bokeh.set_log_level(\"info\");\n", + " },\n", + "function(Bokeh) {} // ensure no trailing comma for IE\n", + " ];\n", + "\n", + " function run_inline_js() {\n", + " if ((root.Bokeh !== undefined) || (force === true)) {\n", + " for (let i = 0; i < inline_js.length; i++) {\n", + " try {\n", + " inline_js[i].call(root, root.Bokeh);\n", + " } catch(e) {\n", + " if (!reloading) {\n", + " throw e;\n", + " }\n", + " }\n", + " }\n", + " } else if (Date.now() < root._bokeh_timeout) {\n", + " setTimeout(run_inline_js, 100);\n", + " } else if (!root._bokeh_failed_load) {\n", + " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", + " root._bokeh_failed_load = true;\n", + " }\n", + " root._bokeh_is_initializing = false;\n", + " }\n", + "\n", + " function load_or_wait() {\n", + " // Implement a backoff loop that tries to ensure we do not load multiple\n", + " // versions of Bokeh and its dependencies at the same time.\n", + " // In recent versions we use the root._bokeh_is_initializing flag\n", + " // to determine whether there is an ongoing attempt to initialize\n", + " // bokeh, however for backward compatibility we also try to ensure\n", + " // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n", + " // before older versions are fully initialized.\n", + " if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n", + " // If the timeout and bokeh was not successfully loaded we reset\n", + " // everything and try loading again\n", + " root._bokeh_timeout = Date.now() + 5000;\n", + " root._bokeh_is_initializing = false;\n", + " root._bokeh_onload_callbacks = undefined;\n", + " root._bokeh_is_loading = 0;\n", + " console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n", + " load_or_wait();\n", + " } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n", + " setTimeout(load_or_wait, 100);\n", + " } else {\n", + " root._bokeh_is_initializing = true;\n", + " root._bokeh_onload_callbacks = [];\n", + " const bokeh_loaded = Bokeh != null && ((Bokeh.version === version && Bokeh.Panel) || (Bokeh.versions?.has(version) && Bokeh.versions.get(version)?.Panel));\n", + " if (!reloading && !bokeh_loaded) {\n", + " if (root.Bokeh) {\n", + " root.Bokeh = undefined;\n", + " }\n", + " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", + " }\n", + " load_libs(css_urls, js_urls, js_modules, js_exports, Bokeh, function() {\n", + " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", + " run_inline_js();\n", + " if (Bokeh != undefined && !reloading) {\n", + " const NewBokeh = root.Bokeh;\n", + " if (Bokeh.versions === undefined) {\n", + " Bokeh.versions = new Map();\n", + " }\n", + " if (NewBokeh.version !== Bokeh.version) {\n", + " Bokeh[NewBokeh.version] = NewBokeh;\n", + " Bokeh.versions.set(NewBokeh.version, NewBokeh);\n", + " }\n", + " root.Bokeh = Bokeh;\n", + " }\n", + " });\n", + " }\n", + " }\n", + " // Give older versions of the autoload script a head-start to ensure\n", + " // they initialize before we start loading newer version.\n", + " setTimeout(load_or_wait, 100)\n", + "}(window));" + ], + "application/vnd.holoviews_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n const version = '3.9.0'.replace('rc', '-rc.').replace('.dev', '-dev.');\n const reloading = false;\n const Bokeh = root.Bokeh;\n const BK_RE = /^https:\\/\\/cdn\\.bokeh\\.org\\/bokeh\\/(release|dev)\\/bokeh-/;\n const PN_RE = /^https:\\/\\/cdn\\.holoviz\\.org\\/panel\\/[^/]+\\/dist\\/panel/i;\n\n // Set a timeout for this load but only if we are not already initializing\n if (typeof (root._bokeh_timeout) === \"undefined\" || (force || !root._bokeh_is_initializing)) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks;\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, js_modules, js_exports, Bokeh, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n if (js_modules == null) js_modules = [];\n if (js_exports == null) js_exports = {};\n\n root._bokeh_onload_callbacks.push(callback);\n\n if (root._bokeh_is_loading > 0) {\n // Don't load bokeh if it is still initializing\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n } else if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n // There is nothing to load\n run_callbacks();\n return null;\n }\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n window._bokeh_on_load = on_load\n\n function on_error(e) {\n const src_el = e.srcElement\n console.error(\"failed to load \" + (src_el.href || src_el.src));\n }\n\n const skip = [];\n if (window.requirejs) {\n window.requirejs.config({'packages': {}, 'paths': {}, 'shim': {}});\n root._bokeh_is_loading = css_urls.length + 0;\n } else {\n root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n }\n\n const existing_stylesheets = []\n const links = document.getElementsByTagName('link')\n for (let i = 0; i < links.length; i++) {\n const link = links[i]\n if (link.href != null) {\n existing_stylesheets.push(link.href)\n }\n }\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const escaped = encodeURI(url)\n if (existing_stylesheets.indexOf(escaped) !== -1) {\n on_load()\n continue;\n }\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n } var existing_scripts = []\n const scripts = document.getElementsByTagName('script')\n for (let i = 0; i < scripts.length; i++) {\n var script = scripts[i]\n if (script.src != null) {\n existing_scripts.push(script.src)\n }\n }\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const escaped = encodeURI(url)\n const shouldSkip = skip.includes(escaped) || existing_scripts.includes(escaped)\n const isBokehOrPanel = BK_RE.test(escaped) || PN_RE.test(escaped)\n const missingOrBroken = Bokeh == null || Bokeh.Panel == null || (Bokeh.version != version && !Bokeh.versions?.has(version)) || Bokeh.versions?.get(version)?.Panel == null;\n if (shouldSkip && !(isBokehOrPanel && missingOrBroken)) {\n if (!window.requirejs) {\n on_load();\n }\n continue;\n }\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (let i = 0; i < js_modules.length; i++) {\n const url = js_modules[i];\n const escaped = encodeURI(url)\n if (skip.indexOf(escaped) !== -1 || existing_scripts.indexOf(escaped) !== -1) {\n if (!window.requirejs) {\n on_load();\n }\n continue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (const name in js_exports) {\n const url = js_exports[name];\n const escaped = encodeURI(url)\n if (skip.indexOf(escaped) >= 0 || root[name] != null) {\n if (!window.requirejs) {\n on_load();\n }\n continue;\n }\n var element = document.createElement('script');\n element.onerror = on_error;\n element.async = false;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n element.textContent = `\n import ${name} from \"${url}\"\n window.${name} = ${name}\n window._bokeh_on_load()\n `\n document.head.appendChild(element);\n }\n if (!js_urls.length && !js_modules.length) {\n on_load()\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.holoviz.org/panel/1.8.10/dist/bundled/reactiveesm/es-module-shims@^1.10.0/dist/es-module-shims.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.9.0.min.js\", \"https://cdn.holoviz.org/panel/1.8.10/dist/panel.min.js\"];\n const js_modules = [];\n const js_exports = {};\n const css_urls = [];\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n ];\n\n function run_inline_js() {\n if ((root.Bokeh !== undefined) || (force === true)) {\n for (let i = 0; i < inline_js.length; i++) {\n try {\n inline_js[i].call(root, root.Bokeh);\n } catch(e) {\n if (!reloading) {\n throw e;\n }\n }\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n }\n root._bokeh_is_initializing = false;\n }\n\n function load_or_wait() {\n // Implement a backoff loop that tries to ensure we do not load multiple\n // versions of Bokeh and its dependencies at the same time.\n // In recent versions we use the root._bokeh_is_initializing flag\n // to determine whether there is an ongoing attempt to initialize\n // bokeh, however for backward compatibility we also try to ensure\n // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n // before older versions are fully initialized.\n if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n // If the timeout and bokeh was not successfully loaded we reset\n // everything and try loading again\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_is_initializing = false;\n root._bokeh_onload_callbacks = undefined;\n root._bokeh_is_loading = 0;\n console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n load_or_wait();\n } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n setTimeout(load_or_wait, 100);\n } else {\n root._bokeh_is_initializing = true;\n root._bokeh_onload_callbacks = [];\n const bokeh_loaded = Bokeh != null && ((Bokeh.version === version && Bokeh.Panel) || (Bokeh.versions?.has(version) && Bokeh.versions.get(version)?.Panel));\n if (!reloading && !bokeh_loaded) {\n if (root.Bokeh) {\n root.Bokeh = undefined;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n }\n load_libs(css_urls, js_urls, js_modules, js_exports, Bokeh, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n if (Bokeh != undefined && !reloading) {\n const NewBokeh = root.Bokeh;\n if (Bokeh.versions === undefined) {\n Bokeh.versions = new Map();\n }\n if (NewBokeh.version !== Bokeh.version) {\n Bokeh[NewBokeh.version] = NewBokeh;\n Bokeh.versions.set(NewBokeh.version, NewBokeh);\n }\n root.Bokeh = Bokeh;\n }\n });\n }\n }\n // Give older versions of the autoload script a head-start to ensure\n // they initialize before we start loading newer version.\n setTimeout(load_or_wait, 100)\n}(window));" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "\n", + "if ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n", + " window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n", + "}\n", + "\n", + "\n", + " function JupyterCommManager() {\n", + " }\n", + "\n", + " JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n", + " if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", + " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", + " comm_manager.register_target(comm_id, function(comm) {\n", + " comm.on_msg(msg_handler);\n", + " });\n", + " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", + " window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n", + " comm.onMsg = msg_handler;\n", + " });\n", + " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", + " google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n", + " var messages = comm.messages[Symbol.asyncIterator]();\n", + " function processIteratorResult(result) {\n", + " var message = result.value;\n", + " var content = {data: message.data, comm_id};\n", + " var buffers = []\n", + " for (var buffer of message.buffers || []) {\n", + " buffers.push(new DataView(buffer))\n", + " }\n", + " var metadata = message.metadata || {};\n", + " var msg = {content, buffers, metadata}\n", + " msg_handler(msg);\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " return messages.next().then(processIteratorResult);\n", + " })\n", + " }\n", + " }\n", + "\n", + " JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n", + " if (comm_id in window.PyViz.comms) {\n", + " return window.PyViz.comms[comm_id];\n", + " } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", + " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", + " var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n", + " if (msg_handler) {\n", + " comm.on_msg(msg_handler);\n", + " }\n", + " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", + " var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n", + " let retries = 0;\n", + " const open = () => {\n", + " if (comm.active) {\n", + " comm.open();\n", + " } else if (retries > 3) {\n", + " console.warn('Comm target never activated')\n", + " } else {\n", + " retries += 1\n", + " setTimeout(open, 500)\n", + " }\n", + " }\n", + " if (comm.active) {\n", + " comm.open();\n", + " } else {\n", + " setTimeout(open, 500)\n", + " }\n", + " if (msg_handler) {\n", + " comm.onMsg = msg_handler;\n", + " }\n", + " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", + " var comm_promise = google.colab.kernel.comms.open(comm_id)\n", + " comm_promise.then((comm) => {\n", + " window.PyViz.comms[comm_id] = comm;\n", + " if (msg_handler) {\n", + " var messages = comm.messages[Symbol.asyncIterator]();\n", + " function processIteratorResult(result) {\n", + " var message = result.value;\n", + " var content = {data: message.data};\n", + " var metadata = message.metadata || {comm_id};\n", + " var msg = {content, metadata}\n", + " msg_handler(msg);\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " })\n", + " var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n", + " return comm_promise.then((comm) => {\n", + " comm.send(data, metadata, buffers, disposeOnDone);\n", + " });\n", + " };\n", + " var comm = {\n", + " send: sendClosure\n", + " };\n", + " }\n", + " window.PyViz.comms[comm_id] = comm;\n", + " return comm;\n", + " }\n", + " window.PyViz.comm_manager = new JupyterCommManager();\n", + " \n", + "\n", + "\n", + "var JS_MIME_TYPE = 'application/javascript';\n", + "var HTML_MIME_TYPE = 'text/html';\n", + "var EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\n", + "var CLASS_NAME = 'output';\n", + "\n", + "/**\n", + " * Render data to the DOM node\n", + " */\n", + "function render(props, node) {\n", + " var div = document.createElement(\"div\");\n", + " var script = document.createElement(\"script\");\n", + " node.appendChild(div);\n", + " node.appendChild(script);\n", + "}\n", + "\n", + "/**\n", + " * Handle when a new output is added\n", + " */\n", + "function handle_add_output(event, handle) {\n", + " var output_area = handle.output_area;\n", + " var output = handle.output;\n", + " if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n", + " return\n", + " }\n", + " var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", + " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", + " if (id !== undefined) {\n", + " var nchildren = toinsert.length;\n", + " var html_node = toinsert[nchildren-1].children[0];\n", + " html_node.innerHTML = output.data[HTML_MIME_TYPE];\n", + " var scripts = [];\n", + " var nodelist = html_node.querySelectorAll(\"script\");\n", + " for (var i in nodelist) {\n", + " if (nodelist.hasOwnProperty(i)) {\n", + " scripts.push(nodelist[i])\n", + " }\n", + " }\n", + "\n", + " scripts.forEach( function (oldScript) {\n", + " var newScript = document.createElement(\"script\");\n", + " var attrs = [];\n", + " var nodemap = oldScript.attributes;\n", + " for (var j in nodemap) {\n", + " if (nodemap.hasOwnProperty(j)) {\n", + " attrs.push(nodemap[j])\n", + " }\n", + " }\n", + " attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n", + " newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n", + " oldScript.parentNode.replaceChild(newScript, oldScript);\n", + " });\n", + " if (JS_MIME_TYPE in output.data) {\n", + " toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n", + " }\n", + " output_area._hv_plot_id = id;\n", + " if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n", + " window.PyViz.plot_index[id] = Bokeh.index[id];\n", + " } else {\n", + " window.PyViz.plot_index[id] = null;\n", + " }\n", + " } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", + " var bk_div = document.createElement(\"div\");\n", + " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", + " var script_attrs = bk_div.children[0].attributes;\n", + " for (var i = 0; i < script_attrs.length; i++) {\n", + " toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n", + " }\n", + " // store reference to server id on output_area\n", + " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", + " }\n", + "}\n", + "\n", + "/**\n", + " * Handle when an output is cleared or removed\n", + " */\n", + "function handle_clear_output(event, handle) {\n", + " var id = handle.cell.output_area._hv_plot_id;\n", + " var server_id = handle.cell.output_area._bokeh_server_id;\n", + " if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n", + " var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n", + " if (server_id !== null) {\n", + " comm.send({event_type: 'server_delete', 'id': server_id});\n", + " return;\n", + " } else if (comm !== null) {\n", + " comm.send({event_type: 'delete', 'id': id});\n", + " }\n", + " delete PyViz.plot_index[id];\n", + " if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n", + " var doc = window.Bokeh.index[id].model.document\n", + " doc.clear();\n", + " const i = window.Bokeh.documents.indexOf(doc);\n", + " if (i > -1) {\n", + " window.Bokeh.documents.splice(i, 1);\n", + " }\n", + " }\n", + "}\n", + "\n", + "/**\n", + " * Handle kernel restart event\n", + " */\n", + "function handle_kernel_cleanup(event, handle) {\n", + " delete PyViz.comms[\"hv-extension-comm\"];\n", + " window.PyViz.plot_index = {}\n", + "}\n", + "\n", + "/**\n", + " * Handle update_display_data messages\n", + " */\n", + "function handle_update_output(event, handle) {\n", + " handle_clear_output(event, {cell: {output_area: handle.output_area}})\n", + " handle_add_output(event, handle)\n", + "}\n", + "\n", + "function register_renderer(events, OutputArea) {\n", + " function append_mime(data, metadata, element) {\n", + " // create a DOM node to render to\n", + " var toinsert = this.create_output_subarea(\n", + " metadata,\n", + " CLASS_NAME,\n", + " EXEC_MIME_TYPE\n", + " );\n", + " this.keyboard_manager.register_events(toinsert);\n", + " // Render to node\n", + " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", + " render(props, toinsert[0]);\n", + " element.append(toinsert);\n", + " return toinsert\n", + " }\n", + "\n", + " events.on('output_added.OutputArea', handle_add_output);\n", + " events.on('output_updated.OutputArea', handle_update_output);\n", + " events.on('clear_output.CodeCell', handle_clear_output);\n", + " events.on('delete.Cell', handle_clear_output);\n", + " events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n", + "\n", + " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", + " safe: true,\n", + " index: 0\n", + " });\n", + "}\n", + "\n", + "if (window.Jupyter !== undefined) {\n", + " try {\n", + " var events = require('base/js/events');\n", + " var OutputArea = require('notebook/js/outputarea').OutputArea;\n", + " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", + " register_renderer(events, OutputArea);\n", + " }\n", + " } catch(err) {\n", + " }\n", + "}\n" + ], + "application/vnd.holoviews_load.v0+json": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n function JupyterCommManager() {\n }\n\n JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n comm_manager.register_target(comm_id, function(comm) {\n comm.on_msg(msg_handler);\n });\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n comm.onMsg = msg_handler;\n });\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data, comm_id};\n var buffers = []\n for (var buffer of message.buffers || []) {\n buffers.push(new DataView(buffer))\n }\n var metadata = message.metadata || {};\n var msg = {content, buffers, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n })\n }\n }\n\n JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n if (comm_id in window.PyViz.comms) {\n return window.PyViz.comms[comm_id];\n } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n if (msg_handler) {\n comm.on_msg(msg_handler);\n }\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n let retries = 0;\n const open = () => {\n if (comm.active) {\n comm.open();\n } else if (retries > 3) {\n console.warn('Comm target never activated')\n } else {\n retries += 1\n setTimeout(open, 500)\n }\n }\n if (comm.active) {\n comm.open();\n } else {\n setTimeout(open, 500)\n }\n if (msg_handler) {\n comm.onMsg = msg_handler;\n }\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n var comm_promise = google.colab.kernel.comms.open(comm_id)\n comm_promise.then((comm) => {\n window.PyViz.comms[comm_id] = comm;\n if (msg_handler) {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data};\n var metadata = message.metadata || {comm_id};\n var msg = {content, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n }\n })\n var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n return comm_promise.then((comm) => {\n comm.send(data, metadata, buffers, disposeOnDone);\n });\n };\n var comm = {\n send: sendClosure\n };\n }\n window.PyViz.comms[comm_id] = comm;\n return comm;\n }\n window.PyViz.comm_manager = new JupyterCommManager();\n \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n var div = document.createElement(\"div\");\n var script = document.createElement(\"script\");\n node.appendChild(div);\n node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n var output_area = handle.output_area;\n var output = handle.output;\n if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n return\n }\n var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n if (id !== undefined) {\n var nchildren = toinsert.length;\n var html_node = toinsert[nchildren-1].children[0];\n html_node.innerHTML = output.data[HTML_MIME_TYPE];\n var scripts = [];\n var nodelist = html_node.querySelectorAll(\"script\");\n for (var i in nodelist) {\n if (nodelist.hasOwnProperty(i)) {\n scripts.push(nodelist[i])\n }\n }\n\n scripts.forEach( function (oldScript) {\n var newScript = document.createElement(\"script\");\n var attrs = [];\n var nodemap = oldScript.attributes;\n for (var j in nodemap) {\n if (nodemap.hasOwnProperty(j)) {\n attrs.push(nodemap[j])\n }\n }\n attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n oldScript.parentNode.replaceChild(newScript, oldScript);\n });\n if (JS_MIME_TYPE in output.data) {\n toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n }\n output_area._hv_plot_id = id;\n if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n window.PyViz.plot_index[id] = Bokeh.index[id];\n } else {\n window.PyViz.plot_index[id] = null;\n }\n } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n var bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n var script_attrs = bk_div.children[0].attributes;\n for (var i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n var id = handle.cell.output_area._hv_plot_id;\n var server_id = handle.cell.output_area._bokeh_server_id;\n if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n if (server_id !== null) {\n comm.send({event_type: 'server_delete', 'id': server_id});\n return;\n } else if (comm !== null) {\n comm.send({event_type: 'delete', 'id': id});\n }\n delete PyViz.plot_index[id];\n if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n var doc = window.Bokeh.index[id].model.document\n doc.clear();\n const i = window.Bokeh.documents.indexOf(doc);\n if (i > -1) {\n window.Bokeh.documents.splice(i, 1);\n }\n }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n delete PyViz.comms[\"hv-extension-comm\"];\n window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n handle_clear_output(event, {cell: {output_area: handle.output_area}})\n handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n var toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[0]);\n element.append(toinsert);\n return toinsert\n }\n\n events.on('output_added.OutputArea', handle_add_output);\n events.on('output_updated.OutputArea', handle_update_output);\n events.on('clear_output.CodeCell', handle_clear_output);\n events.on('delete.Cell', handle_clear_output);\n events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n safe: true,\n index: 0\n });\n}\n\nif (window.Jupyter !== undefined) {\n try {\n var events = require('base/js/events');\n var OutputArea = require('notebook/js/outputarea').OutputArea;\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n } catch(err) {\n }\n}\n" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.holoviews_exec.v0+json": "", + "text/html": [ + "
\n", + "
\n", + "
\n", + "" + ] + }, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "383dc158-b95c-4b78-923f-2b37baa80300" + } + }, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 18.7 s, sys: 3.57 s, total: 22.2 s\n", + "Wall time: 26.4 s\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 70TB\n",
+              "Dimensions:           (time: 3868, lat: 17999, lon: 36000)\n",
+              "Coordinates:\n",
+              "  * time              (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n",
+              "  * lat               (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n",
+              "  * lon               (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n",
+              "Data variables:\n",
+              "    analysed_sst      (time, lat, lon) float64 20TB ...\n",
+              "    analysis_error    (time, lat, lon) float64 20TB ...\n",
+              "    mask              (time, lat, lon) float32 10TB ...\n",
+              "    sea_ice_fraction  (time, lat, lon) float64 20TB ...\n",
+              "Attributes: (12/47)\n",
+              "    Conventions:                CF-1.5\n",
+              "    Metadata_Conventions:       Unidata Observation Dataset v1.0\n",
+              "    acknowledgment:             Please acknowledge the use of these data with...\n",
+              "    cdm_data_type:              grid\n",
+              "    comment:                    MUR = "Multi-scale Ultra-high Reolution"\n",
+              "    creator_email:              ghrsst@podaac.jpl.nasa.gov\n",
+              "    ...                         ...\n",
+              "    summary:                    A merged, multi-sensor L4 Foundation SST anal...\n",
+              "    time_coverage_end:          20020601T210000Z\n",
+              "    time_coverage_start:        20020531T210000Z\n",
+              "    title:                      Daily MUR SST, Final product\n",
+              "    uuid:                       27665bc0-d5fc-11e1-9b23-0800200c9a66\n",
+              "    westernmost_longitude:      -180.0
" + ], + "text/plain": [ + " Size: 70TB\n", + "Dimensions: (time: 3868, lat: 17999, lon: 36000)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n", + " * lat (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n", + " * lon (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n", + "Data variables:\n", + " analysed_sst (time, lat, lon) float64 20TB ...\n", + " analysis_error (time, lat, lon) float64 20TB ...\n", + " mask (time, lat, lon) float32 10TB ...\n", + " sea_ice_fraction (time, lat, lon) float64 20TB ...\n", + "Attributes: (12/47)\n", + " Conventions: CF-1.5\n", + " Metadata_Conventions: Unidata Observation Dataset v1.0\n", + " acknowledgment: Please acknowledge the use of these data with...\n", + " cdm_data_type: grid\n", + " comment: MUR = \"Multi-scale Ultra-high Reolution\"\n", + " creator_email: ghrsst@podaac.jpl.nasa.gov\n", + " ... ...\n", + " summary: A merged, multi-sensor L4 Foundation SST anal...\n", + " time_coverage_end: 20020601T210000Z\n", + " time_coverage_start: 20020531T210000Z\n", + " title: Daily MUR SST, Final product\n", + " uuid: 27665bc0-d5fc-11e1-9b23-0800200c9a66\n", + " westernmost_longitude: -180.0" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } ], - "application/vnd.holoviews_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n const version = '3.9.0'.replace('rc', '-rc.').replace('.dev', '-dev.');\n const reloading = false;\n const Bokeh = root.Bokeh;\n const BK_RE = /^https:\\/\\/cdn\\.bokeh\\.org\\/bokeh\\/(release|dev)\\/bokeh-/;\n const PN_RE = /^https:\\/\\/cdn\\.holoviz\\.org\\/panel\\/[^/]+\\/dist\\/panel/i;\n\n // Set a timeout for this load but only if we are not already initializing\n if (typeof (root._bokeh_timeout) === \"undefined\" || (force || !root._bokeh_is_initializing)) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks;\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, js_modules, js_exports, Bokeh, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n if (js_modules == null) js_modules = [];\n if (js_exports == null) js_exports = {};\n\n root._bokeh_onload_callbacks.push(callback);\n\n if (root._bokeh_is_loading > 0) {\n // Don't load bokeh if it is still initializing\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n } else if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n // There is nothing to load\n run_callbacks();\n return null;\n }\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n window._bokeh_on_load = on_load\n\n function on_error(e) {\n const src_el = e.srcElement\n console.error(\"failed to load \" + (src_el.href || src_el.src));\n }\n\n const skip = [];\n if (window.requirejs) {\n window.requirejs.config({'packages': {}, 'paths': {}, 'shim': {}});\n root._bokeh_is_loading = css_urls.length + 0;\n } else {\n root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n }\n\n const existing_stylesheets = []\n const links = document.getElementsByTagName('link')\n for (let i = 0; i < links.length; i++) {\n const link = links[i]\n if (link.href != null) {\n existing_stylesheets.push(link.href)\n }\n }\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const escaped = encodeURI(url)\n if (existing_stylesheets.indexOf(escaped) !== -1) {\n on_load()\n continue;\n }\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n } var existing_scripts = []\n const scripts = document.getElementsByTagName('script')\n for (let i = 0; i < scripts.length; i++) {\n var script = scripts[i]\n if (script.src != null) {\n existing_scripts.push(script.src)\n }\n }\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const escaped = encodeURI(url)\n const shouldSkip = skip.includes(escaped) || existing_scripts.includes(escaped)\n const isBokehOrPanel = BK_RE.test(escaped) || PN_RE.test(escaped)\n const missingOrBroken = Bokeh == null || Bokeh.Panel == null || (Bokeh.version != version && !Bokeh.versions?.has(version)) || Bokeh.versions?.get(version)?.Panel == null;\n if (shouldSkip && !(isBokehOrPanel && missingOrBroken)) {\n if (!window.requirejs) {\n on_load();\n }\n continue;\n }\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (let i = 0; i < js_modules.length; i++) {\n const url = js_modules[i];\n const escaped = encodeURI(url)\n if (skip.indexOf(escaped) !== -1 || existing_scripts.indexOf(escaped) !== -1) {\n if (!window.requirejs) {\n on_load();\n }\n continue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (const name in js_exports) {\n const url = js_exports[name];\n const escaped = encodeURI(url)\n if (skip.indexOf(escaped) >= 0 || root[name] != null) {\n if (!window.requirejs) {\n on_load();\n }\n continue;\n }\n var element = document.createElement('script');\n element.onerror = on_error;\n element.async = false;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n element.textContent = `\n import ${name} from \"${url}\"\n window.${name} = ${name}\n window._bokeh_on_load()\n `\n document.head.appendChild(element);\n }\n if (!js_urls.length && !js_modules.length) {\n on_load()\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.holoviz.org/panel/1.8.10/dist/bundled/reactiveesm/es-module-shims@^1.10.0/dist/es-module-shims.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.9.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.9.0.min.js\", \"https://cdn.holoviz.org/panel/1.8.10/dist/panel.min.js\"];\n const js_modules = [];\n const js_exports = {};\n const css_urls = [];\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n ];\n\n function run_inline_js() {\n if ((root.Bokeh !== undefined) || (force === true)) {\n for (let i = 0; i < inline_js.length; i++) {\n try {\n inline_js[i].call(root, root.Bokeh);\n } catch(e) {\n if (!reloading) {\n throw e;\n }\n }\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n }\n root._bokeh_is_initializing = false;\n }\n\n function load_or_wait() {\n // Implement a backoff loop that tries to ensure we do not load multiple\n // versions of Bokeh and its dependencies at the same time.\n // In recent versions we use the root._bokeh_is_initializing flag\n // to determine whether there is an ongoing attempt to initialize\n // bokeh, however for backward compatibility we also try to ensure\n // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n // before older versions are fully initialized.\n if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n // If the timeout and bokeh was not successfully loaded we reset\n // everything and try loading again\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_is_initializing = false;\n root._bokeh_onload_callbacks = undefined;\n root._bokeh_is_loading = 0;\n console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n load_or_wait();\n } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n setTimeout(load_or_wait, 100);\n } else {\n root._bokeh_is_initializing = true;\n root._bokeh_onload_callbacks = [];\n const bokeh_loaded = Bokeh != null && ((Bokeh.version === version && Bokeh.Panel) || (Bokeh.versions?.has(version) && Bokeh.versions.get(version)?.Panel));\n if (!reloading && !bokeh_loaded) {\n if (root.Bokeh) {\n root.Bokeh = undefined;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n }\n load_libs(css_urls, js_urls, js_modules, js_exports, Bokeh, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n if (Bokeh != undefined && !reloading) {\n const NewBokeh = root.Bokeh;\n if (Bokeh.versions === undefined) {\n Bokeh.versions = new Map();\n }\n if (NewBokeh.version !== Bokeh.version) {\n Bokeh[NewBokeh.version] = NewBokeh;\n Bokeh.versions.set(NewBokeh.version, NewBokeh);\n }\n root.Bokeh = Bokeh;\n }\n });\n }\n }\n // Give older versions of the autoload script a head-start to ensure\n // they initialize before we start loading newer version.\n setTimeout(load_or_wait, 100)\n}(window));" - }, - "metadata": {}, - "output_type": "display_data" + "source": [ + "%%time\n", + "\n", + "import warnings\n", + "\n", + "import earthaccess as ea\n", + "import xarray as xr\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "auth = ea.login()\n", + "\n", + "collection = ea.search_datasets(concept_id=\"C1996881146-POCLOUD\")[\n", + " 0\n", + "] # <- note that we are using the first item\n", + "\n", + "vds = ea.open_virtual(collection, load=True, force_external=True)\n", + "vds" + ] + }, + { + "cell_type": "markdown", + "id": "08b7fcea-bd3b-4d4d-bd86-ee2e190c79ba", + "metadata": {}, + "source": [ + "### open_virtual()\n", + "\n", + "`open_virtual()` can open virtual collections listed at the collection level metadata, fow now following the convention adopted by JPL of using the following URL in `RelatedUrls` \n", + "\n", + "```json\n", + "{\n", + " \"Description\": \"Virtual data set reference \",\n", + " \"URLContentType\": \"DistributionURL\",\n", + " \"URL\": \"https://archive.podaac.earthdata.nasa.gov/../MUR-JPL-L4-GLOB-v4.1_combined-ref.json\",\n", + " \"Type\": \"GET DATA\",\n", + " \"Subtype\": \"VIRTUAL COLLECTION\"\n", + "}\n", + "```\n", + "\n", + "> Note: We may have multiple virtual stores for any given collection so we anticipate this convention may change in the near future, for now the assumption is that each collection has at most one virtual representation. \n", + "\n", + "`open_virtual()` has the following signature that we explain in short:\n", + "\n", + "* **uri**: this can be a collection level result from `collection = earthaccess.search_datasets()[0]` a local path, or a publicly accessible reference URL. If we use a collection from earthaccess, we don't need to deal with the cumbersome auth scafolding for the underlaying libraries; in all cases we'll get an xarray dataset instance pointing to our virtual dataset.This can be kerchunk or Icechunk stores \n", + "* **access**: this one informs earthaccess if the virtual store points to S3 buckets or HTTP, without inspecting the references we need to be explicit for now. values are `direct` or `external` (same as `indirect`)\n", + "* **storage_options**: if we need custom authentication for the virtual store this works the same way xarray uses storage options, not needed for NASA data as it's automatic.\n", + "* **force_external**: This is a handy flag to force reference translation, this is if the only available store points to S3 but we are accessing the data from outside AWS `us-west-2` setting this to `True` will translate the URLs on the fly for out of region access.\n", + "* **load**: if we should load the indexes in our dataset, if we do we can address the dataset as any other xarray dataset and perform operations like subsetting and aggregations, if not we get a `virtualizarr` dataset backed by `ManifestArray` instances, this is only useful if we plan to add more virtual references and update the cube.\n", + "\n", + "The method signature is as follows:\n", + "\n", + "```python\n", + "ea.open_virtual(\n", + " uri: 'str | Path | earthaccess.DataCollection',\n", + " *,\n", + " access: 'str' = 'indirect',\n", + " storage_options: 'dict[str, Any] | None' = None,\n", + " force_external: 'bool' = False,\n", + " load: 'bool' = True,\n", + " **kwargs: 'Any',\n", + ") -> 'xr.Dataset'\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "0e2366e8-aab1-49ce-a66a-bfc019df2a3b", + "metadata": {}, + "source": [ + "If we try to open a virtual dataset on a collection that has no virtual distribution will result in an exception." + ] }, { - "data": { - "application/javascript": [ - "\n", - "if ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n", - " window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n", - "}\n", - "\n", - "\n", - " function JupyterCommManager() {\n", - " }\n", - "\n", - " JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n", - " if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", - " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", - " comm_manager.register_target(comm_id, function(comm) {\n", - " comm.on_msg(msg_handler);\n", - " });\n", - " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", - " window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n", - " comm.onMsg = msg_handler;\n", - " });\n", - " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", - " google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n", - " var messages = comm.messages[Symbol.asyncIterator]();\n", - " function processIteratorResult(result) {\n", - " var message = result.value;\n", - " var content = {data: message.data, comm_id};\n", - " var buffers = []\n", - " for (var buffer of message.buffers || []) {\n", - " buffers.push(new DataView(buffer))\n", - " }\n", - " var metadata = message.metadata || {};\n", - " var msg = {content, buffers, metadata}\n", - " msg_handler(msg);\n", - " return messages.next().then(processIteratorResult);\n", - " }\n", - " return messages.next().then(processIteratorResult);\n", - " })\n", - " }\n", - " }\n", - "\n", - " JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n", - " if (comm_id in window.PyViz.comms) {\n", - " return window.PyViz.comms[comm_id];\n", - " } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", - " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", - " var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n", - " if (msg_handler) {\n", - " comm.on_msg(msg_handler);\n", - " }\n", - " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", - " var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n", - " let retries = 0;\n", - " const open = () => {\n", - " if (comm.active) {\n", - " comm.open();\n", - " } else if (retries > 3) {\n", - " console.warn('Comm target never activated')\n", - " } else {\n", - " retries += 1\n", - " setTimeout(open, 500)\n", - " }\n", - " }\n", - " if (comm.active) {\n", - " comm.open();\n", - " } else {\n", - " setTimeout(open, 500)\n", - " }\n", - " if (msg_handler) {\n", - " comm.onMsg = msg_handler;\n", - " }\n", - " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", - " var comm_promise = google.colab.kernel.comms.open(comm_id)\n", - " comm_promise.then((comm) => {\n", - " window.PyViz.comms[comm_id] = comm;\n", - " if (msg_handler) {\n", - " var messages = comm.messages[Symbol.asyncIterator]();\n", - " function processIteratorResult(result) {\n", - " var message = result.value;\n", - " var content = {data: message.data};\n", - " var metadata = message.metadata || {comm_id};\n", - " var msg = {content, metadata}\n", - " msg_handler(msg);\n", - " return messages.next().then(processIteratorResult);\n", - " }\n", - " return messages.next().then(processIteratorResult);\n", - " }\n", - " })\n", - " var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n", - " return comm_promise.then((comm) => {\n", - " comm.send(data, metadata, buffers, disposeOnDone);\n", - " });\n", - " };\n", - " var comm = {\n", - " send: sendClosure\n", - " };\n", - " }\n", - " window.PyViz.comms[comm_id] = comm;\n", - " return comm;\n", - " }\n", - " window.PyViz.comm_manager = new JupyterCommManager();\n", - " \n", - "\n", - "\n", - "var JS_MIME_TYPE = 'application/javascript';\n", - "var HTML_MIME_TYPE = 'text/html';\n", - "var EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\n", - "var CLASS_NAME = 'output';\n", - "\n", - "/**\n", - " * Render data to the DOM node\n", - " */\n", - "function render(props, node) {\n", - " var div = document.createElement(\"div\");\n", - " var script = document.createElement(\"script\");\n", - " node.appendChild(div);\n", - " node.appendChild(script);\n", - "}\n", - "\n", - "/**\n", - " * Handle when a new output is added\n", - " */\n", - "function handle_add_output(event, handle) {\n", - " var output_area = handle.output_area;\n", - " var output = handle.output;\n", - " if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n", - " return\n", - " }\n", - " var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", - " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", - " if (id !== undefined) {\n", - " var nchildren = toinsert.length;\n", - " var html_node = toinsert[nchildren-1].children[0];\n", - " html_node.innerHTML = output.data[HTML_MIME_TYPE];\n", - " var scripts = [];\n", - " var nodelist = html_node.querySelectorAll(\"script\");\n", - " for (var i in nodelist) {\n", - " if (nodelist.hasOwnProperty(i)) {\n", - " scripts.push(nodelist[i])\n", - " }\n", - " }\n", - "\n", - " scripts.forEach( function (oldScript) {\n", - " var newScript = document.createElement(\"script\");\n", - " var attrs = [];\n", - " var nodemap = oldScript.attributes;\n", - " for (var j in nodemap) {\n", - " if (nodemap.hasOwnProperty(j)) {\n", - " attrs.push(nodemap[j])\n", - " }\n", - " }\n", - " attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n", - " newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n", - " oldScript.parentNode.replaceChild(newScript, oldScript);\n", - " });\n", - " if (JS_MIME_TYPE in output.data) {\n", - " toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n", - " }\n", - " output_area._hv_plot_id = id;\n", - " if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n", - " window.PyViz.plot_index[id] = Bokeh.index[id];\n", - " } else {\n", - " window.PyViz.plot_index[id] = null;\n", - " }\n", - " } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", - " var bk_div = document.createElement(\"div\");\n", - " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", - " var script_attrs = bk_div.children[0].attributes;\n", - " for (var i = 0; i < script_attrs.length; i++) {\n", - " toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n", - " }\n", - " // store reference to server id on output_area\n", - " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", - " }\n", - "}\n", - "\n", - "/**\n", - " * Handle when an output is cleared or removed\n", - " */\n", - "function handle_clear_output(event, handle) {\n", - " var id = handle.cell.output_area._hv_plot_id;\n", - " var server_id = handle.cell.output_area._bokeh_server_id;\n", - " if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n", - " var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n", - " if (server_id !== null) {\n", - " comm.send({event_type: 'server_delete', 'id': server_id});\n", - " return;\n", - " } else if (comm !== null) {\n", - " comm.send({event_type: 'delete', 'id': id});\n", - " }\n", - " delete PyViz.plot_index[id];\n", - " if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n", - " var doc = window.Bokeh.index[id].model.document\n", - " doc.clear();\n", - " const i = window.Bokeh.documents.indexOf(doc);\n", - " if (i > -1) {\n", - " window.Bokeh.documents.splice(i, 1);\n", - " }\n", - " }\n", - "}\n", - "\n", - "/**\n", - " * Handle kernel restart event\n", - " */\n", - "function handle_kernel_cleanup(event, handle) {\n", - " delete PyViz.comms[\"hv-extension-comm\"];\n", - " window.PyViz.plot_index = {}\n", - "}\n", - "\n", - "/**\n", - " * Handle update_display_data messages\n", - " */\n", - "function handle_update_output(event, handle) {\n", - " handle_clear_output(event, {cell: {output_area: handle.output_area}})\n", - " handle_add_output(event, handle)\n", - "}\n", - "\n", - "function register_renderer(events, OutputArea) {\n", - " function append_mime(data, metadata, element) {\n", - " // create a DOM node to render to\n", - " var toinsert = this.create_output_subarea(\n", - " metadata,\n", - " CLASS_NAME,\n", - " EXEC_MIME_TYPE\n", - " );\n", - " this.keyboard_manager.register_events(toinsert);\n", - " // Render to node\n", - " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", - " render(props, toinsert[0]);\n", - " element.append(toinsert);\n", - " return toinsert\n", - " }\n", - "\n", - " events.on('output_added.OutputArea', handle_add_output);\n", - " events.on('output_updated.OutputArea', handle_update_output);\n", - " events.on('clear_output.CodeCell', handle_clear_output);\n", - " events.on('delete.Cell', handle_clear_output);\n", - " events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n", - "\n", - " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", - " safe: true,\n", - " index: 0\n", - " });\n", - "}\n", - "\n", - "if (window.Jupyter !== undefined) {\n", - " try {\n", - " var events = require('base/js/events');\n", - " var OutputArea = require('notebook/js/outputarea').OutputArea;\n", - " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", - " register_renderer(events, OutputArea);\n", - " }\n", - " } catch(err) {\n", - " }\n", - "}\n" + "cell_type": "code", + "execution_count": 3, + "id": "8ab91cc0-b12b-410c-88af-37ce03153c34", + "metadata": {}, + "outputs": [ + { + "ename": "ValueError", + "evalue": "Collection C3564876127-NSIDC_CPRD does not have a virtual store (no VIRTUAL COLLECTION URL found in its RelatedUrls).", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mValueError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m collection = ea.search_datasets(short_name=\u001b[33m\"\u001b[39m\u001b[33mATL06\u001b[39m\u001b[33m\"\u001b[39m)[\u001b[32m0\u001b[39m]\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m vds = \u001b[43mea\u001b[49m\u001b[43m.\u001b[49m\u001b[43mopen_virtual\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcollection\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 4\u001b[39m vds\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/github/betolink/earthaccess/earthaccess/virtual/core.py:700\u001b[39m, in \u001b[36mopen_virtual\u001b[39m\u001b[34m(uri, access, storage_options, force_external, load, **kwargs)\u001b[39m\n\u001b[32m 698\u001b[39m url = uri.virtual_collection_url()\n\u001b[32m 699\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m url \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m700\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m 701\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mCollection \u001b[39m\u001b[38;5;132;01m{\u001b[39;00muri.get(\u001b[33m'\u001b[39m\u001b[33mmeta\u001b[39m\u001b[33m'\u001b[39m,\u001b[38;5;250m \u001b[39m{}).get(\u001b[33m'\u001b[39m\u001b[33mconcept-id\u001b[39m\u001b[33m'\u001b[39m,\u001b[38;5;250m \u001b[39m\u001b[33m'\u001b[39m\u001b[33m'\u001b[39m)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 702\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mdoes not have a virtual store (no VIRTUAL COLLECTION \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 703\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mURL found in its RelatedUrls).\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 704\u001b[39m )\n\u001b[32m 705\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m _is_icechunk_uri(url) \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m _is_kerchunk_uri(url):\n\u001b[32m 706\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m 707\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mUnrecognised virtual store URL in collection: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00murl\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m. \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 708\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mExpected a .icechunk, .parquet, or .json file.\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 709\u001b[39m )\n", + "\u001b[31mValueError\u001b[39m: Collection C3564876127-NSIDC_CPRD does not have a virtual store (no VIRTUAL COLLECTION URL found in its RelatedUrls)." + ] + } ], - "application/vnd.holoviews_load.v0+json": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n function JupyterCommManager() {\n }\n\n JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n comm_manager.register_target(comm_id, function(comm) {\n comm.on_msg(msg_handler);\n });\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n comm.onMsg = msg_handler;\n });\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data, comm_id};\n var buffers = []\n for (var buffer of message.buffers || []) {\n buffers.push(new DataView(buffer))\n }\n var metadata = message.metadata || {};\n var msg = {content, buffers, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n })\n }\n }\n\n JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n if (comm_id in window.PyViz.comms) {\n return window.PyViz.comms[comm_id];\n } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n if (msg_handler) {\n comm.on_msg(msg_handler);\n }\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n let retries = 0;\n const open = () => {\n if (comm.active) {\n comm.open();\n } else if (retries > 3) {\n console.warn('Comm target never activated')\n } else {\n retries += 1\n setTimeout(open, 500)\n }\n }\n if (comm.active) {\n comm.open();\n } else {\n setTimeout(open, 500)\n }\n if (msg_handler) {\n comm.onMsg = msg_handler;\n }\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n var comm_promise = google.colab.kernel.comms.open(comm_id)\n comm_promise.then((comm) => {\n window.PyViz.comms[comm_id] = comm;\n if (msg_handler) {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data};\n var metadata = message.metadata || {comm_id};\n var msg = {content, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n }\n })\n var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n return comm_promise.then((comm) => {\n comm.send(data, metadata, buffers, disposeOnDone);\n });\n };\n var comm = {\n send: sendClosure\n };\n }\n window.PyViz.comms[comm_id] = comm;\n return comm;\n }\n window.PyViz.comm_manager = new JupyterCommManager();\n \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n var div = document.createElement(\"div\");\n var script = document.createElement(\"script\");\n node.appendChild(div);\n node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n var output_area = handle.output_area;\n var output = handle.output;\n if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n return\n }\n var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n if (id !== undefined) {\n var nchildren = toinsert.length;\n var html_node = toinsert[nchildren-1].children[0];\n html_node.innerHTML = output.data[HTML_MIME_TYPE];\n var scripts = [];\n var nodelist = html_node.querySelectorAll(\"script\");\n for (var i in nodelist) {\n if (nodelist.hasOwnProperty(i)) {\n scripts.push(nodelist[i])\n }\n }\n\n scripts.forEach( function (oldScript) {\n var newScript = document.createElement(\"script\");\n var attrs = [];\n var nodemap = oldScript.attributes;\n for (var j in nodemap) {\n if (nodemap.hasOwnProperty(j)) {\n attrs.push(nodemap[j])\n }\n }\n attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n oldScript.parentNode.replaceChild(newScript, oldScript);\n });\n if (JS_MIME_TYPE in output.data) {\n toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n }\n output_area._hv_plot_id = id;\n if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n window.PyViz.plot_index[id] = Bokeh.index[id];\n } else {\n window.PyViz.plot_index[id] = null;\n }\n } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n var bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n var script_attrs = bk_div.children[0].attributes;\n for (var i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n var id = handle.cell.output_area._hv_plot_id;\n var server_id = handle.cell.output_area._bokeh_server_id;\n if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n if (server_id !== null) {\n comm.send({event_type: 'server_delete', 'id': server_id});\n return;\n } else if (comm !== null) {\n comm.send({event_type: 'delete', 'id': id});\n }\n delete PyViz.plot_index[id];\n if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n var doc = window.Bokeh.index[id].model.document\n doc.clear();\n const i = window.Bokeh.documents.indexOf(doc);\n if (i > -1) {\n window.Bokeh.documents.splice(i, 1);\n }\n }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n delete PyViz.comms[\"hv-extension-comm\"];\n window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n handle_clear_output(event, {cell: {output_area: handle.output_area}})\n handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n var toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[0]);\n element.append(toinsert);\n return toinsert\n }\n\n events.on('output_added.OutputArea', handle_add_output);\n events.on('output_updated.OutputArea', handle_update_output);\n events.on('clear_output.CodeCell', handle_clear_output);\n events.on('delete.Cell', handle_clear_output);\n events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n safe: true,\n index: 0\n });\n}\n\nif (window.Jupyter !== undefined) {\n try {\n var events = require('base/js/events');\n var OutputArea = require('notebook/js/outputarea').OutputArea;\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n } catch(err) {\n }\n}\n" - }, - "metadata": {}, - "output_type": "display_data" + "source": [ + "collection = ea.search_datasets(short_name=\"ATL06\")[0]\n", + "\n", + "vds = ea.open_virtual(collection)\n", + "vds" + ] }, { - "data": { - "application/vnd.holoviews_exec.v0+json": "", - "text/html": [ - "
\n", - "
\n", - "
\n", - "" + "cell_type": "markdown", + "id": "54bf4cd2-3650-49f7-bdd8-24ba11da80f7", + "metadata": {}, + "source": [ + "#### `load = False`\n", + "\n", + "When we do not load indexes into our VDS variables, we cannot access byte ranges using slicing, this prevents us from normal xarray operations but it also allows us to treat the dataset as appendable, we can search for new granules, virtualize them with `earthaccess.virtualize()` and concatenate both virtual datasets to persist the updated cube! \n", + "\n", + "Something not intuitive is that because of embeding values in virtual arrays, is better to reconstruct the coordinates than interpreting native byte types and this roundtrip takes more time than just loading the indexes themselves. Thus, not loading takes longer specially if the original virtual store uses JSON serialization. `earthaccess` will cache it but a reference file that's 100+ MB still needs to be fetched and parsed. " ] - }, - "metadata": { - "application/vnd.holoviews_exec.v0+json": { - "id": "383dc158-b95c-4b78-923f-2b37baa80300" - } - }, - "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 18.7 s, sys: 3.57 s, total: 22.2 s\n", - "Wall time: 26.4 s\n" - ] + "cell_type": "code", + "execution_count": 4, + "id": "3b9e877f-f910-4c0d-901d-6a93f6be5237", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 56.1 s, sys: 5.72 s, total: 1min 1s\n", + "Wall time: 1min 5s\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 15TB\n",
+              "Dimensions:           (time: 3868, lat: 17999, lon: 36000)\n",
+              "Coordinates:\n",
+              "  * time              (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n",
+              "  * lat               (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n",
+              "  * lon               (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n",
+              "Data variables:\n",
+              "    analysed_sst      (time, lat, lon) int16 5TB ManifestArray<shape=(3868, 1...\n",
+              "    analysis_error    (time, lat, lon) int16 5TB ManifestArray<shape=(3868, 1...\n",
+              "    mask              (time, lat, lon) int8 3TB ManifestArray<shape=(3868, 17...\n",
+              "    sea_ice_fraction  (time, lat, lon) int8 3TB ManifestArray<shape=(3868, 17...\n",
+              "Attributes: (12/47)\n",
+              "    Conventions:                CF-1.5\n",
+              "    Metadata_Conventions:       Unidata Observation Dataset v1.0\n",
+              "    acknowledgment:             Please acknowledge the use of these data with...\n",
+              "    cdm_data_type:              grid\n",
+              "    comment:                    MUR = "Multi-scale Ultra-high Reolution"\n",
+              "    creator_email:              ghrsst@podaac.jpl.nasa.gov\n",
+              "    ...                         ...\n",
+              "    summary:                    A merged, multi-sensor L4 Foundation SST anal...\n",
+              "    time_coverage_end:          20020601T210000Z\n",
+              "    time_coverage_start:        20020531T210000Z\n",
+              "    title:                      Daily MUR SST, Final product\n",
+              "    uuid:                       27665bc0-d5fc-11e1-9b23-0800200c9a66\n",
+              "    westernmost_longitude:      -180.0
" + ], + "text/plain": [ + " Size: 15TB\n", + "Dimensions: (time: 3868, lat: 17999, lon: 36000)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n", + " * lat (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n", + " * lon (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n", + "Data variables:\n", + " analysed_sst (time, lat, lon) int16 5TB ManifestArray\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset> Size: 70TB\n",
-       "Dimensions:           (time: 3868, lat: 17999, lon: 36000)\n",
-       "Coordinates:\n",
-       "  * time              (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n",
-       "  * lat               (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n",
-       "  * lon               (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n",
-       "Data variables:\n",
-       "    analysed_sst      (time, lat, lon) float64 20TB ...\n",
-       "    analysis_error    (time, lat, lon) float64 20TB ...\n",
-       "    mask              (time, lat, lon) float32 10TB ...\n",
-       "    sea_ice_fraction  (time, lat, lon) float64 20TB ...\n",
-       "Attributes: (12/47)\n",
-       "    Conventions:                CF-1.5\n",
-       "    Metadata_Conventions:       Unidata Observation Dataset v1.0\n",
-       "    acknowledgment:             Please acknowledge the use of these data with...\n",
-       "    cdm_data_type:              grid\n",
-       "    comment:                    MUR = "Multi-scale Ultra-high Reolution"\n",
-       "    creator_email:              ghrsst@podaac.jpl.nasa.gov\n",
-       "    ...                         ...\n",
-       "    summary:                    A merged, multi-sensor L4 Foundation SST anal...\n",
-       "    time_coverage_end:          20020601T210000Z\n",
-       "    time_coverage_start:        20020531T210000Z\n",
-       "    title:                      Daily MUR SST, Final product\n",
-       "    uuid:                       27665bc0-d5fc-11e1-9b23-0800200c9a66\n",
-       "    westernmost_longitude:      -180.0
" + "cell_type": "code", + "execution_count": 5, + "id": "4e675eb5-9d7d-47c0-ab42-1a72e71fcfe9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dataset spaning from 2002-06-01T09:00:00.000000000 to 2013-01-01T09:00:00.000000000\n" + ] + } ], - "text/plain": [ - " Size: 70TB\n", - "Dimensions: (time: 3868, lat: 17999, lon: 36000)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n", - " * lat (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n", - " * lon (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n", - "Data variables:\n", - " analysed_sst (time, lat, lon) float64 20TB ...\n", - " analysis_error (time, lat, lon) float64 20TB ...\n", - " mask (time, lat, lon) float32 10TB ...\n", - " sea_ice_fraction (time, lat, lon) float64 20TB ...\n", - "Attributes: (12/47)\n", - " Conventions: CF-1.5\n", - " Metadata_Conventions: Unidata Observation Dataset v1.0\n", - " acknowledgment: Please acknowledge the use of these data with...\n", - " cdm_data_type: grid\n", - " comment: MUR = \"Multi-scale Ultra-high Reolution\"\n", - " creator_email: ghrsst@podaac.jpl.nasa.gov\n", - " ... ...\n", - " summary: A merged, multi-sensor L4 Foundation SST anal...\n", - " time_coverage_end: 20020601T210000Z\n", - " time_coverage_start: 20020531T210000Z\n", - " title: Daily MUR SST, Final product\n", - " uuid: 27665bc0-d5fc-11e1-9b23-0800200c9a66\n", - " westernmost_longitude: -180.0" + "source": [ + "# see the overall time coverage:\n", + "print(\n", + " f\"Dataset spaning from {baseline_vds.time.min().values} to {baseline_vds.time.max().values}\"\n", + ")" ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "\n", - "import warnings\n", - "\n", - "import earthaccess as ea\n", - "import xarray as xr\n", - "\n", - "warnings.filterwarnings(\"ignore\")\n", - "\n", - "auth = ea.login()\n", - "\n", - "collection = ea.search_datasets(concept_id=\"C1996881146-POCLOUD\")[\n", - " 0\n", - "] # <- note that we are using the first item\n", - "\n", - "vds = ea.open_virtual(collection, load=True, force_external=True)\n", - "vds" - ] - }, - { - "cell_type": "markdown", - "id": "08b7fcea-bd3b-4d4d-bd86-ee2e190c79ba", - "metadata": {}, - "source": [ - "### open_virtual()\n", - "\n", - "`open_virtual()` can open virtual collections listed at the collection level metadata, fow now following the convention adopted by JPL of using the following URL in `RelatedUrls` \n", - "\n", - "```json\n", - "{\n", - " \"Description\": \"Virtual data set reference \",\n", - " \"URLContentType\": \"DistributionURL\",\n", - " \"URL\": \"https://archive.podaac.earthdata.nasa.gov/../MUR-JPL-L4-GLOB-v4.1_combined-ref.json\",\n", - " \"Type\": \"GET DATA\",\n", - " \"Subtype\": \"VIRTUAL COLLECTION\"\n", - "}\n", - "```\n", - "\n", - "> Note: We may have multiple virtual stores for any given collection so we anticipate this convention may change in the near future, for now the assumption is that each collection has at most one virtual representation. \n", - "\n", - "`open_virtual()` has the following signature that we explain in short:\n", - "\n", - "* **uri**: this can be a collection level result from `collection = earthaccess.search_datasets()[0]` a local path, or a publicly accessible reference URL. If we use a collection from earthaccess, we don't need to deal with the cumbersome auth scafolding for the underlaying libraries; in all cases we'll get an xarray dataset instance pointing to our virtual dataset.This can be kerchunk or Icechunk stores \n", - "* **access**: this one informs earthaccess if the virtual store points to S3 buckets or HTTP, without inspecting the references we need to be explicit for now. values are `direct` or `external` (same as `indirect`)\n", - "* **storage_options**: if we need custom authentication for the virtual store this works the same way xarray uses storage options, not needed for NASA data as it's automatic.\n", - "* **force_external**: This is a handy flag to force reference translation, this is if the only available store points to S3 but we are accessing the data from outside AWS `us-west-2` setting this to `True` will translate the URLs on the fly for out of region access.\n", - "* **load**: if we should load the indexes in our dataset, if we do we can address the dataset as any other xarray dataset and perform operations like subsetting and aggregations, if not we get a `virtualizarr` dataset backed by `ManifestArray` instances, this is only useful if we plan to add more virtual references and update the cube.\n", - "\n", - "The method signature is as follows:\n", - "\n", - "```python\n", - "ea.open_virtual(\n", - " uri: 'str | Path | earthaccess.DataCollection',\n", - " *,\n", - " access: 'str' = 'indirect',\n", - " storage_options: 'dict[str, Any] | None' = None,\n", - " force_external: 'bool' = False,\n", - " load: 'bool' = True,\n", - " **kwargs: 'Any',\n", - ") -> 'xr.Dataset'\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "0e2366e8-aab1-49ce-a66a-bfc019df2a3b", - "metadata": {}, - "source": [ - "If we try to open a virtual dataset on a collection that has no virtual distribution will result in an exception." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "8ab91cc0-b12b-410c-88af-37ce03153c34", - "metadata": {}, - "outputs": [ + }, { - "ename": "ValueError", - "evalue": "Collection C3564876127-NSIDC_CPRD does not have a virtual store (no VIRTUAL COLLECTION URL found in its RelatedUrls).", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mValueError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m collection = ea.search_datasets(short_name=\u001b[33m\"\u001b[39m\u001b[33mATL06\u001b[39m\u001b[33m\"\u001b[39m)[\u001b[32m0\u001b[39m]\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m vds = \u001b[43mea\u001b[49m\u001b[43m.\u001b[49m\u001b[43mopen_virtual\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcollection\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 4\u001b[39m vds\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/github/betolink/earthaccess/earthaccess/virtual/core.py:700\u001b[39m, in \u001b[36mopen_virtual\u001b[39m\u001b[34m(uri, access, storage_options, force_external, load, **kwargs)\u001b[39m\n\u001b[32m 698\u001b[39m url = uri.virtual_collection_url()\n\u001b[32m 699\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m url \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m700\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m 701\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mCollection \u001b[39m\u001b[38;5;132;01m{\u001b[39;00muri.get(\u001b[33m'\u001b[39m\u001b[33mmeta\u001b[39m\u001b[33m'\u001b[39m,\u001b[38;5;250m \u001b[39m{}).get(\u001b[33m'\u001b[39m\u001b[33mconcept-id\u001b[39m\u001b[33m'\u001b[39m,\u001b[38;5;250m \u001b[39m\u001b[33m'\u001b[39m\u001b[33m'\u001b[39m)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 702\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mdoes not have a virtual store (no VIRTUAL COLLECTION \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 703\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mURL found in its RelatedUrls).\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 704\u001b[39m )\n\u001b[32m 705\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m _is_icechunk_uri(url) \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m _is_kerchunk_uri(url):\n\u001b[32m 706\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m 707\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mUnrecognised virtual store URL in collection: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00murl\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m. \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 708\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mExpected a .icechunk, .parquet, or .json file.\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 709\u001b[39m )\n", - "\u001b[31mValueError\u001b[39m: Collection C3564876127-NSIDC_CPRD does not have a virtual store (no VIRTUAL COLLECTION URL found in its RelatedUrls)." - ] - } - ], - "source": [ - "collection = ea.search_datasets(short_name=\"ATL06\")[0]\n", - "\n", - "vds = ea.open_virtual(collection)\n", - "vds" - ] - }, - { - "cell_type": "markdown", - "id": "54bf4cd2-3650-49f7-bdd8-24ba11da80f7", - "metadata": {}, - "source": [ - "#### `load = False`\n", - "\n", - "When we do not load indexes into our VDS variables, we cannot access byte ranges using slicing, this prevents us from normal xarray operations but it also allows us to treat the dataset as appendable, we can search for new granules, virtualize them with `earthaccess.virtualize()` and concatenate both virtual datasets to persist the updated cube! \n", - "\n", - "Something not intuitive is that because of embeding values in virtual arrays, is better to reconstruct the coordinates than interpreting native byte types and this roundtrip takes more time than just loading the indexes themselves. Thus, not loading takes longer specially if the original virtual store uses JSON serialization. `earthaccess` will cache it but a reference file that's 100+ MB still needs to be fetched and parsed. " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "3b9e877f-f910-4c0d-901d-6a93f6be5237", - "metadata": {}, - "outputs": [ + "cell_type": "markdown", + "id": "904adbc4-904b-44c9-b3fc-f8a263f06774", + "metadata": {}, + "source": [ + "> Note how the variables when we use `load=False` are backed by `ManifestArray` instances, they do not load indexes.\n", + "\n", + "Now we can search for new granules with `search_data()` using the last date from our virtual cube so we get granules in the following time steps. " + ] + }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 56.1 s, sys: 5.72 s, total: 1min 1s\n", - "Wall time: 1min 5s\n" - ] + "cell_type": "code", + "execution_count": 6, + "id": "e263f631-358d-4131-b147-f1ab3ae6987e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "31" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sst_granules = ea.search_data(\n", + " concept_id=\"C1996881146-POCLOUD\", temporal=(\"2013-01-02\", \"2013-01-31\")\n", + ")\n", + "len(sst_granules)" + ] }, { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset> Size: 15TB\n",
-       "Dimensions:           (time: 3868, lat: 17999, lon: 36000)\n",
-       "Coordinates:\n",
-       "  * time              (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n",
-       "  * lat               (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n",
-       "  * lon               (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n",
-       "Data variables:\n",
-       "    analysed_sst      (time, lat, lon) int16 5TB ManifestArray<shape=(3868, 1...\n",
-       "    analysis_error    (time, lat, lon) int16 5TB ManifestArray<shape=(3868, 1...\n",
-       "    mask              (time, lat, lon) int8 3TB ManifestArray<shape=(3868, 17...\n",
-       "    sea_ice_fraction  (time, lat, lon) int8 3TB ManifestArray<shape=(3868, 17...\n",
-       "Attributes: (12/47)\n",
-       "    Conventions:                CF-1.5\n",
-       "    Metadata_Conventions:       Unidata Observation Dataset v1.0\n",
-       "    acknowledgment:             Please acknowledge the use of these data with...\n",
-       "    cdm_data_type:              grid\n",
-       "    comment:                    MUR = "Multi-scale Ultra-high Reolution"\n",
-       "    creator_email:              ghrsst@podaac.jpl.nasa.gov\n",
-       "    ...                         ...\n",
-       "    summary:                    A merged, multi-sensor L4 Foundation SST anal...\n",
-       "    time_coverage_end:          20020601T210000Z\n",
-       "    time_coverage_start:        20020531T210000Z\n",
-       "    title:                      Daily MUR SST, Final product\n",
-       "    uuid:                       27665bc0-d5fc-11e1-9b23-0800200c9a66\n",
-       "    westernmost_longitude:      -180.0
" + "cell_type": "markdown", + "id": "e3a8acf1-7b63-46e5-941e-f2a2b638763d", + "metadata": {}, + "source": [ + "### Virtualizing with `virtualize()`\n", + "\n", + "`earthaccess` can virtualize data cubes from granules, this is meant for data distributors (NASA) or power users that want to try this beta feature for faster data access. Is very important to note that virtualize won't work for most data as they need to be:\n", + "\n", + "* *Homogeneous*: The underlying files must share the exact same variables, data types, and internal schema across all granules. If one file is missing a variable, or if data types differ (e.g., float32 in one file and float64 in another), the virtual concatenation will fail.\n", + "* *CF compliant*: The files' metadata must strictly follow Climate and Forecast (CF) conventions. This is required so the analysis engine (like Xarray) can automatically decode time arrays, correctly identify coordinate variables, and handle scale factors or fill values without manual intervention.\n", + "* *Unified grid*: The granules must share a consistent spatial resolution and coordinate reference system. They need to stitch together seamlessly (usually along a time dimension) without any spatial overlapping, gaps, or need for regridding and interpolation.\n", + "\n", + "For NASA Earth datasets this falls into the level 4 processing datasets, they have global coverage and are distributed in NetCDF or HDF5 format with these characteristics. This is not a requirement but many metadata pitfalls will be encountered if we want to virtualize data from lower processing levels." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "f78c350c-e10f-435f-8675-3674b631f150", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 121GB\n",
+              "Dimensions:           (time: 31, lat: 17999, lon: 36000)\n",
+              "Coordinates:\n",
+              "  * time              (time) datetime64[ns] 248B 2013-01-02T09:00:00 ... 2013...\n",
+              "  * lat               (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n",
+              "  * lon               (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n",
+              "Data variables:\n",
+              "    mask              (time, lat, lon) int8 20GB ManifestArray<shape=(31, 179...\n",
+              "    sea_ice_fraction  (time, lat, lon) int8 20GB ManifestArray<shape=(31, 179...\n",
+              "    analysed_sst      (time, lat, lon) int16 40GB ManifestArray<shape=(31, 17...\n",
+              "    analysis_error    (time, lat, lon) int16 40GB ManifestArray<shape=(31, 17...\n",
+              "Attributes: (12/42)\n",
+              "    Conventions:                CF-1.5\n",
+              "    title:                      Daily MUR SST, Final product\n",
+              "    summary:                    A merged, multi-sensor L4 Foundation SST anal...\n",
+              "    references:                 http://podaac.jpl.nasa.gov/Multi-scale_Ultra-...\n",
+              "    institution:                Jet Propulsion Laboratory\n",
+              "    history:                    created at nominal 4-day latency; replaced nr...\n",
+              "    ...                         ...\n",
+              "    project:                    NASA Making Earth Science Data Records for Us...\n",
+              "    publisher_name:             GHRSST Project Office\n",
+              "    publisher_url:              http://www.ghrsst.org\n",
+              "    publisher_email:            ghrsst-po@nceo.ac.uk\n",
+              "    processing_level:           L4\n",
+              "    cdm_data_type:              grid
" + ], + "text/plain": [ + " Size: 121GB\n", + "Dimensions: (time: 31, lat: 17999, lon: 36000)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 248B 2013-01-02T09:00:00 ... 2013...\n", + " * lat (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n", + " * lon (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n", + "Data variables:\n", + " mask (time, lat, lon) int8 20GB ManifestArray Size: 15TB\n", - "Dimensions: (time: 3868, lat: 17999, lon: 36000)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n", - " * lat (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n", - " * lon (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n", - "Data variables:\n", - " analysed_sst (time, lat, lon) int16 5TB ManifestArray Note how the variables when we use `load=False` are backed by `ManifestArray` instances, they do not load indexes.\n", - "\n", - "Now we can search for new granules with `search_data()` using the last date from our virtual cube so we get granules in the following time steps. " - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "e263f631-358d-4131-b147-f1ab3ae6987e", - "metadata": {}, - "outputs": [ + "cell_type": "markdown", + "id": "e36ed1b0-df22-45cc-8ece-4bbbef8d552b", + "metadata": {}, + "source": [ + "Homogenize codec compression level\n", + "\n", + "> NOTE: Do not use this in production, this is to show the workflow of what it would be to concatenate homogeneous datasets" + ] + }, { - "data": { - "text/plain": [ - "31" + "cell_type": "code", + "execution_count": 8, + "id": "161ea6a2-7b26-42bf-a695-3a45b8ae7053", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 121GB\n",
+              "Dimensions:           (time: 31, lat: 17999, lon: 36000)\n",
+              "Coordinates:\n",
+              "  * time              (time) datetime64[ns] 248B 2013-01-02T09:00:00 ... 2013...\n",
+              "  * lat               (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n",
+              "  * lon               (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n",
+              "Data variables:\n",
+              "    mask              (time, lat, lon) int8 20GB ManifestArray<shape=(31, 179...\n",
+              "    sea_ice_fraction  (time, lat, lon) int8 20GB ManifestArray<shape=(31, 179...\n",
+              "    analysed_sst      (time, lat, lon) int16 40GB ManifestArray<shape=(31, 17...\n",
+              "    analysis_error    (time, lat, lon) int16 40GB ManifestArray<shape=(31, 17...\n",
+              "Attributes: (12/42)\n",
+              "    Conventions:                CF-1.5\n",
+              "    title:                      Daily MUR SST, Final product\n",
+              "    summary:                    A merged, multi-sensor L4 Foundation SST anal...\n",
+              "    references:                 http://podaac.jpl.nasa.gov/Multi-scale_Ultra-...\n",
+              "    institution:                Jet Propulsion Laboratory\n",
+              "    history:                    created at nominal 4-day latency; replaced nr...\n",
+              "    ...                         ...\n",
+              "    project:                    NASA Making Earth Science Data Records for Us...\n",
+              "    publisher_name:             GHRSST Project Office\n",
+              "    publisher_url:              http://www.ghrsst.org\n",
+              "    publisher_email:            ghrsst-po@nceo.ac.uk\n",
+              "    processing_level:           L4\n",
+              "    cdm_data_type:              grid
" + ], + "text/plain": [ + " Size: 121GB\n", + "Dimensions: (time: 31, lat: 17999, lon: 36000)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 248B 2013-01-02T09:00:00 ... 2013...\n", + " * lat (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n", + " * lon (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n", + "Data variables:\n", + " mask (time, lat, lon) int8 20GB ManifestArray\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset> Size: 121GB\n",
-       "Dimensions:           (time: 31, lat: 17999, lon: 36000)\n",
-       "Coordinates:\n",
-       "  * time              (time) datetime64[ns] 248B 2013-01-02T09:00:00 ... 2013...\n",
-       "  * lat               (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n",
-       "  * lon               (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n",
-       "Data variables:\n",
-       "    mask              (time, lat, lon) int8 20GB ManifestArray<shape=(31, 179...\n",
-       "    sea_ice_fraction  (time, lat, lon) int8 20GB ManifestArray<shape=(31, 179...\n",
-       "    analysed_sst      (time, lat, lon) int16 40GB ManifestArray<shape=(31, 17...\n",
-       "    analysis_error    (time, lat, lon) int16 40GB ManifestArray<shape=(31, 17...\n",
-       "Attributes: (12/42)\n",
-       "    Conventions:                CF-1.5\n",
-       "    title:                      Daily MUR SST, Final product\n",
-       "    summary:                    A merged, multi-sensor L4 Foundation SST anal...\n",
-       "    references:                 http://podaac.jpl.nasa.gov/Multi-scale_Ultra-...\n",
-       "    institution:                Jet Propulsion Laboratory\n",
-       "    history:                    created at nominal 4-day latency; replaced nr...\n",
-       "    ...                         ...\n",
-       "    project:                    NASA Making Earth Science Data Records for Us...\n",
-       "    publisher_name:             GHRSST Project Office\n",
-       "    publisher_url:              http://www.ghrsst.org\n",
-       "    publisher_email:            ghrsst-po@nceo.ac.uk\n",
-       "    processing_level:           L4\n",
-       "    cdm_data_type:              grid
" + "cell_type": "markdown", + "id": "e119a373-1d85-4226-833f-3cd07137ca4f", + "metadata": {}, + "source": [ + "### Concatenating and updating virtual store" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "81960b71-a62f-46dc-a28f-39d2f08b22ca", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 15TB\n",
+              "Dimensions:           (time: 3899, lat: 17999, lon: 36000)\n",
+              "Coordinates:\n",
+              "  * time              (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n",
+              "  * lat               (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n",
+              "  * lon               (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n",
+              "Data variables:\n",
+              "    analysed_sst      (time, lat, lon) int16 5TB ManifestArray<shape=(3899, 1...\n",
+              "    analysis_error    (time, lat, lon) int16 5TB ManifestArray<shape=(3899, 1...\n",
+              "    mask              (time, lat, lon) int8 3TB ManifestArray<shape=(3899, 17...\n",
+              "    sea_ice_fraction  (time, lat, lon) int8 3TB ManifestArray<shape=(3899, 17...\n",
+              "Attributes: (12/47)\n",
+              "    Conventions:                CF-1.5\n",
+              "    Metadata_Conventions:       Unidata Observation Dataset v1.0\n",
+              "    acknowledgment:             Please acknowledge the use of these data with...\n",
+              "    cdm_data_type:              grid\n",
+              "    comment:                    MUR = "Multi-scale Ultra-high Reolution"\n",
+              "    creator_email:              ghrsst@podaac.jpl.nasa.gov\n",
+              "    ...                         ...\n",
+              "    summary:                    A merged, multi-sensor L4 Foundation SST anal...\n",
+              "    time_coverage_end:          20020601T210000Z\n",
+              "    time_coverage_start:        20020531T210000Z\n",
+              "    title:                      Daily MUR SST, Final product\n",
+              "    uuid:                       27665bc0-d5fc-11e1-9b23-0800200c9a66\n",
+              "    westernmost_longitude:      -180.0
" + ], + "text/plain": [ + " Size: 15TB\n", + "Dimensions: (time: 3899, lat: 17999, lon: 36000)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n", + " * lat (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n", + " * lon (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n", + "Data variables:\n", + " analysed_sst (time, lat, lon) int16 5TB ManifestArray Size: 121GB\n", - "Dimensions: (time: 31, lat: 17999, lon: 36000)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 248B 2013-01-02T09:00:00 ... 2013...\n", - " * lat (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n", - " * lon (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n", - "Data variables:\n", - " mask (time, lat, lon) int8 20GB ManifestArray NOTE: Do not use this in production, this is to show the workflow of what it would be to concatenate homogeneous datasets" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "161ea6a2-7b26-42bf-a695-3a45b8ae7053", - "metadata": {}, - "outputs": [ + }, { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset> Size: 121GB\n",
-       "Dimensions:           (time: 31, lat: 17999, lon: 36000)\n",
-       "Coordinates:\n",
-       "  * time              (time) datetime64[ns] 248B 2013-01-02T09:00:00 ... 2013...\n",
-       "  * lat               (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n",
-       "  * lon               (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n",
-       "Data variables:\n",
-       "    mask              (time, lat, lon) int8 20GB ManifestArray<shape=(31, 179...\n",
-       "    sea_ice_fraction  (time, lat, lon) int8 20GB ManifestArray<shape=(31, 179...\n",
-       "    analysed_sst      (time, lat, lon) int16 40GB ManifestArray<shape=(31, 17...\n",
-       "    analysis_error    (time, lat, lon) int16 40GB ManifestArray<shape=(31, 17...\n",
-       "Attributes: (12/42)\n",
-       "    Conventions:                CF-1.5\n",
-       "    title:                      Daily MUR SST, Final product\n",
-       "    summary:                    A merged, multi-sensor L4 Foundation SST anal...\n",
-       "    references:                 http://podaac.jpl.nasa.gov/Multi-scale_Ultra-...\n",
-       "    institution:                Jet Propulsion Laboratory\n",
-       "    history:                    created at nominal 4-day latency; replaced nr...\n",
-       "    ...                         ...\n",
-       "    project:                    NASA Making Earth Science Data Records for Us...\n",
-       "    publisher_name:             GHRSST Project Office\n",
-       "    publisher_url:              http://www.ghrsst.org\n",
-       "    publisher_email:            ghrsst-po@nceo.ac.uk\n",
-       "    processing_level:           L4\n",
-       "    cdm_data_type:              grid
" + "cell_type": "code", + "execution_count": 10, + "id": "509cb031-38f7-48f7-9df7-5024114f16b4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dataset now spaning from 2002-06-01T09:00:00.000000000 to 2013-02-01T09:00:00.000000000\n" + ] + } ], - "text/plain": [ - " Size: 121GB\n", - "Dimensions: (time: 31, lat: 17999, lon: 36000)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 248B 2013-01-02T09:00:00 ... 2013...\n", - " * lat (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n", - " * lon (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n", - "Data variables:\n", - " mask (time, lat, lon) int8 20GB ManifestArray\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset> Size: 15TB\n",
-       "Dimensions:           (time: 3899, lat: 17999, lon: 36000)\n",
-       "Coordinates:\n",
-       "  * time              (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n",
-       "  * lat               (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n",
-       "  * lon               (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n",
-       "Data variables:\n",
-       "    analysed_sst      (time, lat, lon) int16 5TB ManifestArray<shape=(3899, 1...\n",
-       "    analysis_error    (time, lat, lon) int16 5TB ManifestArray<shape=(3899, 1...\n",
-       "    mask              (time, lat, lon) int8 3TB ManifestArray<shape=(3899, 17...\n",
-       "    sea_ice_fraction  (time, lat, lon) int8 3TB ManifestArray<shape=(3899, 17...\n",
-       "Attributes: (12/47)\n",
-       "    Conventions:                CF-1.5\n",
-       "    Metadata_Conventions:       Unidata Observation Dataset v1.0\n",
-       "    acknowledgment:             Please acknowledge the use of these data with...\n",
-       "    cdm_data_type:              grid\n",
-       "    comment:                    MUR = "Multi-scale Ultra-high Reolution"\n",
-       "    creator_email:              ghrsst@podaac.jpl.nasa.gov\n",
-       "    ...                         ...\n",
-       "    summary:                    A merged, multi-sensor L4 Foundation SST anal...\n",
-       "    time_coverage_end:          20020601T210000Z\n",
-       "    time_coverage_start:        20020531T210000Z\n",
-       "    title:                      Daily MUR SST, Final product\n",
-       "    uuid:                       27665bc0-d5fc-11e1-9b23-0800200c9a66\n",
-       "    westernmost_longitude:      -180.0
" + "cell_type": "code", + "execution_count": 14, + "id": "46fcf6f7-e1f2-4c62-9101-ba10ae576eec", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(15158470063188, 15037948758940)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } ], - "text/plain": [ - " Size: 15TB\n", - "Dimensions: (time: 3899, lat: 17999, lon: 36000)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 31kB 2002-06-01T09:00:00 ... 2013...\n", - " * lat (lat) float32 72kB -89.99 -89.98 -89.97 ... 89.98 89.99\n", - " * lon (lon) float32 144kB -180.0 -180.0 -180.0 ... 180.0 180.0\n", - "Data variables:\n", - " analysed_sst (time, lat, lon) int16 5TB ManifestArray