diff --git a/.zenodo.json b/.zenodo.json index 648769a0..cd0bd9a9 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -35,7 +35,7 @@ "orcid": "0000-0002-8191-4765" }, { - "affiliation":"University of Colorado at Boulder", + "affiliation":"University of Colorado at Boulder, SW TREC", "name": "Navarro, Luis", "orcid": "0000-0002-6362-6575" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index d67f4a40..86befe52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [0.1.0] - 2025-XX-XX * New Instruments * Mars Global Surveyor Magnetometer (MGS Mag) + * TIMED TIDI * Documentation * Updated controlled information review statement for clarity diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index eddccc00..da07e0e4 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -267,3 +267,11 @@ TIMED SEE .. automodule:: pysatNASA.instruments.timed_see :members: + +.. _timed_tidi: + +TIMED TIDI +--------- + +.. automodule:: pysatNASA.instruments.timed_tidi + :members: diff --git a/pysatNASA/instruments/__init__.py b/pysatNASA/instruments/__init__.py index 8c6d52e3..4837d73d 100644 --- a/pysatNASA/instruments/__init__.py +++ b/pysatNASA/instruments/__init__.py @@ -23,7 +23,7 @@ 'icon_mighti', 'igs_gps', 'iss_fpmu', 'jpl_gps', 'maven_insitu_kp', 'maven_mag', 'maven_sep', 'mgs_mag', 'omni_hro', 'reach_dosimeter', 'ses14_gold', 'timed_guvi', 'timed_saber', - 'timed_see'] + 'timed_see', 'timed_tidi'] for inst in __all__: exec("from pysatNASA.instruments import {x}".format(x=inst)) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 97a0167a..cdb26141 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -18,11 +18,13 @@ """ import datetime as dt +import gzip import numpy as np import os from packaging.version import Version as pack_ver import pandas as pds import requests +import shutil import tempfile from time import sleep import xarray as xr @@ -618,7 +620,7 @@ def _get_file(remote_file, data_path, fname, temp_path=None, zip_method=None): Path to temporary directory. Must be specified if zip_method is True. (Default=None) zip_method : str - The method used to zip the file. Supports 'zip' and None. + The method used to zip the file. Supports 'zip', 'gz' and None. If None, downloads files directly. (default=None) Raises @@ -645,6 +647,11 @@ def _get_file(remote_file, data_path, fname, temp_path=None, zip_method=None): if zip_method == 'zip': with zipfile.ZipFile(dl_fname, 'r') as open_zip: open_zip.extractall(data_path) + elif zip_method == 'gz': + dest = os.path.join(data_path, fname.replace('.gz', '')) + with gzip.open(dl_fname, 'rb') as open_gz: + with open(dest, 'wb') as open_file: + shutil.copyfileobj(open_gz, open_file) elif zip_method is not None: logger.warning('{:} is not a recognized zip method'.format(zip_method)) diff --git a/pysatNASA/instruments/methods/timed.py b/pysatNASA/instruments/methods/timed.py index 12de1c87..1dc24b7c 100644 --- a/pysatNASA/instruments/methods/timed.py +++ b/pysatNASA/instruments/methods/timed.py @@ -13,7 +13,8 @@ rules_url = {'guvi': 'http://guvitimed.jhuapl.edu/home_guvi-datausage', 'saber': 'https://saber.gats-inc.com/data_services.php', - 'see': 'https://www.timed.jhuapl.edu/WWW/scripts/mdc_rules.pl'} + 'see': 'https://www.timed.jhuapl.edu/WWW/scripts/mdc_rules.pl', + 'tidi': 'https://tidi.engin.umich.edu/conditions-of-use/'} ackn_str = "".join(["This Thermosphere Ionosphere Mesosphere Energetics ", "Dynamics (TIMED) satellite data is provided through ", @@ -51,4 +52,10 @@ 'W. K., and Woodraska, D. L. (2005),', 'Solar EUV Experiment (SEE): Mission', 'overview and first results, J. Geophys.', - 'Res., 110, A01312, doi:10.1029/2004JA010765.'))} + 'Res., 110, A01312, doi:10.1029/2004JA010765.')), + 'tidi': ' '.join(('Killeen, T. L., Skinner, W. R., Johnson,', + 'R. M., Edmonson, C. J., Wu, Q., Niciejewski,', + 'R. J., Grassl, H. J., Gell, D. A., Hansen,', + 'P. E., Harvey, J. D., Kafkalidis, J. F. (1999),', + 'TIMED Doppler Interferometer, Proc. SPIE,', + '3756, 289–315.'))} diff --git a/pysatNASA/instruments/timed_tidi.py b/pysatNASA/instruments/timed_tidi.py new file mode 100644 index 00000000..627b630d --- /dev/null +++ b/pysatNASA/instruments/timed_tidi.py @@ -0,0 +1,264 @@ +# -*- coding: utf-8 -*- +"""Module for the TIMED TIDI instrument. + +Supports the TIMED Doppler Interferometer (TIDI) instrument on the Thermosphere +Ionosphere Mesosphere Energetics Dynamics (TIMED) satellite data from the +NASA Coordinated Data Analysis Web (CDAWeb). + +Properties +---------- +platform + 'timed' +name + 'tidi' +tag + ['profile','los','vector',] +inst_id + '' + 'ncar' + +Warnings +-------- +- The cleaning parameters for the instrument are still under development. + +Example +------- +:: + + import pysat + tidi = pysat.Instrument('timed', 'tidi', tag='vecetor', + inst_id='', clean_level='None') + tidi.download(dt.datetime(2020, 1, 30), dt.datetime(2020, 1, 31)) + tidi.load(2020, 2) + +:: + +""" + +import datetime as dt +import functools +import numpy as np +import xarray as xr + +from pysat.instruments.methods import general as mm_gen +from pysat.utils.io import load_netcdf + +from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import timed as mm_timed + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'timed' +name = 'tidi' +tags = {'profile': 'Level 1 TIDI data', + 'los': 'Level 2 TIDI data', + 'vector': 'Level 3 TIDI data'} +inst_ids = {'': ['los', 'profile', 'vector'], + 'ncar': ['vector']} + +pandas_format = False + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {jj: {kk: dt.datetime(2019, 1, 1) for kk in inst_ids[jj]} + for jj in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +# Use standard init routine +def init(self, module=mm_timed, name=name): + """Initialize instrument support and set parameters upon first import.""" + mm_nasa.init(self, module=module, name=name) + + # Same timing cold/warm for Michigan files. + self.strict_time_flag = False + # Set multi_file_day flag as needed + if self.tag == 'vector': + self.multi_file_day = True + + return + + +# No cleaning, use standard warning function instead +clean = mm_nasa.clean_warn + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default CDAWeb and pysat methods + +# Set the list_files routine +fname = ''.join(('TIDI_PB_{{year:04d}}{{day:03d}}_P????_S????_', + 'D{{version:03d}}_R{{revision:02d}}.{ext:s}{gz:s}')) +fname_ext = {'vector': 'VEC', + 'los': 'LOS', + 'profile': 'PRF'} +fname_ncar = ''.join(('timed_windvectorsncar_tidi_', + '{year:04d}{month:02d}{day:02d}', + '????_v??.cdf')) +supported_tags = {'': {tag: fname.format(ext=fname_ext[tag], gz='') + for tag in tags}, + 'ncar': {'vector': fname_ncar}} +list_files = functools.partial(mm_gen.list_files, + supported_tags=supported_tags,) + + +# Set the load routine +def load(fnames, tag='', inst_id=''): + """Load TIMED TIDI data into `xarray.DataSet` and `pysat.Meta` objects. + + This routine is called as needed by pysat. It is not intended + for direct user interaction. + + Parameters + ---------- + fnames : array-like + iterable of filename strings, full path, to data files to be loaded. + This input is nominally provided by pysat itself. + tag : str + tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. + inst_id : str + Satellite ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. + + Returns + ------- + data : xr.DataSet + A xarray DataSet with data prepared for the pysat.Instrument + meta : pysat.Meta + Metadata formatted for a pysat.Instrument object. + + Raises + ------ + ValueError + If temporal dimensions are not consistent + + Note + ---- + Any additional keyword arguments passed to pysat.Instrument + upon instantiation are passed along to this routine. + + Examples + -------- + :: + + inst = pysat.Instrument('timed', 'tidi', + inst_id='', tag='vector') + inst.load(2005, 179) + + """ + + labels = {'units': ('Units', str), 'name': ('Long_Name', str), + 'notes': ('Var_Notes', str), 'desc': ('CatDesc', str), + 'plot': ('plot', str), 'axis': ('axis', str), + 'scale': ('scale', str), + 'min_val': ('Valid_Min', np.float64), + 'max_val': ('Valid_Max', np.float64), + 'fill_val': ('fill', np.float64)} + + # Generate custom meta translation table. When left unspecified the default + # table handles the multiple values for fill. We must recreate that + # functionality in our table. The targets for meta_translation should + # map to values in `labels` above. + meta_translation = {'FIELDNAM': 'plot', 'LABLAXIS': 'axis', + 'ScaleTyp': 'scale', 'VALIDMIN': 'Valid_Min', + 'Valid_Min': 'Valid_Min', 'VALIDMAX': 'Valid_Max', + 'Valid_Max': 'Valid_Max', '_FillValue': 'fill', + 'FillVal': 'fill', 'TIME_BASE': 'time_base'} + if inst_id == 'ncar': + if tag == 'vector': + data, meta = cdw.load(fnames, tag, inst_id, + pandas_format=True) + data = data.to_xarray() + data = data.rename(index='time') + + elif inst_id == '': + data = [] + for fname in fnames: + idata, meta = load_netcdf(fname, pandas_format=pandas_format, + epoch_name='time', epoch_unit='s', + epoch_origin='1980-01-06 00:00:00', + meta_kwargs={'labels': labels}, + meta_translation=meta_translation, + drop_meta_labels='FILLVAL', + ) + data.append(idata) + + if tag in ['vector', 'profile']: + _dim = 'nvec' if tag == 'vector' else 'nprofs' + alt_retrieved = data[0].alt_retrieved + for i, idata in enumerate(data): + idata = idata.drop_vars('alt_retrieved') + idata = idata.assign_coords(time=idata.time) + data[i] = idata.rename({_dim: 'time'}) + data = xr.concat(data, 'time') + data = data.assign_coords(alt=('nalts', alt_retrieved.data)) + data = data.rename(nalts='alt') + data = data.sortby('time') + + elif tag == 'los': + for i, idata in enumerate(data): + idata = idata.assign_coords(time=idata.time) + data[i] = idata.rename(nlos='time') + + hh = [t.drop_dims(['time', 'nrecs_size']) for t in data] + names2avoid = list(hh[0].data_vars.keys()) + ff = [t.drop_dims(['time']).drop_vars(names2avoid) for t in data] + ff = xr.concat(ff, 'nrecs_size') + ee = [t.drop_dims(['nrecs_size']).drop_vars(names2avoid) + for t in data] + ee = xr.concat(ee, 'time') + data = xr.merge([ff, ee, hh[0]]) + + return data, meta + + +# Set download tags. Note that tlimb uses the general implementation, while +# other tags use the cdasws implementation. +url = '/pub/data/timed/tidi/{tag:s}/{{year:04d}}/' +download_tags = {'': {tag: {'remote_dir': url.format(tag=tag), + 'zip_method': 'gz', + 'fname': fname.format(ext=fname_ext[tag], + gz='.gz')} + for tag in tags.keys()}, + 'ncar': {'vector': 'TIMED_WINDVECTORSNCAR_TIDI'}} + + +# Set the download routine +def download(date_array, tag='', inst_id='', data_path=None): + """Download NASA TIMED/TIDI data. + + This routine is intended to be used by pysat instrument modules supporting + a particular NASA CDAWeb dataset. + + Parameters + ---------- + date_array : array-like + Array of datetimes to download data for. Provided by pysat. + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + data_path : str or NoneType + Path to data directory. If None is specified, the value previously + set in Instrument.files.data_path is used. (default=None) + + """ + + if inst_id in ['ncar',]: + cdw.cdas_download(date_array, tag=tag, inst_id=inst_id, + supported_tags=download_tags, data_path=data_path) + else: + cdw.download(date_array, tag=tag, inst_id=inst_id, + supported_tags=download_tags, data_path=data_path) + + +# Set the list_remote_files routine +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) diff --git a/pysatNASA/tests/test_methods_platform.py b/pysatNASA/tests/test_methods_platform.py index 169724a8..5b70d399 100644 --- a/pysatNASA/tests/test_methods_platform.py +++ b/pysatNASA/tests/test_methods_platform.py @@ -20,7 +20,7 @@ class TestTIMEDMethods(object): def setup_method(self): """Set up the unit test environment for each method.""" - self.names = ['see', 'saber', 'guvi'] + self.names = ['see', 'saber', 'guvi', 'tidi'] self.module = methods.timed self.platform_str = '(TIMED)' return