diff --git a/pyPRMS/control/Control.py b/pyPRMS/control/Control.py index e84da81..051889e 100644 --- a/pyPRMS/control/Control.py +++ b/pyPRMS/control/Control.py @@ -24,10 +24,6 @@ con = None -# Rich library -# pretty.install() -# con = Console(record=False, width=200) - cond_check = {'=': operator.eq, '>': operator.gt, '<': operator.lt} @@ -40,7 +36,9 @@ class Control(object): # Author: Parker Norton (pnorton@usgs.gov) # Create date: 2019-04-18 - def __init__(self, metadata: MetaDataType, verbose: Optional[bool] = False): + def __init__(self, metadata: MetaDataType, + include_missing: Optional[bool] = True, + verbose: Optional[bool] = False): """Create Control object. """ @@ -48,20 +46,14 @@ def __init__(self, metadata: MetaDataType, verbose: Optional[bool] = False): con = get_console_instance() # Container to hold dicionary of ControlVariables - # self.__control_vars = OrderedDict() self.__control_vars: Dict = {} self.__header: Optional[List[str]] = None - self.__verbose = verbose - # Create an entry for each variable in the control section of - # the metadata dictionary - # for cvar, cvals in metadata['control'].items(): - # self.add(name=cvar, meta=cvals) - for cvar in metadata['control'].keys(): - self.add(name=cvar, meta=metadata['control']) + self.__metadata = metadata + self.__verbose = verbose - if verbose: - con.print('[bold]Pre-populate control variables done[/]') + if include_missing: + self._preload_metadata() def __getitem__(self, item: str) -> ControlVariable: """Get ControlVariable object for a variable. @@ -203,7 +195,7 @@ def modules(self) -> Dict[str, str]: return mod_dict - def add(self, name: str, meta=None): + def add(self, name: str): # , meta=None): """Add a control variable by name. :param name: Name of the control variable @@ -214,8 +206,7 @@ def add(self, name: str, meta=None): if self.exists(name): raise ControlError("Control variable already exists") - self.__control_vars[name] = ControlVariable(name=name, meta=meta) - # self.__control_vars[name] = ControlVariable(name=name, datatype=datatype, meta=meta) + self.__control_vars[name] = ControlVariable(name=name, meta=self.__metadata['control']) def diff(self, other: 'Control') -> dict: """A difference listing/dictionary against another Control object. @@ -421,3 +412,12 @@ def _read(self): """Abstract function for reading. """ assert False, 'Control._read() must be defined by child class' + + def _preload_metadata(self): + # Create an entry for each variable in the control section of + # the metadata dictionary + for cvar in self.__metadata['control'].keys(): + self.add(name=cvar) # , meta=self.__metadata['control']) + + if self.__verbose: + con.print('[bold]Pre-populate control variables done[/]') diff --git a/pyPRMS/control/ControlFile.py b/pyPRMS/control/ControlFile.py index 0ff05cd..670ef5a 100644 --- a/pyPRMS/control/ControlFile.py +++ b/pyPRMS/control/ControlFile.py @@ -23,14 +23,16 @@ class ControlFile(Control): def __init__(self, filename: Union[str, Path], metadata, + include_missing: Optional[bool] = False, verbose: Optional[bool] = False): - super(ControlFile, self).__init__(metadata=metadata, verbose=verbose) + super(ControlFile, self).__init__(metadata=metadata, include_missing=include_missing, verbose=verbose) global con con = get_console_instance() self.__verbose = verbose self.__isloaded = False + self.__include_missing = include_missing if isinstance(filename, str): filename = Path(filename) @@ -85,6 +87,12 @@ def _read(self): con.print(f'[orange3]WARNING[/]: [bold]{varname}[/] already exists') chk_vars.append(varname) + if not self.__include_missing: + try: + self.add(name=varname) # , meta=self.__metadata['control']) + except ControlError: + con.print(f'[orange3]WARNING[/]: [bold]{varname}[/] duplicated in the control file') + numval = int(next(it)) # number of values for this variable valuetype = int(next(it)) # Variable type (1 - integer, 2 - float, 4 - character) diff --git a/pyPRMS/metadata/metadata.py b/pyPRMS/metadata/metadata.py index 55427c2..c09681d 100644 --- a/pyPRMS/metadata/metadata.py +++ b/pyPRMS/metadata/metadata.py @@ -106,7 +106,7 @@ def __control_to_dict(self, xml_root: xmlET.Element, if var_version > req_version: if self.__verbose: # pragma: no cover - con.print(f'[green]INFO[/]: [bold]{name}[/] rejected by version {str(var_version)}, req: {str(req_version)}') + con.print(f'[green]INFO[/]: [bold]{name}[/] requires version {str(var_version)}') del meta_dict[name] continue @@ -119,7 +119,7 @@ def __control_to_dict(self, xml_root: xmlET.Element, if depr_version <= req_version: if self.__verbose: # pragma: no cover - con.print(f'[green]INFO[/]: [bold]{name}[/] rejected by deprecation version {str(depr_version)}, req: {str(req_version)}') + con.print(f'[green]INFO[/]: [bold]{name}[/] was deprecated at version {str(depr_version)}') del meta_dict[name] continue diff --git a/tests/func/test_Cbh.py b/tests/func/test_Cbh.py index 2ba15f6..f7ae39f 100644 --- a/tests/func/test_Cbh.py +++ b/tests/func/test_Cbh.py @@ -1,5 +1,7 @@ import pytest import os +import numpy as np +import xarray as xr from pathlib import Path from distutils import dir_util @@ -54,7 +56,8 @@ def test_read_ctl_ascii_roundtrip_ascii(self, datadir, pdb_instance, meta_instan nhm_ids = pdb_instance.get('nhm_id').data ctl = ControlFile(datadir / 'control.default.bandit', metadata=meta_instance.metadata, verbose=False) - cbh = Cbh(str(datadir), engine='ascii', metadata=meta_instance.metadata, control=ctl) + cbh = Cbh(str(datadir), engine='ascii', metadata=meta_instance.metadata, control=ctl, + parameters=pdb_instance, verbose=True) assert not cbh.has_nhm_id cbh.set_nhm_id(nhm_ids) @@ -92,3 +95,21 @@ def test_read_single_ascii_roundtrip_ascii(self, datadir, pdb_instance, meta_ins lines_chk = f.readlines() assert lines_orig == lines_chk + + def test_read_netcdf_roundtrip_netcdf(self, datadir, pdb_instance, meta_instance, tmp_path): + out_path = tmp_path / 'run_files' + out_path.mkdir() + + cbh = Cbh(str(datadir.join('cbh.nc')), engine='netcdf', metadata=meta_instance.metadata) + + out_file = out_path / 'cbh.nc' + cbh.write_netcdf(out_file) + + # Check that the values of the data variables match + ds_tmp = xr.open_dataset(out_file, chunks={}) + ds_tmp = ds_tmp.assign_coords(nhru=ds_tmp.nhm_id) + + ds_orig = cbh.data + + for vv in ds_orig.data_vars: + np.testing.assert_equal(ds_orig[vv].values, ds_tmp[vv].values) diff --git a/tests/func/test_Cbh/control.default.bandit b/tests/func/test_Cbh/control.default.bandit index 6a27972..8900b52 100644 --- a/tests/func/test_Cbh/control.default.bandit +++ b/tests/func/test_Cbh/control.default.bandit @@ -745,7 +745,7 @@ precip_map_file 4 precip.map #### -potet_coef_dynamic +potetcoef_dynamic 1 4 dyn_potet_coef.param diff --git a/tests/func/test_Control.py b/tests/func/test_Control.py index a912475..f98bcb4 100644 --- a/tests/func/test_Control.py +++ b/tests/func/test_Control.py @@ -8,7 +8,7 @@ @pytest.fixture(scope='class') def control_object(): prms_meta = MetaData(verbose=False).metadata - ctl = Control(metadata=prms_meta) + ctl = Control(metadata=prms_meta, verbose=True) return ctl @@ -224,6 +224,19 @@ def test_set_header_with_none(self, control_object): control_object.header = None assert control_object.header is None + def test_cbh_files(self, control_object): + """Check the default set of CBH files""" + expected = ['cloudcover.day', + 'humidity.day', + 'potet.day', + 'precip.day', + 'swrad.day', + 'tmax.day', + 'tmin.day', + 'transp.day', + 'windspeed.day'] + assert control_object.cbh_files == expected + def test_default_modules(self, control_object): """Check the default set of modules is correct""" expected = {'et_module': 'potet_jh', @@ -275,13 +288,13 @@ def test_add_invalid_variable(self, control_object, metadata_ctl, name): """Add an invalid control variable name""" with pytest.raises(ValueError): - control_object.add(name=name, meta=metadata_ctl) + control_object.add(name=name) def test_add_duplicate_variable(self, control_object, metadata_ctl): """Add a duplicate control variable""" with pytest.raises(ControlError): - control_object.add(name='et_module', meta=metadata_ctl) + control_object.add(name='et_module') def test_remove_variable(self, control_object): control_object.remove('albedo_day') diff --git a/tests/func/test_ControlFile.py b/tests/func/test_ControlFile.py index bd2d6a9..9e3844f 100644 --- a/tests/func/test_ControlFile.py +++ b/tests/func/test_ControlFile.py @@ -31,7 +31,7 @@ def test_diff_control(self, datadir): prms_meta = MetaData(verbose=False).metadata - ctl = ControlFile(control_file, metadata=prms_meta, verbose=False) + ctl = ControlFile(control_file, metadata=prms_meta, include_missing=True, verbose=False) # Create a instance of a base control class ctl_base = Control(metadata=prms_meta) @@ -77,9 +77,8 @@ def test_bad_var_in_file(self, datadir): prms_meta = MetaData(verbose=True).metadata - ctl = ControlFile(control_file, metadata=prms_meta, verbose=False) - - assert not ctl.exists('random_ctl_var') + with pytest.raises(ValueError): + ctl = ControlFile(control_file, metadata=prms_meta, verbose=False) def test_bad_num_vals_in_file(self, datadir): """Too many values for a variable should raise ControlError""" diff --git a/tests/func/test_ParameterFile/control.default.bandit b/tests/func/test_ParameterFile/control.default.bandit index 94d4a6d..efa4c5d 100644 --- a/tests/func/test_ParameterFile/control.default.bandit +++ b/tests/func/test_ParameterFile/control.default.bandit @@ -744,8 +744,3 @@ segment_transfer_file 1 4 seg.transfer -#### -cbh_binary_flag -1 -1 -0