Control default values pan

pyPRMS/control/Control.py

@@ -24,10 +24,6 @@
 
 con = None
 
-# Rich library
-# pretty.install()
-# con = Console(record=False, width=200)
-
 cond_check = {'=': operator.eq,
               '>': operator.gt,
               '<': operator.lt}
@@ -40,28 +36,24 @@ class Control(object):
     # Author: Parker Norton (pnorton@usgs.gov)
     # Create date: 2019-04-18
 
-    def __init__(self, metadata: MetaDataType, verbose: Optional[bool] = False):
+    def __init__(self, metadata: MetaDataType,
+                 include_missing: Optional[bool] = True,
+                 verbose: Optional[bool] = False):
         """Create Control object.
         """
 
         global con
         con = get_console_instance()
 
         # Container to hold dicionary of ControlVariables
-        # self.__control_vars = OrderedDict()
         self.__control_vars: Dict = {}
         self.__header: Optional[List[str]] = None
-        self.__verbose = verbose
 
-        # Create an entry for each variable in the control section of
-        # the metadata dictionary
-        # for cvar, cvals in metadata['control'].items():
-        #     self.add(name=cvar, meta=cvals)
-        for cvar in metadata['control'].keys():
-            self.add(name=cvar, meta=metadata['control'])
+        self.__metadata = metadata
+        self.__verbose = verbose
 
-        if verbose:
-            con.print('[bold]Pre-populate control variables done[/]')
+        if include_missing:
+            self._preload_metadata()
 
     def __getitem__(self, item: str) -> ControlVariable:
         """Get ControlVariable object for a variable.
@@ -203,7 +195,7 @@ def modules(self) -> Dict[str, str]:
 
         return mod_dict
 
-    def add(self, name: str, meta=None):
+    def add(self, name: str):   # , meta=None):
         """Add a control variable by name.
 
         :param name: Name of the control variable
@@ -214,8 +206,7 @@ def add(self, name: str, meta=None):
 
         if self.exists(name):
             raise ControlError("Control variable already exists")
-        self.__control_vars[name] = ControlVariable(name=name, meta=meta)
-        # self.__control_vars[name] = ControlVariable(name=name, datatype=datatype, meta=meta)
+        self.__control_vars[name] = ControlVariable(name=name, meta=self.__metadata['control'])
 
     def diff(self, other: 'Control') -> dict:
         """A difference listing/dictionary against another Control object.
@@ -421,3 +412,12 @@ def _read(self):
         """Abstract function for reading.
         """
         assert False, 'Control._read() must be defined by child class'
+
+    def _preload_metadata(self):
+        # Create an entry for each variable in the control section of
+        # the metadata dictionary
+        for cvar in self.__metadata['control'].keys():
+            self.add(name=cvar)   # , meta=self.__metadata['control'])
+
+        if self.__verbose:
+            con.print('[bold]Pre-populate control variables done[/]')

pyPRMS/control/ControlFile.py

@@ -23,14 +23,16 @@ class ControlFile(Control):
 
     def __init__(self, filename: Union[str, Path],
                  metadata,
+                 include_missing: Optional[bool] = False,
                  verbose: Optional[bool] = False):
-        super(ControlFile, self).__init__(metadata=metadata, verbose=verbose)
+        super(ControlFile, self).__init__(metadata=metadata, include_missing=include_missing, verbose=verbose)
 
         global con
         con = get_console_instance()
 
         self.__verbose = verbose
         self.__isloaded = False
+        self.__include_missing = include_missing
 
         if isinstance(filename, str):
             filename = Path(filename)
@@ -85,6 +87,12 @@ def _read(self):
                         con.print(f'[orange3]WARNING[/]: [bold]{varname}[/] already exists')
                     chk_vars.append(varname)
 
+                if not self.__include_missing:
+                    try:
+                        self.add(name=varname)   # , meta=self.__metadata['control'])
+                    except ControlError:
+                        con.print(f'[orange3]WARNING[/]: [bold]{varname}[/] duplicated in the control file')
+
                 numval = int(next(it))  # number of values for this variable
                 valuetype = int(next(it))  # Variable type (1 - integer, 2 - float, 4 - character)
 

pyPRMS/metadata/metadata.py

@@ -106,7 +106,7 @@ def __control_to_dict(self, xml_root: xmlET.Element,
 
                 if var_version > req_version:
                     if self.__verbose:   # pragma: no cover
-                        con.print(f'[green]INFO[/]: [bold]{name}[/] rejected by version {str(var_version)}, req: {str(req_version)}')
+                        con.print(f'[green]INFO[/]: [bold]{name}[/] requires version {str(var_version)}')
 
                     del meta_dict[name]
                     continue
@@ -119,7 +119,7 @@ def __control_to_dict(self, xml_root: xmlET.Element,
 
                 if depr_version <= req_version:
                     if self.__verbose:   # pragma: no cover
-                        con.print(f'[green]INFO[/]: [bold]{name}[/] rejected by deprecation version {str(depr_version)}, req: {str(req_version)}')
+                        con.print(f'[green]INFO[/]: [bold]{name}[/] was deprecated at version {str(depr_version)}')
 
                     del meta_dict[name]
                     continue

tests/func/test_Cbh.py

@@ -1,5 +1,7 @@
 import pytest
 import os
+import numpy as np
+import xarray as xr
 from pathlib import Path
 from distutils import dir_util
 
@@ -54,7 +56,8 @@ def test_read_ctl_ascii_roundtrip_ascii(self, datadir, pdb_instance, meta_instan
         nhm_ids = pdb_instance.get('nhm_id').data
 
         ctl = ControlFile(datadir / 'control.default.bandit', metadata=meta_instance.metadata, verbose=False)
-        cbh = Cbh(str(datadir), engine='ascii', metadata=meta_instance.metadata, control=ctl)
+        cbh = Cbh(str(datadir), engine='ascii', metadata=meta_instance.metadata, control=ctl,
+                  parameters=pdb_instance, verbose=True)
 
         assert not cbh.has_nhm_id
         cbh.set_nhm_id(nhm_ids)
@@ -92,3 +95,21 @@ def test_read_single_ascii_roundtrip_ascii(self, datadir, pdb_instance, meta_ins
                 lines_chk = f.readlines()
 
             assert lines_orig == lines_chk
+
+    def test_read_netcdf_roundtrip_netcdf(self, datadir, pdb_instance, meta_instance, tmp_path):
+        out_path = tmp_path / 'run_files'
+        out_path.mkdir()
+
+        cbh = Cbh(str(datadir.join('cbh.nc')), engine='netcdf', metadata=meta_instance.metadata)
+
+        out_file = out_path / 'cbh.nc'
+        cbh.write_netcdf(out_file)
+
+        # Check that the values of the data variables match
+        ds_tmp = xr.open_dataset(out_file, chunks={})
+        ds_tmp = ds_tmp.assign_coords(nhru=ds_tmp.nhm_id)
+
+        ds_orig = cbh.data
+
+        for vv in ds_orig.data_vars:
+            np.testing.assert_equal(ds_orig[vv].values, ds_tmp[vv].values)

tests/func/test_Cbh/control.default.bandit

@@ -745,7 +745,7 @@ precip_map_file
 4
 precip.map
 ####
-potet_coef_dynamic
+potetcoef_dynamic
 1
 4
 dyn_potet_coef.param

tests/func/test_Control.py

@@ -8,7 +8,7 @@
 @pytest.fixture(scope='class')
 def control_object():
     prms_meta = MetaData(verbose=False).metadata
-    ctl = Control(metadata=prms_meta)
+    ctl = Control(metadata=prms_meta, verbose=True)
 
     return ctl
 
@@ -224,6 +224,19 @@ def test_set_header_with_none(self, control_object):
         control_object.header = None
         assert control_object.header is None
 
+    def test_cbh_files(self, control_object):
+        """Check the default set of CBH files"""
+        expected = ['cloudcover.day',
+                    'humidity.day',
+                    'potet.day',
+                    'precip.day',
+                    'swrad.day',
+                    'tmax.day',
+                    'tmin.day',
+                    'transp.day',
+                    'windspeed.day']
+        assert control_object.cbh_files == expected
+
     def test_default_modules(self, control_object):
         """Check the default set of modules is correct"""
         expected = {'et_module': 'potet_jh',
@@ -275,13 +288,13 @@ def test_add_invalid_variable(self, control_object, metadata_ctl, name):
         """Add an invalid control variable name"""
 
         with pytest.raises(ValueError):
-            control_object.add(name=name, meta=metadata_ctl)
+            control_object.add(name=name)
 
     def test_add_duplicate_variable(self, control_object, metadata_ctl):
         """Add a duplicate control variable"""
 
         with pytest.raises(ControlError):
-            control_object.add(name='et_module', meta=metadata_ctl)
+            control_object.add(name='et_module')
 
     def test_remove_variable(self, control_object):
         control_object.remove('albedo_day')

tests/func/test_ControlFile.py

@@ -31,7 +31,7 @@ def test_diff_control(self, datadir):
 
         prms_meta = MetaData(verbose=False).metadata
 
-        ctl = ControlFile(control_file, metadata=prms_meta, verbose=False)
+        ctl = ControlFile(control_file, metadata=prms_meta, include_missing=True, verbose=False)
 
         # Create a instance of a base control class
         ctl_base = Control(metadata=prms_meta)
@@ -77,9 +77,8 @@ def test_bad_var_in_file(self, datadir):
 
         prms_meta = MetaData(verbose=True).metadata
 
-        ctl = ControlFile(control_file, metadata=prms_meta, verbose=False)
-
-        assert not ctl.exists('random_ctl_var')
+        with pytest.raises(ValueError):
+            ctl = ControlFile(control_file, metadata=prms_meta, verbose=False)
 
     def test_bad_num_vals_in_file(self, datadir):
         """Too many values for a variable  should raise ControlError"""

tests/func/test_ParameterFile/control.default.bandit

@@ -744,8 +744,3 @@ segment_transfer_file
 1
 4
 seg.transfer
-####
-cbh_binary_flag
-1
-1
-0