Bnb/multi gpu

.gitignore

@@ -124,3 +124,4 @@ tags
 
 # test dirs
 exo_cache
+.timings

sup3r/batch/batch_cli.py

@@ -1,10 +1,14 @@
 # pylint: disable=all
 """Batch Job CLI entry points."""
+import logging
+
 import click
 from gaps.batch import BatchJob
 
 from sup3r import __version__
 
+logger = logging.getLogger(__name__)
+
 
 @click.group()
 @click.version_option(version=__version__)
@@ -41,6 +45,15 @@ def from_config(ctx, config_file, dry_run, cancel, delete, monitor_background,
     """Run Sup3r batch from a config file."""
     ctx.ensure_object(dict)
     ctx.obj['VERBOSE'] = verbose or ctx.obj.get('VERBOSE', False)
+    logger.info(
+        'Starting batch job from %s (dry_run=%s, cancel=%s, delete=%s, '
+        'monitor_background=%s).',
+        config_file,
+        dry_run,
+        cancel,
+        delete,
+        monitor_background,
+    )
     batch = BatchJob(config_file)
 
     if cancel:
@@ -50,6 +63,8 @@ def from_config(ctx, config_file, dry_run, cancel, delete, monitor_background,
     else:
         batch.run(dry_run=dry_run, monitor_background=monitor_background)
 
+    logger.info('Finished batch CLI invocation for %s.', config_file)
+
 
 if __name__ == '__main__':
     main(obj={})

sup3r/bias/base.py

@@ -115,8 +115,10 @@ class is used, all data will be loaded in this class'
         """
 
         logger.info(
-            'Initializing DataRetrievalBase for base dset "{}" '
-            'correcting biased dataset(s): {}'.format(base_dset, bias_feature)
+            'Initializing DataRetrievalBase for base dset "%s" correcting '
+            'biased dataset(s): %s',
+            base_dset,
+            bias_feature,
         )
         self.base_fps = base_fps
         self.bias_fps = bias_fps
@@ -235,9 +237,8 @@ def distance_upper_bound(self):
             diff = np.max(np.median(diff, axis=0))
             self._distance_upper_bound = diff
             logger.info(
-                'Set distance upper bound to {:.4f}'.format(
-                    self._distance_upper_bound
-                )
+                'Set distance upper bound to %.4f',
+                self._distance_upper_bound,
             )
         return self._distance_upper_bound
 
@@ -587,13 +588,11 @@ def _match_zero_rate(bias_data, base_data):
         q_zero_bias_out = np.nanmean(bias_data == 0)
 
         logger.debug(
-            'Input bias/base zero rate is {:.3e}/{:.3e}, '
-            'output is {:.3e}/{:.3e}'.format(
-                q_zero_bias_in,
-                q_zero_base_in,
-                q_zero_bias_out,
-                q_zero_base_out,
-            )
+            'Input bias/base zero rate is %.3e/%.3e, output is %.3e/%.3e',
+            q_zero_bias_in,
+            q_zero_base_in,
+            q_zero_bias_out,
+            q_zero_base_out,
         )
 
         return bias_data

sup3r/bias/bias_calc.py

@@ -158,9 +158,7 @@ def write_outputs(self, fp_out, out):
                 for k, v in self.meta.items():
                     f.attrs[k] = json.dumps(v)
 
-                logger.info(
-                    'Wrote scalar adder factors to file: {}'.format(fp_out)
-                )
+                logger.info('Wrote scalar adder factors to file: %s', fp_out)
 
     def _get_run_kwargs(self, **kwargs_extras):
         """Get dictionary of kwarg dictionaries to use for calls to
@@ -236,9 +234,8 @@ def run(
         logger.debug('Starting linear correction calculation...')
 
         logger.info(
-            'Initialized scalar / adder with shape: {}'.format(
-                self.bias_gid_raster.shape
-            )
+            'Initialized scalar / adder with shape: %s',
+            self.bias_gid_raster.shape,
         )
         self.out = self._run(
             out=self.out,

sup3r/bias/bias_calc_cli.py

@@ -1,4 +1,5 @@
 """sup3r bias correction calculation CLI entry points."""
+
 import copy
 import logging
 import os
@@ -15,8 +16,12 @@
 
 @click.group()
 @click.version_option(version=__version__)
-@click.option('-v', '--verbose', is_flag=True,
-              help='Flag to turn on debug logging. Default is not verbose.')
+@click.option(
+    '-v',
+    '--verbose',
+    is_flag=True,
+    help='Flag to turn on debug logging. Default is not verbose.',
+)
 @click.pass_context
 def main(ctx, verbose):
     """Sup3r bias calc Command Line Interface"""
@@ -25,16 +30,25 @@ def main(ctx, verbose):
 
 
 @main.command()
-@click.option('--config_file', '-c', required=True,
-              type=click.Path(exists=True),
-              help='sup3r bias correction calculation config .json file.')
-@click.option('-v', '--verbose', is_flag=True,
-              help='Flag to turn on debug logging. Default is not verbose.')
+@click.option(
+    '--config_file',
+    '-c',
+    required=True,
+    type=click.Path(exists=True),
+    help='sup3r bias correction calculation config .json file.',
+)
+@click.option(
+    '-v',
+    '--verbose',
+    is_flag=True,
+    help='Flag to turn on debug logging. Default is not verbose.',
+)
 @click.pass_context
 def from_config(ctx, config_file, verbose=False, pipeline_step=None):
     """Run sup3r bias correction calculation from a config file."""
-    config = BaseCLI.from_config_preflight(ModuleName.BIAS_CALC, ctx,
-                                           config_file, verbose)
+    config = BaseCLI.from_config_preflight(
+        ModuleName.BIAS_CALC, ctx, config_file, verbose
+    )
 
     exec_kwargs = config.get('execution_control', {})
     hardware_option = exec_kwargs.pop('option', 'local')
@@ -44,31 +58,57 @@ def from_config(ctx, config_file, verbose=False, pipeline_step=None):
     log_pattern = config.get('log_pattern', None)
 
     jobs = config['jobs']
+    logger.info(
+        'Preparing bias calculation from %s with hardware=%s across %s jobs '
+        'using %s.',
+        config_file,
+        hardware_option,
+        len(jobs),
+        calc_class_name,
+    )
     for i_node, job in enumerate(jobs):
         node_config = copy.deepcopy(job)
         node_config['status_dir'] = config['status_dir']
         node_config['log_file'] = (
-            log_pattern if log_pattern is None
-            else os.path.normpath(log_pattern.format(node_index=i_node)))
-        name = ('{}_{}'.format(basename, str(i_node).zfill(6)))
+            log_pattern
+            if log_pattern is None
+            else os.path.normpath(log_pattern.format(node_index=i_node))
+        )
+        name = '{}_{}'.format(basename, str(i_node).zfill(6))
         ctx.obj['NAME'] = name
         node_config['job_name'] = name
-        node_config["pipeline_step"] = pipeline_step
+        node_config['pipeline_step'] = pipeline_step
 
         cmd = BiasCalcClass.get_node_cmd(node_config)
 
         cmd_log = '\n\t'.join(cmd.split('\n'))
-        logger.debug(f'Running command:\n\t{cmd_log}')
+        logger.debug('Running command:\n\t%s', cmd_log)
+        logger.info(
+            'Queueing bias calculation node %s as job "%s".',
+            i_node,
+            name,
+        )
 
         if hardware_option.lower() in AVAILABLE_HARDWARE_OPTIONS:
             kickoff_slurm_job(ctx, cmd, pipeline_step, **exec_kwargs)
         else:
             kickoff_local_job(ctx, cmd, pipeline_step)
 
-
-def kickoff_slurm_job(ctx, cmd, pipeline_step=None, alloc='sup3r',
-                      memory=None, walltime=4, feature=None,
-                      stdout_path='./stdout/'):
+    logger.info(
+        'Finished queueing bias calculation work for %s jobs.', len(jobs)
+    )
+
+
+def kickoff_slurm_job(
+    ctx,
+    cmd,
+    pipeline_step=None,
+    alloc='sup3r',
+    memory=None,
+    walltime=4,
+    feature=None,
+    stdout_path='./stdout/',
+):
     """Run sup3r on HPC via SLURM job submission.
 
     Parameters
@@ -94,8 +134,17 @@ def kickoff_slurm_job(ctx, cmd, pipeline_step=None, alloc='sup3r',
     stdout_path : str
         Path to print .stdout and .stderr files.
     """
-    BaseCLI.kickoff_slurm_job(ModuleName.BIAS_CALC, ctx, cmd, alloc, memory,
-                              walltime, feature, stdout_path, pipeline_step)
+    BaseCLI.kickoff_slurm_job(
+        ModuleName.BIAS_CALC,
+        ctx,
+        cmd,
+        alloc,
+        memory,
+        walltime,
+        feature,
+        stdout_path,
+        pipeline_step,
+    )
 
 
 def kickoff_local_job(ctx, cmd, pipeline_step=None):