From 6c8fb455da714a3904268ad4ef4e805504dd51fb Mon Sep 17 00:00:00 2001 From: Azusa Date: Mon, 12 Dec 2022 14:34:36 +0800 Subject: [PATCH 1/3] Add a way to set optimizer hyperparameters quickly. --- easytorch/core/optimizer_builder.py | 42 ++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/easytorch/core/optimizer_builder.py b/easytorch/core/optimizer_builder.py index f64965f..30d4115 100644 --- a/easytorch/core/optimizer_builder.py +++ b/easytorch/core/optimizer_builder.py @@ -37,6 +37,14 @@ def build_optim(optim_cfg: Dict, model: nn.Module) -> optim.Optimizer: optim_cfg (Dict): optimizer config model (nn.Module): model defined by user + Option: + Add parameters with special optimizer hyperparameters by set _optim attribute + example: + net = nn.Parameter(torch.zeros(10)) + setattr(net, "_optim", {'lr': 0.01,"weight_decay": 0.0}) + + Information of optimizer will be printed. You can check it. + Returns: optimizer (optim.Optimizer) """ @@ -48,8 +56,40 @@ def build_optim(optim_cfg: Dict, model: nn.Module) -> optim.Optimizer: optim_type = getattr(optim, optim_cfg['TYPE']) else: optim_type = getattr(easyoptim, optim_cfg['TYPE']) + + # Obtain General parameters optim_param = optim_cfg['PARAM'].copy() - optimizer = optim_type(model.parameters(), **optim_param) + + # All parameters in the model + all_parameters = list(model.parameters()) + + # General parameters don't contain the special _optim key + params = [p for p in all_parameters if not hasattr(p, "_optim")] + + # Create an optimizer with the general parameters + optimizer = optim_type(params, **optim_param) + + # Add parameters with special hyperparameters + hps = [getattr(p, "_optim") for p in all_parameters if hasattr(p, "_optim")] + hps = [ + # Create unique special hyperparameters dicts + dict(s) for s in sorted(list(dict.fromkeys(frozenset(hp.items()) for hp in hps))) + ] + for hp in hps: + params = [p for p in all_parameters if getattr(p, "_optim", None) == hp] + optimizer.add_param_group( + {"params": params, **hp} + ) + + # Print optimizer info + keys = sorted(set([k for hp in hps for k in hp.keys()])) + for i, g in enumerate(optimizer.param_groups): + group_hps = {k: g.get(k, None) for k in keys} + print(' | '.join([ + f"Optimizer group {i}", + f"{len(g['params'])} tensors", + ] + [f"{k} {v}" for k, v in group_hps.items()])) + return optimizer From 127cb77747b71ba46cfea9b0bbc285681e2f2f96 Mon Sep 17 00:00:00 2001 From: Azusa Date: Wed, 21 Dec 2022 10:02:23 +0800 Subject: [PATCH 2/3] you can define your optimizer by override def init_optim(self, cfg: Config) --- easytorch/core/optimizer_builder.py | 2 +- easytorch/core/runner.py | 40 ++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/easytorch/core/optimizer_builder.py b/easytorch/core/optimizer_builder.py index 30d4115..7d48308 100644 --- a/easytorch/core/optimizer_builder.py +++ b/easytorch/core/optimizer_builder.py @@ -57,7 +57,7 @@ def build_optim(optim_cfg: Dict, model: nn.Module) -> optim.Optimizer: else: optim_type = getattr(easyoptim, optim_cfg['TYPE']) - # Obtain General parameters + # Obtain general parameters optim_param = optim_cfg['PARAM'].copy() # All parameters in the model diff --git a/easytorch/core/runner.py b/easytorch/core/runner.py index 37f4830..49c5156 100644 --- a/easytorch/core/runner.py +++ b/easytorch/core/runner.py @@ -25,6 +25,28 @@ class Runner(metaclass=ABCMeta): """Base EasyTorch Runner """ + """Base EasyTorch Runner + init_logger() + define_model() unrealized + build_train_dataset unrealized + build_val_dataset() unrealized + get_ckpt_path() + build_model() + get_ckpt_path() + save_model() + load_model_resume() + load_model() + train() + init_training() + on_epoch_start() + on_training_end() only close tensorboard + train_iters() unrealized + backward() + validate() + init_validation() + on_validating_start() unrealized + on_validating_end() unrealized + """ def __init__(self, cfg: Config): # default logger @@ -362,6 +384,16 @@ def train(self, cfg: Config): self.on_training_end() + def init_optim(self, cfg: Config): + """Initialize optimizer + + Args: + cfg (Dict): config + """ + # create lr_scheduler + self.optim = build_optim(cfg['TRAIN.OPTIM'], self.model) + self.logger.info('Set optim: {}'.format(self.optim)) + def init_lr_scheduler(self, cfg: Config): """Initialize lr_scheduler @@ -397,8 +429,7 @@ def init_training(self, cfg: Config): self.register_epoch_meter('train_time', 'train', '{:.2f} (s)', plt=False) # create optim - self.optim = build_optim(cfg['TRAIN.OPTIM'], self.model) - self.logger.info('Set optim: {}'.format(self.optim)) + self.init_optim(cfg) # create lr_scheduler self.init_lr_scheduler(cfg) @@ -433,7 +464,10 @@ def on_epoch_start(self, epoch: int): self.logger.info('Epoch {:d} / {:d}'.format(epoch, self.num_epochs)) # update lr meter if self.scheduler is not None: - self.update_epoch_meter('lr', self.scheduler.get_last_lr()[0]) + try: + self.update_epoch_meter('lr', self.scheduler.get_last_lr()[0]) + except NotImplementedError: + self.update_epoch_meter('lr', self.scheduler.get_lr()[0]) # set epoch for sampler in distributed mode # see https://pytorch.org/docs/stable/data.html From 38f78f32ad68e4d683865b8a89f76d1bd3359951 Mon Sep 17 00:00:00 2001 From: Azusa Date: Thu, 5 Jan 2023 10:41:19 +0800 Subject: [PATCH 3/3] add return in save_best_model to support some extension --- easytorch/core/runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easytorch/core/runner.py b/easytorch/core/runner.py index 49c5156..90f0644 100644 --- a/easytorch/core/runner.py +++ b/easytorch/core/runner.py @@ -645,6 +645,8 @@ def save_best_model(self, epoch: int, metric_name: str, greater_best: bool = Tru '{}_best_{}.pt'.format(self.model_name, metric_name.replace('/', '_')) ) save_ckpt(ckpt_dict, ckpt_path, self.logger) + return True + return False @master_only def register_epoch_meter(self, name, meter_type, fmt='{:f}', plt=True):