Module utils.utils
Expand source code
#!/usr/bin/env python3
import numpy as np
import os
import glob
import json
class AverageMeter(object):
"""Computes and stores the average and current value"""
def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count if self.count > 0 else None
def get_val(self):
return self.val
def get_avg(self):
return self.avg
def init_metrics_meter(round):
metrics_meter = {
'round': round,
'loss': AverageMeter(),
'top_1_acc': AverageMeter(),
'top_5_acc': AverageMeter(),
'neq_perplexity': AverageMeter(),
}
return metrics_meter
def neg_perplexity_from_loss(loss):
return - np.exp(loss)
def get_model_str_from_obj(model):
return str(list(model.modules())[0]).split("\n")[0][:-1]
def create_metrics_dict(metrics):
metrics_dict = {}
for k in metrics:
if k == 'round' or k == 'regression':
metrics_dict[k] = metrics[k]
continue
metrics_dict[k] = metrics[k].get_avg()
return metrics_dict
def create_model_dir(args, lr=True):
model_dataset = '_'.join([args.model, args.dataset])
run_id = f'id={args.run_id}'
model_dir = os.path.join(args.checkpoint_dir, model_dataset, run_id)
if lr:
run_hp = os.path.join(f"lr=l_{str(args.local_lr)}_g_{str(args.global_lr)}",
f"seed_init={str(args.manual_init_seed)}",
f"seed_runtime={str(args.manual_runtime_seed)}"
)
model_dir = os.path.join(model_dir, run_hp)
return model_dir
# TODO: Update, to work for dual master and client stepsize (applies to 2 functions below)
def get_best_lr_and_metric(args, last=True):
best_lookup = np.argmin if args.metric in ['loss'] else np.argmax
model_dir_no_lr = create_model_dir(args, lr=False)
lr_dirs = [lr_dir for lr_dir in os.listdir(model_dir_no_lr)
if os.path.isdir(os.path.join(model_dir_no_lr, lr_dir))
and not lr_dir.startswith('.')]
lrs = np.array([float(lr_dir.split('=')[-1]) for lr_dir in lr_dirs])
best_runs_metric = list()
best_runs_dir = list()
for lr_dir in lr_dirs:
json_dir = 'best_metrics.json' if last else 'best_metrics.json'
lr_metric_dirs = glob.glob(model_dir_no_lr + '/' + lr_dir + '/*/' + json_dir)
lr_metric = list()
for lr_metric_dir in lr_metric_dirs:
with open(lr_metric_dir) as json_file:
metric = json.load(json_file)
lr_metric.append(metric[get_metric_key_name(args.metric)])
i_best_run = best_lookup(lr_metric)
best_runs_metric.append(lr_metric[i_best_run])
best_runs_dir.append(lr_metric_dirs[i_best_run])
i_best_lr = best_lookup(best_runs_metric)
best_metric = best_runs_metric[i_best_lr]
best_lr = lrs[i_best_lr]
return best_lr, best_metric, np.sort(lrs)
def get_best_runs(args_exp, metric_name, last=True):
model_dir_no_lr = create_model_dir(args_exp, lr=False)
best_lr, _, _ = get_best_lr_and_metric(args_exp, last=last)
model_dir_lr = os.path.join(model_dir_no_lr, f"lr={str(best_lr)}")
json_dir = 'last_metrics.json' if last else 'best_metrics.json'
metric_dirs = glob.glob(model_dir_lr + '/*/' + json_dir)
with open(metric_dirs[0]) as json_file:
metric = json.load(json_file)
x = {
'eval_p': metric['eval_p'],
'params_p': metric['params_p'],
'macs_p': metric['macs_p']
}
runs = [metric[metric_name]]
for metric_dir in metric_dirs[1:]:
with open(metric_dir) as json_file:
metric = json.load(json_file)
# ignores failed runs
#if not np.isnan(metric['avg_loss']):
runs.append(metric[metric_name])
return x, runs
def log_epoch_info(logger, i, loader, metrics_meter, dataload_duration,
inference_duration, backprop_duration, train=True):
mode_str = 'Train' if train else 'Test'
msg = "{mode_str} [{round}][{current_batch}/{total_batches}]\t" \
"DataLoad time {dataload_duration:.3f}\t" \
"F/W time {inference_duration:.3f}\t" \
"B/W time {backprop_duration:.3f}\t" "Loss {loss:.4f}\t" \
"Prec@1 {prec1:.3f}\t" "Prec@5 {prec5:.3f}\t".format(
mode_str=mode_str,
round=metrics_meter['round'],
current_batch=i,
total_batches=len(loader),
dataload_duration=dataload_duration,
inference_duration=inference_duration,
backprop_duration=backprop_duration,
loss=metrics_meter['loss'].get_avg(),
prec1=metrics_meter['top_1_acc'].get_avg(),
prec5=metrics_meter['top_5_acc'].get_avg())
logger.info(msg)
Functions
def create_metrics_dict(metrics)
-
Expand source code
def create_metrics_dict(metrics): metrics_dict = {} for k in metrics: if k == 'round' or k == 'regression': metrics_dict[k] = metrics[k] continue metrics_dict[k] = metrics[k].get_avg() return metrics_dict
def create_model_dir(args, lr=True)
-
Expand source code
def create_model_dir(args, lr=True): model_dataset = '_'.join([args.model, args.dataset]) run_id = f'id={args.run_id}' model_dir = os.path.join(args.checkpoint_dir, model_dataset, run_id) if lr: run_hp = os.path.join(f"lr=l_{str(args.local_lr)}_g_{str(args.global_lr)}", f"seed_init={str(args.manual_init_seed)}", f"seed_runtime={str(args.manual_runtime_seed)}" ) model_dir = os.path.join(model_dir, run_hp) return model_dir
def get_best_lr_and_metric(args, last=True)
-
Expand source code
def get_best_lr_and_metric(args, last=True): best_lookup = np.argmin if args.metric in ['loss'] else np.argmax model_dir_no_lr = create_model_dir(args, lr=False) lr_dirs = [lr_dir for lr_dir in os.listdir(model_dir_no_lr) if os.path.isdir(os.path.join(model_dir_no_lr, lr_dir)) and not lr_dir.startswith('.')] lrs = np.array([float(lr_dir.split('=')[-1]) for lr_dir in lr_dirs]) best_runs_metric = list() best_runs_dir = list() for lr_dir in lr_dirs: json_dir = 'best_metrics.json' if last else 'best_metrics.json' lr_metric_dirs = glob.glob(model_dir_no_lr + '/' + lr_dir + '/*/' + json_dir) lr_metric = list() for lr_metric_dir in lr_metric_dirs: with open(lr_metric_dir) as json_file: metric = json.load(json_file) lr_metric.append(metric[get_metric_key_name(args.metric)]) i_best_run = best_lookup(lr_metric) best_runs_metric.append(lr_metric[i_best_run]) best_runs_dir.append(lr_metric_dirs[i_best_run]) i_best_lr = best_lookup(best_runs_metric) best_metric = best_runs_metric[i_best_lr] best_lr = lrs[i_best_lr] return best_lr, best_metric, np.sort(lrs)
def get_best_runs(args_exp, metric_name, last=True)
-
Expand source code
def get_best_runs(args_exp, metric_name, last=True): model_dir_no_lr = create_model_dir(args_exp, lr=False) best_lr, _, _ = get_best_lr_and_metric(args_exp, last=last) model_dir_lr = os.path.join(model_dir_no_lr, f"lr={str(best_lr)}") json_dir = 'last_metrics.json' if last else 'best_metrics.json' metric_dirs = glob.glob(model_dir_lr + '/*/' + json_dir) with open(metric_dirs[0]) as json_file: metric = json.load(json_file) x = { 'eval_p': metric['eval_p'], 'params_p': metric['params_p'], 'macs_p': metric['macs_p'] } runs = [metric[metric_name]] for metric_dir in metric_dirs[1:]: with open(metric_dir) as json_file: metric = json.load(json_file) # ignores failed runs #if not np.isnan(metric['avg_loss']): runs.append(metric[metric_name]) return x, runs
def get_model_str_from_obj(model)
-
Expand source code
def get_model_str_from_obj(model): return str(list(model.modules())[0]).split("\n")[0][:-1]
def init_metrics_meter(round)
-
Expand source code
def init_metrics_meter(round): metrics_meter = { 'round': round, 'loss': AverageMeter(), 'top_1_acc': AverageMeter(), 'top_5_acc': AverageMeter(), 'neq_perplexity': AverageMeter(), } return metrics_meter
def log_epoch_info(logger, i, loader, metrics_meter, dataload_duration, inference_duration, backprop_duration, train=True)
-
Expand source code
def log_epoch_info(logger, i, loader, metrics_meter, dataload_duration, inference_duration, backprop_duration, train=True): mode_str = 'Train' if train else 'Test' msg = "{mode_str} [{round}][{current_batch}/{total_batches}]\t" \ "DataLoad time {dataload_duration:.3f}\t" \ "F/W time {inference_duration:.3f}\t" \ "B/W time {backprop_duration:.3f}\t" "Loss {loss:.4f}\t" \ "Prec@1 {prec1:.3f}\t" "Prec@5 {prec5:.3f}\t".format( mode_str=mode_str, round=metrics_meter['round'], current_batch=i, total_batches=len(loader), dataload_duration=dataload_duration, inference_duration=inference_duration, backprop_duration=backprop_duration, loss=metrics_meter['loss'].get_avg(), prec1=metrics_meter['top_1_acc'].get_avg(), prec5=metrics_meter['top_5_acc'].get_avg()) logger.info(msg)
def neg_perplexity_from_loss(loss)
-
Expand source code
def neg_perplexity_from_loss(loss): return - np.exp(loss)
Classes
class AverageMeter
-
Computes and stores the average and current value
Expand source code
class AverageMeter(object): """Computes and stores the average and current value""" def __init__(self): self.reset() def reset(self): self.val = 0 self.avg = 0 self.sum = 0 self.count = 0 def update(self, val, n=1): self.val = val self.sum += val * n self.count += n self.avg = self.sum / self.count if self.count > 0 else None def get_val(self): return self.val def get_avg(self): return self.avg
Methods
def get_avg(self)
-
Expand source code
def get_avg(self): return self.avg
def get_val(self)
-
Expand source code
def get_val(self): return self.val
def reset(self)
-
Expand source code
def reset(self): self.val = 0 self.avg = 0 self.sum = 0 self.count = 0
def update(self, val, n=1)
-
Expand source code
def update(self, val, n=1): self.val = val self.sum += val * n self.count += n self.avg = self.sum / self.count if self.count > 0 else None