Spaces:
Runtime error
Runtime error
| # Copyright (C) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | |
| # | |
| # This work is made available under the Nvidia Source Code License-NC. | |
| # To view a copy of this license, check out LICENSE.md | |
| import os | |
| import time | |
| import math | |
| import subprocess | |
| import shutil | |
| from os.path import join | |
| import numpy as np | |
| from inspect import isclass | |
| from pytz import timezone | |
| from datetime import datetime | |
| import inspect | |
| import torch | |
| def datestr(): | |
| pacific = timezone('US/Pacific') | |
| now = datetime.now(pacific) | |
| return '{}{:02}{:02}_{:02}{:02}'.format( | |
| now.year, now.month, now.day, now.hour, now.minute) | |
| def module_to_dict(module, exclude=[]): | |
| return dict([(x, getattr(module, x)) for x in dir(module) | |
| if isclass(getattr(module, x)) | |
| and x not in exclude | |
| and getattr(module, x) not in exclude]) | |
| class TimerBlock: | |
| def __init__(self, title): | |
| print(("{}".format(title))) | |
| def __enter__(self): | |
| self.start = time.clock() | |
| return self | |
| def __exit__(self, exc_type, exc_value, traceback): | |
| self.end = time.clock() | |
| self.interval = self.end - self.start | |
| if exc_type is not None: | |
| self.log("Operation failed\n") | |
| else: | |
| self.log("Operation finished\n") | |
| def log(self, string): | |
| duration = time.clock() - self.start | |
| units = 's' | |
| if duration > 60: | |
| duration = duration / 60. | |
| units = 'm' | |
| print((" [{:.3f}{}] {}".format(duration, units, string))) | |
| def log2file(self, fid, string): | |
| fid = open(fid, 'a') | |
| fid.write("%s\n" % (string)) | |
| fid.close() | |
| def add_arguments_for_module( | |
| parser, | |
| module, | |
| argument_for_class, | |
| default, | |
| skip_params=[], | |
| parameter_defaults={}): | |
| argument_group = parser.add_argument_group(argument_for_class.capitalize()) | |
| module_dict = module_to_dict(module) | |
| argument_group.add_argument( | |
| '--' + argument_for_class, | |
| type=str, | |
| default=default, | |
| choices=list( | |
| module_dict.keys())) | |
| args, unknown_args = parser.parse_known_args() | |
| class_obj = module_dict[vars(args)[argument_for_class]] | |
| argspec = inspect.getargspec(class_obj.__init__) | |
| defaults = argspec.defaults[::-1] if argspec.defaults else None | |
| args = argspec.args[::-1] | |
| for i, arg in enumerate(args): | |
| cmd_arg = '{}_{}'.format(argument_for_class, arg) | |
| if arg not in skip_params + ['self', 'args']: | |
| if arg in list(parameter_defaults.keys()): | |
| argument_group.add_argument( | |
| '--{}'.format(cmd_arg), | |
| type=type( | |
| parameter_defaults[arg]), | |
| default=parameter_defaults[arg]) | |
| elif (defaults is not None and i < len(defaults)): | |
| argument_group.add_argument( | |
| '--{}'.format(cmd_arg), | |
| type=type( | |
| defaults[i]), | |
| default=defaults[i]) | |
| else: | |
| print(("[Warning]: non-default argument '{}' " | |
| "detected on class '{}'. This argument " | |
| "cannot be modified via the command line" | |
| .format(arg, module.__class__.__name__))) | |
| # We don't have a good way of dealing with | |
| # inferring the type of the argument | |
| # TODO: try creating a custom action and using ast's infer type? | |
| # else: | |
| # argument_group.add_argument('--{}'.format( | |
| # cmd_arg), required=True) | |
| def kwargs_from_args(args, argument_for_class): | |
| argument_for_class = argument_for_class + '_' | |
| return {key[len(argument_for_class):]: value for key, value in list(vars( | |
| args).items()) if | |
| argument_for_class in key and key != argument_for_class + 'class'} | |
| def format_dictionary_of_losses(labels, values): | |
| try: | |
| string = ', '.join([('{}: {:' + | |
| ('.3f' if value >= 0.001 else '.1e') + | |
| '}').format(name, value) for name, value in | |
| zip(labels, values)]) | |
| except (TypeError, ValueError) as e: | |
| print((list(zip(labels, values)))) | |
| string = '[Log Error] ' + str(e) | |
| return string | |
| class IteratorTimer(): | |
| def __init__(self, iterable): | |
| self.iterable = iterable | |
| self.iterator = self.iterable.__iter__() | |
| def __iter__(self): | |
| return self | |
| def __len__(self): | |
| return len(self.iterable) | |
| def __next__(self): | |
| start = time.time() | |
| n = next(self.iterator) | |
| self.last_duration = (time.time() - start) | |
| return n | |
| next = __next__ | |
| def gpumemusage(): | |
| gpu_mem = subprocess.check_output( | |
| "nvidia-smi | grep MiB | cut -f 3 -d '|'", | |
| shell=True).replace( | |
| ' ', | |
| '').replace( | |
| '\n', | |
| '').replace( | |
| 'i', | |
| '') | |
| all_stat = [float(a) for a in gpu_mem.replace('/', '').split('MB')[:-1]] | |
| gpu_mem = '' | |
| for i in range(len(all_stat) / 2): | |
| curr, tot = all_stat[2 * i], all_stat[2 * i + 1] | |
| util = "%1.2f" % (100 * curr / tot) + '%' | |
| cmem = str(int(math.ceil(curr / 1024.))) + 'GB' | |
| gmem = str(int(math.ceil(tot / 1024.))) + 'GB' | |
| gpu_mem += util + '--' + join(cmem, gmem) + ' ' | |
| return gpu_mem | |
| def update_hyperparameter_schedule(args, epoch, global_iteration, optimizer): | |
| if args.schedule_lr_frequency > 0: | |
| for param_group in optimizer.param_groups: | |
| if (global_iteration + 1) % args.schedule_lr_frequency == 0: | |
| param_group['lr'] /= float(args.schedule_lr_fraction) | |
| param_group['lr'] = float( | |
| np.maximum(param_group['lr'], 0.000001)) | |
| def save_checkpoint(state, is_best, path, prefix, | |
| filename='checkpoint.pth.tar'): | |
| prefix_save = os.path.join(path, prefix) | |
| name = prefix_save + '_' + filename | |
| torch.save(state, name) | |
| if is_best: | |
| shutil.copyfile(name, prefix_save + '_model_best.pth.tar') | |