import argparse
import json
import math
import os
import pdb
import shutil
import time

import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim
import torch.utils.data
import torchvision.transforms as transforms
import torchvision.datasets as datasets
from tensorboardX import SummaryWriter
import resnet

try:
    from frob import FactorizedConv, frobdecay, frobenius_norm, non_orthogonality, patch_module
except ImportError:
    print("Failed to import factorization")

model_names = sorted(name for name in resnet.__dict__
    if name.islower() and not name.startswith("__")
                     and name.startswith("resnet")
                     and callable(resnet.__dict__[name]))

print(model_names)

parser = argparse.ArgumentParser(description='Propert ResNets for CIFAR10 in pytorch')
parser.add_argument('--arch', '-a', metavar='ARCH', default='resnet32',
                    choices=model_names,
                    help='model architecture: ' + ' | '.join(model_names) +
                    ' (default: resnet32)')
parser.add_argument('--data', default='cifar10', type=str)
parser.add_argument('--device', default=0, type=int)
parser.add_argument('-j', '--workers', default=4, type=int, metavar='N',
                    help='number of data loading workers (default: 4)')
parser.add_argument('--epochs', default=200, type=int, metavar='N',
                    help='number of total epochs to run')
parser.add_argument('--start-epoch', default=0, type=int, metavar='N',
                    help='manual epoch number (useful on restarts)')
parser.add_argument('-b', '--batch-size', default=128, type=int,
                    metavar='N', help='mini-batch size (default: 128)')
parser.add_argument('--lr', '--learning-rate', default=0.1, type=float,
                    metavar='LR', help='initial learning rate')
parser.add_argument('--momentum', default=0.9, type=float, metavar='M',
                    help='momentum')
parser.add_argument('--weight-decay', '--wd', default=1e-4, type=float,
                    metavar='W', help='weight decay (default: 1e-4)')
parser.add_argument('--print-freq', '-p', default=50, type=int,
                    metavar='N', help='print frequency (default: 50)')
parser.add_argument('--resume', default='', type=str, metavar='PATH',
                    help='path to latest checkpoint (default: none)')
parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true',
                    help='evaluate model on validation set')
parser.add_argument('--pretrained', dest='pretrained', action='store_true',
                    help='use pre-trained model')
parser.add_argument('--half', dest='half', action='store_true',
                    help='use half-precision(16-bit) ')
parser.add_argument('--save-dir', dest='save_dir',
                    help='The directory used to save the trained models',
                    default='save_temp', type=str)
parser.add_argument('--save-every', dest='save_every',
                    help='Saves checkpoints at every specified number of epochs',
                    type=int, default=10)
parser.add_argument('--rank-scale', default=0.0, type=float)
parser.add_argument('--square', action='store_true')
parser.add_argument('--residual', default=0.0, type=float)
parser.add_argument('--wd2fd', action='store_true')
parser.add_argument('--spectral', action='store_true')
parser.add_argument('--dump-frobnorms', action='store_true')
parser.add_argument('--normalize', default='', type=str)
parser.add_argument('--no-frob', action='store_true')
parser.add_argument('--seed', default=0, type=int)
best_prec1 = 0


def decomposed(model):

    for module in model.modules():
        if hasattr(module, 'frobgrad'):
            yield module


def main():
    global args, best_prec1
    args = parser.parse_args()
    torch.manual_seed(args.seed)

    # Check the save_dir exists or not
    if not os.path.exists(args.save_dir):
        os.makedirs(args.save_dir)

    #model = torch.nn.DataParallel(resnet.__dict__[args.arch]())
    model = resnet.__dict__[args.arch](num_classes=int(args.data[5:]))
    origpar = sum(param.numel() for param in model.parameters())
    print('Original weight count:', origpar)
    convs = ['.conv1', '.conv2']
    if args.rank_scale:
        modules, namelists = [], []
        for module in list(model.modules())[1:]:
            namelist = []
            for name in convs:
                if hasattr(module, name[1:]):
                    namelist.append(name[1:])
            if namelist:
                modules.append(module)
                namelists.append(namelist)
        for module, namelist in zip(modules, namelists):
            for name in namelist:
                patch_module(module, name, FactorizedConv,
                             rank_scale=args.rank_scale, square=args.square,
                             init='spectral' if args.spectral else lambda X: nn.init.kaiming_normal_(X),
                             square_init=lambda I: I+nn.init.normal_(I.clone(), 0.0, args.residual) if args.residual else I)
        no_decay = convs if args.wd2fd else []
        skiplist = [] if args.wd2fd else convs
        newpar = sum(param.numel() for param in model.parameters())
        print('Compressed weight count:', newpar)
        print('Compression ratio:', newpar / origpar)
    else:
        no_decay, skiplist = [], []
    torch.cuda.set_device(args.device)
    model.cuda()
    
    criterion = nn.CrossEntropyLoss().cuda()
    writer = SummaryWriter(args.save_dir)

    # optionally resume from a checkpoint
    if args.resume:
        if os.path.isfile(args.resume):
            print("=> loading checkpoint '{}'".format(args.resume))
            checkpoint = torch.load(args.resume)
            args.start_epoch = checkpoint['epoch']
            best_prec1 = checkpoint['best_prec1']
            model.load_state_dict(checkpoint['state_dict'])
            print("=> loaded checkpoint '{}' (epoch {})"
                  .format(args.evaluate, checkpoint['epoch']))
        else:
            print("=> no checkpoint found at '{}'".format(args.resume))

    cudnn.benchmark = True

    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                     std=[0.229, 0.224, 0.225])

    cifar = datasets.CIFAR100 if args.data == 'cifar100' else datasets.CIFAR10
    train_loader = torch.utils.data.DataLoader(
        cifar(root='./data', train=True, transform=transforms.Compose([
            transforms.RandomHorizontalFlip(),
            transforms.RandomCrop(32, 4),
            transforms.ToTensor(),
            normalize,
        ]), download=True),
        batch_size=args.batch_size, shuffle=True,
        num_workers=args.workers, pin_memory=True)

    val_loader = torch.utils.data.DataLoader(
        cifar(root='./data', train=False, transform=transforms.Compose([
            transforms.ToTensor(),
            normalize,
        ])),
        batch_size=128, shuffle=False,
        num_workers=args.workers, pin_memory=True)

    # define optimizer

    if args.half:
        model.half()
        criterion.half()

    optimizer_grouped_parameters = [
            {'params': [p for n, p in model.named_parameters()
                        if not any(nd in n for nd in no_decay)],
             'weight_decay': args.weight_decay},
            {'params': [p for n, p in model.named_parameters()
                        if any(nd in n for nd in no_decay)],
             'weight_decay': 0.0}
    ]

    optimizer = torch.optim.SGD(optimizer_grouped_parameters, args.lr,
                                momentum=args.momentum,
                                weight_decay=args.weight_decay)

    lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
                                                        milestones=[int(0.5*args.epochs), int(0.75*args.epochs)],
                                                        last_epoch=args.start_epoch - 1)

    if args.arch in ['resnet1202', 'resnet110']:
        # for resnet1202 original paper uses lr=0.01 for first 400 minibatches for warm-up
        # then switch back. In this setup it will correspond for first epoch.
        for param_group in optimizer.param_groups:
            param_group['lr'] = args.lr*0.1


    if args.evaluate:
        validate(val_loader, model, criterion)
        return

    with open(os.path.join(args.save_dir, 'args.json'), 'w') as f:
        json.dump(vars(args), f, indent=4)
    if args.dump_frobnorms:
        frobnorms = [[None for _ in range(len(train_loader))] for _ in range(args.epochs)]
    else:
        frobnorms = []
    if args.normalize:
        normalize = torch.load(args.normalize)
    else:
        normalize = None

    for epoch in range(args.start_epoch, args.epochs):

        if args.rank_scale:
            frobnorm, nucnorm, bound, nonorth = [], [], [], []
            for module in decomposed(model):
                U, VT = module.U.data, module.VT.data
                if args.square:
                    matrices = [U, module.M.data, VT]
                    VT = torch.matmul(module.M.data, VT)
                else:
                    matrices = [U, VT]
                    nonorth.append(sum(non_orthogonality(U, VT)) / 2.)
                frobnorm.append(frobenius_norm(U, VT))
                nucnorm.append(torch.norm(torch.matmul(U, VT), 'nuc'))
                bound.append(sum(matrix.pow(2).sum() for matrix in matrices) / 2.)
            writer.add_scalar('metric/NucNorm-bound', sum(bound) / len(bound), epoch)
            if nonorth:
                writer.add_scalar('metric/NonOrth', sum(nonorth) / len(nonorth), epoch)
        else:
            frobnorm, nucnorm = [], []
            for name, module in model.named_modules():
                if any(conv in name for conv in convs):
                    a, b, c, d = module.weight.shape
                    W = module.weight.data.reshape(a*c, b*d)
                    frobnorm.append(torch.norm(W))
                    nucnorm.append(torch.norm(W, 'nuc'))
        writer.add_scalar('hyper/lr-eff_frob', sum(optimizer.param_groups[0]['lr'] / fn for fn in frobnorm) / len(frobnorm), epoch)
        writer.add_scalar('hyper/lr-eff_frobsq', sum(optimizer.param_groups[0]['lr'] / (fn * fn) for fn in frobnorm) / len(frobnorm), epoch)
        writer.add_scalar('metric/FrobNorm', sum(frobnorm) / len(frobnorm), epoch)
        writer.add_scalar('metric/NucNorm', sum(nucnorm) / len(nucnorm), epoch)
        writer.add_scalar('hyper/lr', optimizer.param_groups[0]['lr'], epoch)

        # train for one epoch
        print('current lr {:.5e}'.format(optimizer.param_groups[0]['lr']))

        acc, loss = train(train_loader, model, criterion, optimizer, epoch,
                          coef=0. if args.no_frob else args.weight_decay, skiplist=skiplist,
                          frobnorms=frobnorms[epoch] if frobnorms else frobnorms, 
                          normalize=normalize if normalize is None else normalize[epoch])
        writer.add_scalar('train/acc', acc, epoch)
        writer.add_scalar('train/loss', loss, epoch)
        lr_scheduler.step()

        # evaluate on validation set
        prec1, loss = validate(val_loader, model, criterion)
        writer.add_scalar('valid/acc', prec1, epoch)
        writer.add_scalar('valid/loss', loss, epoch)

        # remember best prec@1 and save checkpoint
        is_best = prec1 > best_prec1
        best_prec1 = max(prec1, best_prec1)

        if epoch > 0 and epoch % args.save_every == 0:
            save_checkpoint({
                'epoch': epoch + 1,
                'state_dict': model.state_dict(),
                'best_prec1': best_prec1,
            }, is_best, filename=os.path.join(args.save_dir, 'checkpoint.th'))

        save_checkpoint({
            'state_dict': model.state_dict(),
            'best_prec1': best_prec1,
        }, is_best, filename=os.path.join(args.save_dir, 'model.th'))

    if args.dump_frobnorms:
        torch.save(torch.Tensor(frobnorms), os.path.join(args.save_dir, 'frobnorms.tensor'))
    writer.flush()
    with open(os.path.join(args.save_dir, 'results.json'), 'w') as f:
        json.dump({'final validation accuracy': prec1,
                   'best validation accuracy': best_prec1,
                   'original parameter count': origpar,
                   'compressed parameter count': newpar if args.rank_scale else origpar,
                   'compression ratio': newpar / origpar if args.rank_scale else 1.},
                   f, indent=4)



def train(train_loader, model, criterion, optimizer, epoch, frobnorms=[], normalize=None, **kwargs):
    """
        Run one train epoch
    """
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()

    # switch to train mode
    model.train()

    end = time.time()
    for i, (input, target) in enumerate(train_loader):

        # measure data loading time
        data_time.update(time.time() - end)

        target = target.cuda()
        input_var = input.cuda()
        target_var = target
        if args.half:
            input_var = input_var.half()

        # compute output
        output = model(input_var)
        loss = criterion(output, target_var)

        # compute gradient and do SGD step
        optimizer.zero_grad()
        loss.backward()
        frobdecay(model, **kwargs)
        optimizer.step()

        output = output.float()
        loss = loss.float()
        # measure accuracy and record loss
        prec1 = accuracy(output.data, target)[0]
        losses.update(loss.item(), input.size(0))
        top1.update(prec1.item(), input.size(0))

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        if i % args.print_freq == 0:
            print('Epoch: [{0}][{1}/{2}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
                      epoch, i, len(train_loader), batch_time=batch_time,
                      data_time=data_time, loss=losses, top1=top1))

        if frobnorms:
            frobnorms[i] = [frobenius_norm(module.U.data, module.VT.data) for module in decomposed(model)]
        if not normalize is None:
            for frobnorm, module in zip(normalize[i], decomposed(model)):
                factor = math.sqrt(frobnorm / frobenius_norm(module.U.data, module.VT.data))
                module.U.data *= factor
                module.VT.data *= factor

    return top1.avg, losses.avg


def validate(val_loader, model, criterion):
    """
    Run evaluation
    """
    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()

    # switch to evaluate mode
    model.eval()

    end = time.time()
    with torch.no_grad():
        for i, (input, target) in enumerate(val_loader):
            target = target.cuda()
            input_var = input.cuda()
            target_var = target.cuda()

            if args.half:
                input_var = input_var.half()

            # compute output
            output = model(input_var)
            loss = criterion(output, target_var)

            output = output.float()
            loss = loss.float()

            # measure accuracy and record loss
            prec1 = accuracy(output.data, target)[0]
            losses.update(loss.item(), input.size(0))
            top1.update(prec1.item(), input.size(0))

            # measure elapsed time
            batch_time.update(time.time() - end)
            end = time.time()

            if i % args.print_freq == 0:
                print('Test: [{0}/{1}]\t'
                      'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                      'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                      'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
                          i, len(val_loader), batch_time=batch_time, loss=losses,
                          top1=top1))

    print(' * Prec@1 {top1.avg:.3f}'
          .format(top1=top1))

    return top1.avg, losses.avg

def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'):
    """
    Save the training model
    """
    torch.save(state, filename)

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count


def accuracy(output, target, topk=(1,)):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0)
        res.append(correct_k.mul_(100.0 / batch_size))
    return res


if __name__ == '__main__':
    main()
