@misc{
retsinas2022trainable,
title={Trainable Learning Rate},
author={George Retsinas and Giorgos Sfikas and Panagiotis Filntisis and Petros Maragos},
year={2022},
url={https://openreview.net/forum?id=fHeK814NOMO}
}

@inproceedings{
wu2018understanding,
title={Understanding Short-Horizon Bias in Stochastic Meta-Optimization},
author={Yuhuai Wu and Mengye Ren and Renjie Liao and Roger Grosse.},
booktitle={International Conference on Learning Representations},
year={2018},
url={https://openreview.net/forum?id=H1MczcgR-},
}

@article{marcus-etal-1993-building,
    title = "Building a Large Annotated Corpus of {E}nglish: The {P}enn {T}reebank",
    author = "Marcus, Mitchell P.  and
      Santorini, Beatrice  and
      Marcinkiewicz, Mary Ann",
    journal = "Computational Linguistics",
    volume = "19",
    number = "2",
    year = "1993",
    url = "https://aclanthology.org/J93-2004",
    pages = "313--330",
}


@Article{HochSchm97,
  author      = {Sepp Hochreiter and Jürgen Schmidhuber},
  journal     = {Neural Computation},
  title       = {Long Short-Term Memory},
  year        = {1997},
  number      = {8},
  pages       = {1735--1780},
  volume      = {9},
  optdoi      = {10.1162/neco.1997.9.8.1735},
  opteprint   = {http://dx.doi.org/10.1162/neco.1997.9.8.1735},
  opturl      = {http://dx.doi.org/10.1162/neco.1997.9.8.1735},
}

@inproceedings{zhuang2020adabelief,
  title={AdaBelief Optimizer: Adapting Stepsizes by the Belief in Observed Gradients},
  author={Zhuang, J. and Tang, T. and Ding, Y. and Tatikonda, S. and Dvornek, N. and Papademetris, X. and Duncan, J.},
  booktitle={NeurIPS},
  year={2020}
}

@inproceedings{imagenet_cvpr09,
        AUTHOR = {Deng, J. and Dong, W. and Socher, R. and Li, L.-J. and Li, K. and Fei-Fei, L.},
        TITLE = {{ImageNet: A Large-Scale Hierarchical Image Database}},
        BOOKTITLE = {CVPR},
        YEAR = {2009},
}

@INPROCEEDINGS{Zagoruyko2016WRN,
    author = {S. Zagoruyko and N. Komodakis},
    title = {Wide Residual Networks},
    booktitle = {BMVC},
    year = {2016}}
    
@InProceedings{He2016DeepRL,
author = {He, K. and Zhang, X. and Ren, S. and Sun, J.},
title = {Deep Residual Learning for Image Recognition},
booktitle = {CVPR},
year = {2016}
}

@misc{cifar,
  title={Cifar-10 and cifar-100 datasets},
  author={Krizhevsky, A. and Nair, V. and Hinton, G.},
  howpublished={\url{https://www. cs. toronto. edu/kriz/cifar. html}},
  year={2009}
}

@InProceedings{falkner2018,
  title = 	 {{BOHB}: Robust and Efficient Hyperparameter Optimization at Scale},
  author =       {Falkner, S. and Klein, A. and Hutter, F.},
  booktitle = 	 {ICML},
  year = 	 {2018},
}

@inproceedings{zela-automl18,
  author    = {A. Zela and
               A. Klein and
               S. Falkner and 
               F. Hutter},
  title     = {Towards Automated Deep Learning: Efficient Joint Neural Architecture and Hyperparameter Search},
  booktitle = {ICML AutoML Workshop},
  year      = {2018},
}

@inproceedings{hutter2011,
  author    = {F. Hutter and
               H. H. Hoos and
               K. Leyton{-}Brown},
  title     = {Sequential Model-Based Optimization for General Algorithm Configuration},
  booktitle = {LION},
  year      = {2011},
}

@inproceedings{bergstra2011,
 author = {Bergstra, J. and Bardenet, R. and Bengio, Y. and K\'{e}gl, B.},
 booktitle = {NeurIPS},
 title = {Algorithms for Hyper-Parameter Optimization},
 year = {2011}
}

@inproceedings{li2020system,
  author = {Li, L. and Jamieson, K. and Rostamizadeh, A. and Gonina, E. and Ben-tzur, J. and Hardt, M. and Recht, B. and Talwalkar, A.},
  booktitle = {Proceedings of Machine Learning and Systems},
  title = {A System for Massively Parallel Hyperparameter Tuning},
  year = {2020}
}

@article{li2018hyperband,
  author  = {L. Li and K. Jamieson and G. DeSalvo and A. Rostamizadeh and A. Talwalkar},
  title   = {Hyperband: A Novel Bandit-Based Approach to Hyperparameter Optimization},
  journal = {Journal of Machine Learning Research},
  year    = {2018},
  volume  = {18},
  number  = {185},
  pages   = {1-52},
}

@article{jaderberg2017population,
  author    = {M. Jaderberg and
               V. Dalibard and
               S. Osindero and
               W. M. Czarnecki and
               J. Donahue and
               A. Razavi and
               O. Vinyals and
               T. Green and
               I. Dunning and
               K. Simonyan and
               C. Fernando and
               K. Kavukcuoglu},
  title     = {Population Based Training of Neural Networks},
  journal   = {CoRR},
  volume    = {abs/1711.09846},
  year      = {2017},
}

@inproceedings{li2020population,
author = {Li, A. and Spyra, O. and Perel, S. and Dalibard, V. and Jaderberg, M. and Gu, C. and Budden, D. and Harley, T. and Gupta, P.},
title = {A Generalized Framework for Population Based Training},
year = {2019},
booktitle = {ACM SIGKDD}
}

@inproceedings{holder2020prov,
  author    = {J. Parker{-}Holder and
               V. Nguyen and
               S. J. Roberts},
  title     = {Provably Efficient Online Hyperparameter Optimization with Population-Based
               Bandits},
  booktitle = {NeurIPS},
  year      = {2020},
}

@inproceedings{kingma2015,
  author={D. P. Kingma and J. Ba},
  title={Adam: A Method for Stochastic Optimization},
  year={2015},
  booktitle={ICLR (Poster)},
}

@article{tieleman2012lecture,
  title={Lecture 6.5-rmsprop: Divide the gradient by a running average of its recent magnitude},
  author={Tieleman, T. and Hinton, G. and others},
  journal={COURSERA: Neural networks for machine learning},
  volume={4},
  number={2},
  pages={26--31},
  year={2012}
}

@article{Polyak1992AccelerationOS,
title={Acceleration of stochastic approximation by averaging},
author={B. T. Polyak and A. B. Juditsky},
journal={Siam Journal on Control and Optimization},
year={1992},
volume={30},
pages={838-855}
}

@article{zeiler2012adadelta,
  title={Adadelta: an adaptive learning rate method},
  author={Zeiler, M. D},
  journal={CoRR},
  volume = {abs/1212.5701},
  year={2012},
}

@article{duchi2011,
  author  = {J. Duchi and E. Hazan and Y. Singer},
  title   = {Adaptive Subgradient Methods for Online Learning and Stochastic Optimization},
  journal = {Journal of Machine Learning Research},
  year    = {2011},
  volume  = {12},
  number  = {61},
  pages   = {2121-2159},
}

@article{yang2019228,
title = {Mini-batch algorithms with online step size},
journal = {Knowledge-Based Systems},
volume = {165},
pages = {228-240},
year = {2019},
author = {Z. Yang and C. Wang and Z. Zhang and J. Li},
}

@InProceedings{luca2017hyper,
  title = 	 {Forward and Reverse Gradient-Based Hyperparameter Optimization},
  author =       {L. Franceschi and M. Donini and P. Frasconi and M. Pontil},
  booktitle = 	 {ICML},
  year = 	 {2017},
}


@inproceedings{gunes2018online,
  title={Online Learning Rate Adaptation with Hypergradient Descent},
  author={A. G. Baydin and R. Cornish and D. Martinez Rubio and M. Schmidt and F. Wood},
  booktitle={ICLR},
  year={2018},
}

@inproceedings{donini2020marthe,
  title     = {Marthe: Scheduling the Learning Rate Via Online Hypergradients},
  author    = {Donini, M. and Franceschi, L. and Majumder, O. and Pontil, M. and Frasconi, P.},
  booktitle = {IJCAI},
  year      = {2020},
}

@article{smith2018super,
author = {L. N. Smith and
N. Topin},
title = {Super-Convergence: Very Fast Training of Residual Networks Using Large
Learning Rates},
journal = {CoRR},
volume = {abs/1708.07120},
year = {2017},
}

@article{smith2018disciplined,
  title={A disciplined approach to neural network hyper-parameters: Part 1--learning rate, batch size, momentum, and weight decay},
  author={Smith, L. N},
  journal={CoRR},
  volume = {abs/1803.09820},
  year={2018}
}

@inproceedings{carvalho2020autolr,
  title={Autolr: An evolutionary approach to learning rate policies},
  author={Carvalho, Pedro and Louren{\c{c}}o, Nuno and Assun{\c{c}}{\~a}o, Filipe and Machado, Penousal},
  booktitle={Proceedings of the 2020 Genetic and Evolutionary Computation Conference},
  pages={672--680},
  year={2020}
}

@inproceedings{smith2017cyclical,
  title={Cyclical learning rates for training neural networks},
  author={Smith, L. N},
  booktitle={WACV},
  year={2017},
}


@inproceedings{loshchilov2016sgdr,
  author    = {I. Loshchilov and
               F. Hutter},
  title     = {{SGDR:} Stochastic Gradient Descent with Warm Restarts},
  booktitle = {ICLR},
  year      = {2017},
}

@article{baydin2017online,
  title={Online learning rate adaptation with hypergradient descent},
  author={Baydin, Atilim Gunes and Cornish, Robert and Rubio, David Martinez and Schmidt, Mark and Wood, Frank},
  journal={arXiv preprint arXiv:1703.04782},
  year={2017}
}

@inproceedings{jin2021autolrs,
  title={AutoLRS: Automatic Learning-Rate Schedule by Bayesian Optimization on the Fly},
  author={Jin, Y. and Zhou, T. and Zhao, L. and Zhu, Y. and Guo, C. and Canini, M. and Krishnamurthy, A.},
  booktitle={ICLR},
  year={2020}
}

@incollection{Bengio+chapter2007,
author = {Bengio, Yoshua and LeCun, Yann},
booktitle = {Large Scale Kernel Machines},
publisher = {MIT Press},
title = {Scaling Learning Algorithms Towards {AI}},
year = {2007}
}

@article{Hinton06,
author = {Hinton, Geoffrey E. and Osindero, Simon and Teh, Yee Whye},
journal = {Neural Computation},
pages = {1527--1554},
title = {A Fast Learning Algorithm for Deep Belief Nets},
volume = {18},
year = {2006}
}

@book{Goodfellow-et-al-2016,
    title={Deep Learning},
    author={Ian Goodfellow and Yoshua Bengio and Aaron Courville},
    note={\url{http://www.deeplearningbook.org}},
    publisher={MIT Press},
    address	= {Cambridge, MA},
    year={2016}
}

% Entropy-SGD
@article{chaudhari2016entropy,
  title={Entropy-sgd: Biasing gradient descent into wide valleys},
  author={Chaudhari, Pratik and Choromanska, Anna and Soatto, Stefano and LeCun, Yann and Baldassi, Carlo and Borgs, Christian and Chayes, Jennifer and Sagun, Levent and Zecchina, Riccardo},
  journal={arXiv preprint arXiv:1611.01838},
  year={2016}
}
% Limit-Cycle
@inproceedings{chaudhari2018stochastic,
  title={Stochastic gradient descent performs variational inference, converges to limit cycles for deep networks},
  author={Chaudhari, Pratik and Soatto, Stefano},
  booktitle={2018 Information Theory and Applications Workshop (ITA)},
  pages={1--10},
  year={2018},
  organization={IEEE}
}
% Online PCA
@inproceedings{shamir2016convergence,
  title={Convergence of stochastic gradient descent for PCA},
  author={Shamir, Ohad},
  booktitle={International Conference on Machine Learning},
  pages={257--265},
  year={2016}
}
% Convergence analysis (SGD momentum)
@article{yang2016unified,
  title={Unified convergence analysis of stochastic momentum methods for convex and non-convex optimization},
  author={Yang, T. and Lin, Q. and Li, Z.},
  journal={CoRR},
  volume = {abs/1604.03257},
  year={2016},
}
% Model selection

@article{chen2016deeplab,
  title={Deeplab: Semantic image segmentation with deep convolutional nets, atrous convolution, and fully connected crfs},
  author={Chen, L.-C. and Papandreou, G. and Kokkinos, I. and Murphy, K. and Yuille, A. L.},
  journal   = {CoRR},
  volume    = {abs/1606.00915},
  year={2016}
}

@incollection{NIPS2014_5423,
title = {Generative Adversarial Nets},
author = {Goodfellow, I. and Pouget-Abadie, J. and Mirza, M. and Xu, B. and Warde-Farley, D. and Ozair, S. and Courville, A. and Bengio, Y.},
booktitle = {NIPS},
year = {2014},
}

@article{DBLP:journals/corr/ZhaoML16,
  author    = {J. J. Zhao and
               M. Mathieu and
               Y. LeCun},
  title     = {Energy-based Generative Adversarial Network},
  booktitle = {ICLR},
  year      = {2017},
}

@article{DBLP:journals/corr/abs-1810-04805,
  author    = {J. Devlin and
               M.{-}W. Chang and
               K. Lee and
               K. Toutanova},
  title     = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language
               Understanding},
  journal   = {CoRR},
  volume    = {abs/1810.04805},
  year      = {2018},
  url       = {http://arxiv.org/abs/1810.04805},
  archivePrefix = {arXiv},
  eprint    = {1810.04805},
  timestamp = {Tue, 30 Oct 2018 20:39:56 +0100},
  biburl    = {https://dblp.org/rec/bib/journals/corr/abs-1810-04805},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@ARTICLE{2018arXiv180609077C,
       author = {{Choromanska}, A. and {Cowen}, B. and {Kumaravel}, S. and
         {Luss}, R. and {Rigotti}, M. and {Rish}, I. and
         {Kingsbury}, B. and {DiAchille}, P. and {Gurev}, V. and
         {Tejwani}, R. and {Bouneffouf}, D.},
        title = "{Beyond Backprop: Online Alternating Minimization with Auxiliary Variables}",
      journal = {CoRR},
      volume = {abs/1806.09077},
         year = "2019",
}

@INPROCEEDINGS{Deng09imagenet:a,
    author = {J. Deng and W. Dong and R. Socher and L.-j. Li and K. Li and L. Fei-fei},
    title = {Imagenet: A large-scale hierarchical image database},
    booktitle = {CVPR},
    year = {2009}
}

@INPROCEEDINGS{Fang2019,
author = {S. Fang and \textbf{A. Choromanska}},
title = {Reconfigurable Network for Efficient Inferencing in Autonomous Vehicles},
booktitle = {ICRA},
year = {2019},
}

@INPROCEEDINGS{Fang2020,
author = {S. Fang and \textbf{A. Choromanska}},
title = {Multi-modal Experts Networks for Autonomous Driving},
booktitle = {ICRA},
year = {2020},
}

@article{DBLP:journals/corr/abs-1802-06875,
  author    = {B. Cowen and
               A. Nandini Saridena and
               \textbf{A. Choromanska}},
  title     = {{LSALSA:} efficient sparse coding in single and multiple dictionary
               settings},
  journal   = {CoRR},
  volume    = {abs/1802.06875},
  year      = {2018}
}

@INPROCEEDINGS{DBLP:journals/corr/BojarskiCCFJMZ16,
  author    = {M. Bojarski and
               \textbf{A. Choromanska} and
               K. Choromanski and
               B. Firner and
               L. D. Jackel and
               U. Muller and
               K. Zieba},
  title     = {VisualBackProp: efficient visualization of CNNs for autonomous driving},
booktitle = {ICRA},
year = {2018},
}

@article{Lin2014ARXIV,
    author = {Lin, T.-Y. and Maire, M. and Belongie, S. and Hays, J. and Perona, P. and Ramanan, D. and Doll\'{a}r, P. and Zitnick, C. L.},
    title = {Microsoft {COCO:} Common Objects in Context},
    url = {http://arxiv.org/abs/1405.0312},
    journal   = {CoRR},
    volume = {abs/1405.0312},
    year = {2014}
}

@Article{Everingham10, 
   author = "Everingham, M. and Van~Gool, L. and Williams, C. K. I. and Winn, J. and Zisserman, A.", 
   title = "The Pascal Visual Object Classes (VOC) Challenge", 
   journal = "International Journal of Computer Vision", 
   volume = "88", 
   year = "2010", 
   number = "2", 
   month = jun, 
   pages = "303--338", 
} 

@inproceedings{conf/cvpr/XiaoHEOT10,
  author = {Xiao, J. and Hays, J. and Ehinger, K. A. and Oliva, A. and Torralba, A.},
  booktitle = {CVPR},
  title = {SUN database: Large-scale scene recognition from abbey to zoo.},
  url = {http://dblp.uni-trier.de/db/conf/cvpr/cvpr2010.html#XiaoHEOT10},
  year = 2010
}

@inproceedings{DBLP:journals/corr/HeZR016,
  author    = {K. He and
               X. Zhang and
               S. Ren and
               J. Sun},
  title     = {Identity Mappings in Deep Residual Networks},
  booktitle = {ECCV},
  year      = {2016},
}

@inproceedings{DBLP:journals/corr/abs-1709-05011,
 author = {You, Y. and Zhang, Z. and Hsieh, C.-J. and Demmel, J. and Keutzer, K.},
 title = {ImageNet Training in Minutes},
 booktitle = {ICPP},
 year = {2018},
} 

@inproceedings{chiang2013determinism,
  title={Determinism and reproducibility in large-scale HPC systems},
  author={Chiang, W.-F. and Gopalakrishnan, G. and Rakamaric, Z. and Ahn, D. H. and Lee, G. L.},
  booktitle={Workshop on Determinism and Correctness in Parallel Programming (WoDet)},
  year={2013}
}

@inproceedings{NIPS2012_4824,
title = {ImageNet Classification with Deep Convolutional Neural Networks},
author = {A. Krizhevsky and Sutskever, I. and G. E. Hinton},
booktitle = {NIPS},
year = {2012},
}

@inproceedings{DBLP:conf/icassp/Abdel-HamidMJP12,
  author    = {O. Abdel{-}Hamid and
               A.{-}r. Mohamed and
               H. Jiang and
               G. Penn},
  title     = {Applying Convolutional Neural Networks concepts to hybrid {NN-HMM}
               model for speech recognition},
  booktitle = {ICASSP},
  year      = {2012},
}

@InProceedings{pmlr-v80-liang18a,
  title = 	 {Understanding the Loss Surface of Neural Networks for Binary Classification},
  author = 	 {Liang, S. and Sun, R. and Li, Y. and Srikant, R.},
  booktitle = 	 {ICML},
  year = 	 {2018},
}

@inproceedings{DBLP:conf/aistats/ChoromanskaHMAL15,
  author    = {\textbf{A. Choromanska} and
               M. Henaff and
               M. Mathieu and
               G. {Ben Arous} and
               Y. LeCun},
  title     = {The Loss Surfaces of Multilayer Networks},
  booktitle = {AISTATS},
  year      = {2015},
}

@article{DBLP:journals/corr/BadrinarayananM15,
  author    = {V. Badrinarayanan and
               B. Mishra and
               R. Cipolla},
  title     = {Understanding symmetries in deep networks},
  journal   = {CoRR},
  volume    = {abs/1511.01029},
  year      = {2015},
  url       = {http://arxiv.org/abs/1511.01029},
  archivePrefix = {arXiv},
  eprint    = {1511.01029},
  timestamp = {Mon, 13 Aug 2018 16:46:53 +0200},
  biburl    = {https://dblp.org/rec/bib/journals/corr/BadrinarayananM15},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{ge2016matrix,
  title={Matrix completion has no spurious local minimum},
  author={Ge, R. and Lee, J. D. and Ma, T.},
  booktitle={NIPS},
  year={2016}
}

@article{DBLP:journals/tit/SunQW17,
  author    = {J. Sun and
               Q. Qu and
               J. Wright},
  title     = {Complete Dictionary Recovery Over the Sphere {I:} Overview and the
               Geometric Picture},
  journal   = {{IEEE} Trans. Information Theory},
  volume    = {63},
  number    = {2},
  pages     = {853--884},
  year      = {2017},
  url       = {https://doi.org/10.1109/TIT.2016.2632162},
  doi       = {10.1109/TIT.2016.2632162},
  timestamp = {Mon, 11 Mar 2019 00:00:00 +0100},
  biburl    = {https://dblp.org/rec/bib/journals/tit/SunQW17},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/focm/SunQW18,
  author    = {J. Sun and
               Q. Qu and
               J. Wright},
  title     = {A Geometric Analysis of Phase Retrieval},
  journal   = {Foundations of Computational Mathematics},
  volume    = {18},
  number    = {5},
  pages     = {1131--1198},
  year      = {2018},
  url       = {https://doi.org/10.1007/s10208-017-9365-9},
  doi       = {10.1007/s10208-017-9365-9},
  timestamp = {Mon, 11 Mar 2019 00:00:00 +0100},
  biburl    = {https://dblp.org/rec/bib/journals/focm/SunQW18},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{ge2017no,
  title={No spurious local minima in nonconvex low rank problems: A unified geometric analysis},
  author={Ge, R. and Jin, C. and Zheng, Y.},
  booktitle={ICML},
  year={2017},
}

@InProceedings{Simonyan15,
  author       = "Simonyan, K. and Zisserman, A.",
  title        = "Very Deep Convolutional Networks for Large-Scale Image Recognition",
  booktitle    = "ICLR",
  year         = "2015",
}

@article{DBLP:journals/corr/WickramasingheL16,
  author    = {U. Wickramasinghe and
               A. Lumsdaine},
  title     = {A Survey of Methods for Collective Communication Optimization and
               Tuning},
  journal   = {CoRR},
  volume    = {abs/1611.06334},
  year      = {2016},
  url       = {http://arxiv.org/abs/1611.06334},
  archivePrefix = {arXiv},
  eprint    = {1611.06334},
  timestamp = {Mon, 13 Aug 2018 16:47:59 +0200},
  biburl    = {https://dblp.org/rec/bib/journals/corr/WickramasingheL16},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@INPROCEEDINGS{488968, 
author={J. {Kennedy} and R. {Eberhart}}, 
title={Particle swarm optimization},
booktitle={ICNN}, 
year={1995}, 
}

@InProceedings{pmlr-v80-lian18a,
  title = 	 {Asynchronous Decentralized Parallel Stochastic Gradient Descent},
  author = 	 {Lian, X. and Zhang, W. and Zhang, C. and Liu, J.},
  booktitle = 	 {ICML},
  year = 	 {2018},
}


@article{LARS,
  author    = {Y. You and
               I. Gitman and
               B. Ginsburg},
  title     = {Scaling {SGD} Batch Size to 32K for ImageNet Training},
  journal   = {CoRR},
  volume    = {abs/1708.03888},
  year      = {2017},
}

@inproceedings{DBLP:journals/corr/ChaudhariBZST17,
  author    = {P. Chaudhari and
               C. Baldassi and
               R. Zecchina and
               S. Soatto and
               A. Talwalkar},
  title     = {Parle: parallelizing stochastic gradient descent},
  booktitle = {SysML},
  year      = {2018},
}

@article{LLAHLWZ2019IEEETRINFTH,
author = {Li, X. and Lu, J. and Arora, R. and Haupt, J. and Liu, H. and Wang, Z. and Zhao, T.},
year = {2019},
month = {03},
pages = {1-1},
title = {Symmetry, Saddle Points, and Global Optimization Landscape of Nonconvex Matrix Factorization},
volume = {PP},
journal = {IEEE Transactions on Information Theory},
doi = {10.1109/TIT.2019.2898663}
}

@article{polyak1992acceleration,
  title={Acceleration of stochastic approximation by averaging},
  author={Polyak, B. T. and Juditsky, A. B.},
  journal={SIAM Journal on Control and Optimization},
  volume={30},
  number={4},
  pages={838--855},
  year={1992},
}

@article{sergeev2018horovod,
  Author = {A. Sergeev and M. Del Balso},
  journal   = {CoRR},
  volume    = {abs/1802.05799},
  Title = {Horovod: fast and easy distributed deep learning in {TensorFlow}},
  Year = {2018}
}

@incollection{bottou-98x,
  author = {Bottou, L.},
  title = {Online Algorithms and Stochastic Approximations},
  booktitle = {Online Learning and Neural Networks},
  publisher={Cambridge University Press},
  address={Cambridge, MA},
  year = {1998},
}

@inproceedings{DBLP:conf/emnlp/WestonCA14,
  author    = {J. Weston and
               S. Chopra and
               K. Adams},
  title     = {{\#}TagSpace: Semantic Embeddings from Hashtags},
  booktitle = {EMNLP},
  year      = {2014},
}

@inproceedings{EASGD,
 author = {Zhang, S. and \textbf{A. Choromanska} and LeCun, Y.},
 title = {Deep Learning with Elastic Averaging {SGD}},
 booktitle = {NIPS},
 year = {2015},
}

@inproceedings{DOWNPOUR,
  title={Large scale distributed deep networks},
  author={Dean, J. and Corrado, G. and Monga, R. and Chen, K. and Devin, M. and Mao, M. and Senior, A. and Tucker, P. and Yang, K. and Le, Q. V. and others},
  booktitle={NIPS},
  year={2012}
}

@article{PARLE,
  title={Parle: parallelizing stochastic gradient descent},
  author={Chaudhari, Pratik and Baldassi, Carlo and Zecchina, Riccardo and Soatto, Stefano and Talwalkar, Ameet and Oberman, Adam},
  journal={arXiv preprint arXiv:1707.00424},
  year={2017}
}

@misc{CIFAR10,
  title={The CIFAR-10 dataset},
  author={Krizhevsky, A. and Nair, V. and Hinton, G.},
  howpublished={\url{https://www.cs.toronto.edu/~kriz/cifar.html}},
  year={2014}
}

@article{CIFAR100, 
title= {CIFAR-100 (Canadian Institute for Advanced Research)},
journal= {}, 
author= {Alex Krizhevsky and Vinod Nair and Geoffrey Hinton}
}

@article{MNIST,
  title={Backpropagation applied to handwritten zip code recognition},
  author={LeCun, Y. and Boser, B. and Denker, J. S. and Henderson, D. and Howard, R. E. and Hubbard, W. and Jackel, L. D.},
  journal={Neural computation},
  volume={1},
  number={4},
  pages={541--551},
  year={1989},
  publisher={MIT Press}
}

@article{, title= {CIFAR-100 (Canadian Institute for Advanced Research)}, journal= {}, author= {Alex Krizhevsky and Vinod Nair and Geoffrey Hinton}},

@article{hochreiter1997long,
 author = {Hochreiter, S. and Schmidhuber, J.},
 journal = {Neural computation},
 number = {8},
 pages = {1735--1780},
 publisher = {MIT Press},
 title = {Long short-term memory},
 volume = {9},
 year = {1997},
}

@article{li2019symmetry,
  title={Symmetry, saddle points, and global optimization landscape of nonconvex matrix factorization},
  author={Li, Xingguo and Lu, Junwei and Arora, Raman and Haupt, Jarvis and Liu, Han and Wang, Zhaoran and Zhao, Tuo},
  journal={IEEE Transactions on Information Theory},
  year={2019},
  publisher={IEEE}
}

@article{kendall1938measure,
  author = {Kendall, M. G.},
  journal = {Biometrika},
  number = {1/2},
  pages = {81--93},
  title = {A New Measure of Rank Correlation},
  volume = 30,
  year = 1938
}

@Incollection{ref2,
author = {Dodge, Y.},
title="Spearman Rank Correlation Coefficient",
bookTitle="The Concise Encyclopedia of Statistics",
year="2008",
publisher="Springer New York",
pages="502--505",
isbn="978-0-387-32833-1",
doi="10.1007/978-0-387-32833-1_379",
url="https://doi.org/10.1007/978-0-387-32833-1_379"
}

@book{Kullback59,
  author = {Kullback, S.},
  booktitle = {Information Theory and Statistics},
  publisher = {Wiley},
  title = {Information Theory and Statistics},
  year = 1959
}

@incollection{ref1,
author = "Kirch (editor), W.",
title="Pearson's Correlation Coefficient",
booktitle="Encyclopedia of Public Health",
year="2008",
publisher="Springer Netherlands",
pages="1090--1091",
}

@article{rajpurkar2016squad,
 added-at = {2018-02-07T14:37:11.000+0100},
 author = {Rajpurkar, P. and Zhang, J. and Lopyrev, K. and Liang, P.},
 journal   = {CoRR},
 volume    = {1606.05250},
 year = 2016,
}

@article{zellers2018swag,
title={SWAG: A Large-Scale Adversarial Dataset for Grounded Commonsense Inference},
author={Zellers, R. and Bisk, Y. and Schwartz, R. and Choi, Y.},
journal={EMNLP},
year={2018}
}

@inproceedings{wang-etal-2018-glue,
   title = "{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding",
   author = "Wang, A.  and
     Singh, A.  and
     Michael, J.  and
     Hill, F.  and
     Levy, O.  and
     Bowman, S.",
   booktitle = "EMNLP Workshop {B}lackbox{NLP}: Analyzing and Interpreting Neural Networks for {NLP}",
   year = "2018",
}

@article{sun2018geometric,
  title={A geometric analysis of phase retrieval},
  author={Sun, Ju and Qu, Qing and Wright, John},
  journal={Foundations of Computational Mathematics},
  volume={18},
  number={5},
  pages={1131--1198},
  year={2018},
  publisher={Springer}
}

@article{sun2017complete,
  title={Complete dictionary recovery over the sphere I: Overview and the geometric picture},
  author={Sun, Ju and Qu, Qing and Wright, John},
  journal={IEEE Transactions on Information Theory},
  volume={63},
  number={2},
  pages={853--884},
  year={2017},
  publisher={IEEE}
}

@article{BCN2018SIAMREV,
title={Optimization Methods for Large-Scale Machine Learning},
author={L. Bottou and F. E. Curtis and J. Nocedal},
journal={SIAM Review},
volume={60},
number={2},
year={2018},
pages={223--311}}

@InProceedings{VGG,
  author       = "Simonyan, K. and Zisserman, A.",
  title        = "Very Deep Convolutional Networks for Large-Scale Image Recognition",
  booktitle    = "International Conference on Learning Representations",
  year         = "2015",
}

@inproceedings{InceptionNet,
  title={Going deeper with convolutions},
  author={Szegedy, C. and Liu, W. and Jia, Y. and Sermanet, P. and Reed, S. and Anguelov, D. and Erhan, D. and Vanhoucke, V. and Rabinovich, A.},
  booktitle={CVPR},
  year={2015},
}

@article{B-NH2018arxiv,
author={T. Ben-Nun and T. Hoefler},
title={Demystifying Parallel and Distributed Deep Learning: An In-Depth Concurrency Analysis},
journal   = {CoRR},
volume    = {abs/1802.09941},
year={2018}}

@inproceedings{KMNST2017ICLR,
author={N. S. Keskar and D. Mudigere and J. Nocedal and M. Smelyanskiy and P. Tak Peter Tang},
title={On Large-Batch Training for Deep Learning: Generalization Gap and Sharp Minima},
booktitle={ICLR},
year={2017}}

@inproceedings{JKABFBS2018ICLRW,
author={S. Jastrzebski and Z. Kenton and D. Arpit and N. Ballas and A. Fischer and Y. Bengio and A. Storkey},
title={Finding Flatter Minima with SGD},
booktitle={ICLR Workshop Track},
year={2018}}

@inproceedings{SL2018ICLR,
author={S. L. Smith and Q. V. Le},
title={A Bayesian Perspective on Generalization and Stochastic Gradient Descent},
booktitle={ICLR},
year={2018}}

@inproceedings{MBB2018ICML,
author={S. Ma and R. Bassily and M. Belkin},
title={The Power of Interpolation: Understanding the Effectiveness of SGD in Modern Over-parametrized Learning},
booktitle={ICML},
year={2018},
}

@article{GAJKB2018SPAA,
author={A. Gholami and A. Azad and P. Jin and K. Keutzer and A. Buluc},
title={Integrated Model, Batch, and Domain Parallelism in Training Neural Networks},
journal={Proceedings of the 30th Syposium on Parallelism in Algorithms and Architectures},
year={2018},
pages={77--86}}

@inproceedings{DBLP:journals/corr/abs-1905-10395,
  author    = {Y. Teng and
               W. Gao and
               F. Chalus and
               \textbf{A. Choromanska} and
               D. Goldfarb and
               A. Weller},
  title     = {Leader Stochastic Gradient Descent for Distributed Training of Deep
               Learning Models},
  booktitle   = {NeurIPS},
  year      = {2019},
}

@inproceedings{RRWN2011NIPS,
	title={Hogwild: A Lock-Free Approach to Parallelizing Stochastic Gradient Descent},
	author={Recht, B. and Re, C. and Wright, S. and Niu, F.},
	booktitle={NIPS},
	year={2011}
}

@inproceedings{CBZSTO2017arxiv,
	title={Parle: parallelizing stochastic gradient descent},
	author={Chaudhari, P. and Baldassi, C. and Zecchina, R. and Soatto, S. and Talwalkar, A. and Oberman, A.},
	booktitle={arXiv:1707.00424},
	year={2017}
}

% After rebuttal
@InProceedings{baldassi2016unreasonable,
author = {Baldassi, C. and Borgs, C. and Chayes, J. T. and Ingrosso, A. and Lucibello, C. and Saglietti, L. and Zecchina, R.},
title = {Unreasonable effectiveness of learning neural networks: From accessible states and robust ensembles to basic algorithmic schemes},
booktitle = {PNAS},
year = {2016},
}

@article{izmailov2018averaging,
  title={Averaging weights leads to wider optima and better generalization},
  author={{Izmailov, P. et al.}},
  journal={arXiv:1803.05407},
  year={2018}
}

@inproceedings{DBLP:conf/nips/Li0TSG18,
  author    = {H. Li and
               Z. Xu and
               G. Taylor and
               C. Studer and
               T. Goldstein},
  title     = {Visualizing the Loss Landscape of Neural Nets},
  booktitle = {NeurIPS},
  year      = {2018},
}

@inproceedings{DBLP:journals/corr/abs-1905-10428,
  author    = {M. Majzoubi and
               \textbf{A. Choromanska}},
  title     = {LdSM: Logarithm-depth Streaming Multi-label Decision Trees},
  booktitle = {AISTATS},
  year = {2020},
}

@misc{WNonConv,
author = {A. Anandkumar and K. Chaudhuri and P. Liang and S. Oh and U. N. Niranjan},
title = "{Non-convex optimization, Workshop at NIPS}",
year = {2015},
howpublished = {\url{http://www.offconvex.org/2016/01/25/non-convex-workshop/}},
}

@misc{PGDL,
author = {Y. Jiang, P. Foret, S. Yak, B. Neyshabur, I. Guyon, H. Mobahi, G. K. Dziugaite, D. Roy, S. Gunasekar, S. Bengio},
title = "{Predicting Generalization in Deep Learning, Competition at NeurIPS}",
year = {2020},
howpublished = {\url{https://sites.google.com/view/pgdl2020/?fbclid=IwAR1OSnjH4UcOxznHBHWGZopSgXzGSLQHqSff75G9xot1c2sYX2sickxctmI}}
}

@misc{UConf,
  title = "{National Conference on Undergraduate Research.}",
  year = {2020},
  howpublished = {\url{http://www.cur.org/what/events/students/ncur/2020/}},
}

@misc{UGSRP,
title = "{NYU Tandon Undergraduate Summer Research Program}",
howpublished = {\url{https://engineering.nyu.edu/research/student-research/undergraduate-summer-research-program}},
}

@misc{Rhodes,
  title = "{The Rhodes Scholarship, University of Oxford}",
  howpublished = {\url{https://www.rhodeshouse.ox.ac.uk/}},
}

@misc{ODP,
  title = "{Netscape Communication Corporation. Open Directory
Project.}",
  howpublished = {\url{http://www.dmoz.org}},
}

@inproceedings{hardt2016train,
  title={Train faster, generalize better: Stability of stochastic gradient descent},
  author={Hardt, M. and Recht, B. and Singer, Y.},
  booktitle={ICML},
  year={2016}
}

@article{balakrishnan2017,
author = "Balakrishnan, S. and Wainwright, M. J. and Yu, B.",
journal = "Ann. Statist.",
number = "1",
pages = "77--120",
title = "Statistical guarantees for the EM algorithm: From population to sample-based analysis",
volume = "45",
year = "2017"
}

@misc{SemWeb,
  title = "{NYU Tandon ECE Seminar Series on Modern AI, Website}",
  howpublished = {\url{https://engineering.nyu.edu/academics/departments/electrical-and-computer-engineering/ece-seminar-series/modern-artificial}},
}

@misc{Playlist,
  title = "{NYU Tandon ECE Seminar Series on Modern AI, YouTube Playlist}",
  howpublished = {\url{https://www.youtube.com/playlist?list=PLhwo5ntex8iY9xhpSwWas451NgVuqBE7U}},
}

@misc{PIWeb,
  title = "{Website of the PI, \textbf{Anna Choromanska}}",
  howpublished = {\url{https://engineering.nyu.edu/faculty/anna-choromanska}},
}

@misc{DetNVIDIA,
author = {D. Riach},
  title = "{Determinism in Deep Learning, Research Talk at GTC}",
  year = {2019},
  howpublished = {\url{https://developer.download.nvidia.com/video/gputechconf/gtc/2019/presentation/s9911-determinism-in-deep-learning.pdf}},
}

@misc{wire,
  title = "{Inside Amazon's Artificial Intelligence Flywheel.}",
  howpublished = {\url{https://www.wired.com/story/amazon-artificial-intelligence-flywheel/}},
  }


@misc{Amazon,
  author = {P. Grey},
  title = "{How many products does amazon sell?, Technical Report by ExportX}",
  year = {2015},
  howpublished = {\url{https://export-x.com/2015/12/
11/how-many-products-does-amazon-sell-2015/}},
}

@article{MegatronLM,
    title={Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism},
    author={M. Shoeybi and M. Patwary and R. Puri and P. LeGresley and J. Casper and B. Catanzaro},
    year={2019},
    journal   = {CoRR},
    volume    = {abs/1909.08053},
    howpublished = {\url{https://nv-adlr.github.io/MegatronLM}},
}

@inproceedings{conf/sigir/BennettN09,
  author = {Bennett, P. N. and Nguyen, N.},
  booktitle = {SIGIR},
  title = {Refined experts: improving classification in large taxonomies.},
  year = 2009
}

@article{Thomee:2016:YND:2886013.2812802,
 author = {Thomee, B. and Shamma, D. A. and Friedland, G. and Elizalde, B. and Ni, K.l and Poland, D. and Borth, D. and Li, L.-J.},
 title = "{YFCC100M: The New Data in Multimedia Research}",
 journal = {Commun. ACM},
 issue_date = {February 2016},
 volume = {59},
 number = {2},
 month = jan,
 year = {2016},
 pages = {64--73},
 url = {http://doi.acm.org/10.1145/2812802},
} 

@misc{Gutenberg,
  title = "{Project Gutenberg}",
  howpublished = {\url{http://www.gutenberg.org/}},
}

@misc{Auto,
author = {M. Pastor},
  title = "{Data storage is the key to autonomous vehicles’ future, Technical Report by IoT Now Transport}",
  year = {2019},
howpublished = {\url{https://iotnowtransport.com/2019/02/12/71015-data-storage-key-autonomous-vehicles-future/}},
}

@misc{Money,
author = {T. Peng},
title = "{The Staggering Cost of Training SOTA AI Models, Technical Report by Medium}",
year = {2019},
howpublished = {\url{https://medium.com/syncedreview/the-staggering-cost-of-training-sota-ai-models-e329e80fa82}},
}

@article{journals/cacm/ThomeeSFENPBL16,
  author = {Thomee, B. and Shamma, D. A. and Friedland, G. and Elizalde, B. and Ni, K. and Poland, D. and Borth, D. and Li, L-J.},
  journal = {Commun. ACM},
  number = 2,
  pages = {64-73},
  title = {YFCC100M: the new data in multimedia research.},
  volume = 59,
  year = 2016
}


@misc{Wikipedia,
  title = "{Wikipedia Database}",
  howpublished = {\url{https://en.wikipedia.org/wiki/Wikipedia:Database_download}},
}

@article{Radovic:2018dip,
    author = "Radovic, A. and Williams, M. and Rousseau, D. and Kagan, M. and Bonacorsi, D. and Himmel, A. and Aurisano, A. and Terao, K. and Wongjirad, T.",
    title = "{Machine learning at the energy and intensity frontiers of particle physics}",
    journal = "Nature",
    volume = "560",
    number = "7716",
    pages = "41--48",
    year = "2018"
}

@article{citeulike:13779585,
    author = {The 1000 Genomes Project Consortium},
    journal = {Nature},
    number = {7571},
    pages = {68--74},
    title = {{A global reference for human genetic variation}},
    url = {http://dx.doi.org/10.1038/nature15393},
    volume = {526},
    year = {2015}
}

@article{hinton12,
title = {Deep Neural Networks for Acoustic Modeling in Speech Recognition},
author  = {G. Hinton and L. Deng and D. Yu and G. Dahl and A.-r. Mohamed and N. Jaitly and A. Senior and V. Vanhoucke and P. Nguyen and T. Sainath and B. Kingsbury},
year  = 2012,
url = {https://www.microsoft.com/en-us/research/publication/deep-neural-networks-for-acoustic-modeling-in-speech-recognition/},
month = {November},
pages = {82-97},
journal = {IEEE Signal Processing Magazine},
volume = {29},
}

@inproceedings{DBLP:journals/corr/abs-1303-5778,
  author    = {A. Graves and
               A.l{-}r. M. and
               G. E. Hinton},
  title     = {Speech Recognition with Deep Recurrent Neural Networks},
  booktitle = {ICASSP},
  year      = {2013},
}

@inproceedings{Karpathy:2014:LVC:2679600.2680211,
 author = {Karpathy, A. and Toderici, G. and Shetty, S. and Leung, T. and Sukthankar, R. and Fei-Fei, L.},
 title = {Large-Scale Video Classification with Convolutional Neural Networks},
 booktitle = {CVPR},
 year = {2014},
 url = {http://dx.doi.org/10.1109/CVPR.2014.223},
} 

@misc{ILSVRC,
  title = "{ImageNet Large Scale Visual Recognition Challenge}",
  howpublished = {\url{http://image-net.org/challenges/LSVRC/2016/}},
}

@misc{COCO_comp,
  title = "{ Microsoft COCO Image Captioning Challenge}",
  howpublished = {\url{http://mscoco.org/dataset/#overview}},
}

@article{DBLP:journals/corr/BojarskiTDFFGJM16,
  author    = {M. Bojarski and
               D. Del Testa and
               D. Dworakowski and
               B. Firner and
               B. Flepp and
               P. Goyal and
               L. D. Jackel and
               M. Monfort and
               U. Muller and
               J. Zhang and
               X. Zhang and
               J. Zhao and
               K. Zieba},
  title     = {End to End Learning for Self-Driving Cars},
  journal   = {CoRR},
  volume    = {abs/1604.07316},
  year      = {2016},
  url       = {http://arxiv.org/abs/1604.07316},
}

@inproceedings{conf/iccv/ChenSKX15,
  author = {Chen, C. and Seff, A. and Kornhauser, A. L. and Xiao, J.},
  booktitle = {ICCV},
  title = {DeepDriving: Learning Affordance for Direct Perception in Autonomous Driving.},
  year = 2015,
}

@inproceedings{DBLP:journals/corr/Shalev-ShwartzS16a,
  author    = {S. Shalev{-}Shwartz and
               S. Shammah and
               A. Shashua},
  title     = {Safe, Multi-Agent, Reinforcement Learning for Autonomous Driving},
  booktitle = {NIPS Workshop on Learning, Inference and Control of Multi-Agent Systems},
  year      = {2016},
  url       = {http://arxiv.org/abs/1610.03295},
}

@misc{TDLICML2016,
  title = "{Theory of Deep Learning Workshop, ICML'16}",
  howpublished = {\url{https://sites.google.com/site/deeplearningtheory/}},
}

@misc{ICML2019W,
  author = {\textbf{A. Choromanska} and L. Jackel and Li Erran Li and J. C. Niebles and A. Gaidon and W.-L. Chao and I. Posner},
  title = "{AI for Autonomous Driving, Workshop at ICML}",
  year = {2019},
  howpublished = {\url{https://sites.google.com/view/icml2019aiad/home}},
}

@misc{ICML2020W,
author = {\textbf{A. Choromanska} and J. Langford and M. Majzoubi and Y. Prabhu},
  title = "{Extreme Classification Theory and Applications, Workshop at ICML}",
  year = {2020},
  howpublished = {\url{http://manikvarma.org/events/XC20/index.html}},
}

@misc{NO2016,
 author = {A. Anandkumar and P. Liang and H. Mobahi and S. Jegelka and \textbf{A. Choromanska}},
  title = "{Nonconvex Optimization for Machine Learning: Theory and Practice, Workshop at NIPS}",
  year = {2016},
  howpublished = {\url{https://sites.google.com/site/nonconvexnips2016/}},
}

@misc{ECMLPKDD2015,
  title = "{International Workshop on Big Multi-Target Prediction, ECML/PKDD'15}",
  howpublished = {\url{http://www.kermit.ugent.be/big-multi-target-prediction/index.php/}},
}

@misc{SSB2015,
  title = "{Summer School on Non-convex Optimization in Machine Learning at the Indian Institute of Technology, Bombay, India'15}",
  howpublished = {\url{https://www.cse.iitb.ac.in/~ganesh/noml2015/}},
}

@inproceedings{NIPS2014_5486,
title = {Identifying and attacking the saddle point problem in high-dimensional non-convex optimization},
author = {Dauphin, Y. N. and Pascanu, R. and Gulcehre, C. and Cho, K. and Ganguli, S. and Bengio, Y.},
booktitle = {NIPS},
year = {2014},
}

@inproceedings{JoanTopology,
  author    = {C. D. Freeman and J. Bruna},
  title     = {Topology and Geometry of Half-Rectified Network Optimization},
  booktitle = {ICLR},
  year      = {2017},
}
 
 @inproceedings{JoanPS,
  author    = {M. Trager and K. Kohn and J. Bruna},
  title     = {Pure and Spurious Critical Points: a Geometric Study of Linear Networks},
  booktitle = {ICLR},
  year      = {2020},
} 

@inproceedings{DBLP:journals/corr/GoodfellowV14,
  author    = {I. J. Goodfellow and
               O. Vinyals},
  title     = {Qualitatively characterizing neural network optimization problems},
  booktitle = {ICLR},
  year      = {2015},
  url       = {http://arxiv.org/abs/1412.6544},
}

@incollection{NIPS2016_6112,
title = {Deep Learning without Poor Local Minima},
author = {Kawaguchi, K.},
booktitle = {NIPS},
year = {2016},
}

@inproceedings{DBLP:conf/aistats/ChoromanskaHMAL15,
  author    = {\textbf{A. Choromanska} and
               M. Henaff and
               M. Mathieu and
               G. Ben Arous and
               Y. LeCun},
  title     = {The Loss Surfaces of Multilayer Networks},
  booktitle = {AISTATS},
  year      = {2015},
  url       = {http://jmlr.org/proceedings/papers/v38/choromanska15.html},
}

@article{DBLP:journals/corr/KingmaB14,
  author    = {D. P. Kingma and
               J. Ba},
  title     = {Adam: {A} Method for Stochastic Optimization},
  journal   = {CoRR},
  volume    = {abs/1412.6980},
  year      = {2014},
  url       = {http://arxiv.org/abs/1412.6980},
}

@article{DBLP:journals/corr/abs-1810-00150,
  author    = {C. Lee and
               K. Cho and
               W. Kang},
  title     = {Directional Analysis of Stochastic Gradient Descent via von Mises-Fisher
               Distributions in Deep learning},
  journal   = {CoRR},
  volume    = {abs/1810.00150},
  year      = {2018},
}

@inproceedings{DBLP:journals/corr/abs-1810-11910,
  author    = {M. Riemer and
               I. Cases and
               R. Ajemian and
               M. Liu and
               I. Rish and
               Y. Tu and
               G. Tesauro},
  title     = {Learning to Learn without Forgetting By Maximizing Transfer and Minimizing
               Interference},
  booktitle   = {ICLR},
  year      = {2019},
}

@article{feng2020neural,
    title={How neural networks find generalizable solutions: Self-tuned annealing in deep learning},
    author={Y. Feng and Y. Tu},
    year={2020},
    journal   = {CoRR},
    volume = {abs/2001.01678},
}

@article{DBLP:journals/corr/GongLYB14,
  author    = {Y. Gong and
               L. Liu and
               M. Yang and
               L. D. Bourdev},
  title     = {Compressing Deep Convolutional Networks using Vector Quantization},
  journal   = {CoRR},
  volume    = {abs/1412.6115},
  year      = {2014},
  url       = {http://arxiv.org/abs/1412.6115},
}

@inproceedings{DBLP:journals/corr/HanMD15,
  author    = {S. Han and
               H. Mao and
               W. J. Dally},
  title     = {Deep Compression: Compressing Deep Neural Network with Pruning, Trained
               Quantization and Huffman Coding},
  booktitle = {ICLR},
  year      = {2016},
}

@inproceedings{DBLP:conf/kdd/ChenWTWC16,
  author    = {W. Chen and
               J. T. Wilson and
               S. Tyree and
               K. Q. Weinberger and
               Y. Chen},
  title     = {Compressing Convolutional Neural Networks in the Frequency Domain},
  booktitle = {KDD},
  year      = {2016},
  url       = {http://doi.acm.org/10.1145/2939672.2939839},
}

@article{DBLP:journals/corr/SimonyanZ14a,
  author    = {K. Simonyan and
               A. Zisserman},
  title     = {Very Deep Convolutional Networks for Large-Scale Image Recognition},
  journal   = {CoRR},
  volume    = {abs/1409.1556},
  year      = {2014},
  url       = {http://arxiv.org/abs/1409.1556},
}

@article{Rifkin2004,
 author = {Rifkin, R. and Klautau, A.},
 title = {In Defense of One-Vs-All Classification},
 journal = {J. Mach. Learn. Res.},
 month = dec,
 volume = {5},
 year = {2004},
 pages = {101--141},
} 

@incollection{NIPS2008_3583,
title = {A Scalable Hierarchical Distributed Language Model},
author = {Mnih, A. and G. E. Hinton},
booktitle = {NIPS},
year = {2009},
}

@misc{ICML2015W,
  author = {M. Cissé and S. Bengio and P. Gallinari and P. Mineiro and N. Usunier and J. Y. Yu and X. Zhang},
  title = "{Extreme Classification, Workshop at ICML}",
  year = {2015},
  howpublished = {\url{https://sites.google.com/site/extremeclassification/}},
}

@misc{EMCDU14,
author = {International Data Corporation},
  title = "{The Digital Universe of Opportunities: Rich Data and the Increasing Value of the Internet of Things, Technical Report}",
  year = {2014},
  howpublished = {\url{https://uk.emc.com/leadership/digital-universe/2014iview/executive-summary.htm}},
}

@inproceedings{BengioWG10,
  title = {Label Embedding Trees for Large Multi-Class Tasks.},
  author = {Bengio, S. and Weston, J. and Grangier, D.},
  booktitle = {NIPS},
  year = 2010
}

@article{DBLP:journals/corr/JoulinGBM16,
  author    = {A. Joulin and
               E. Grave and
               P. Bojanowski and
               T. Mikolov},
  title     = {Bag of Tricks for Efficient Text Classification},
  journal   = {CoRR},
  volume    = {abs/1607.01759},
  year      = {2016}
}

@article{journals/informaticaSI/MadzarovGC09,
  author = {Madzarov, G. and Gjorgjevikj, D. and Chorbev, I.},
  title = {A Multi-class SVM Classifier Utilizing Binary Decision Tree.},
  journal = {Informatica},
  number = 2,
  pages = {225-233},
  volume = 33,
  year = 2009
}

@inproceedings{weston13, 
    Title = {Label Partitioning For Sublinear Ranking},
    Booktitle = {ICML}, 
    Author = {J. Weston and A. Makadia and H. Yee}, 
    Year = {2013}, 
   }

@inproceedings{DengSBL11,
  title = {Fast and Balanced: Efficient Label Tree Learning for Large Scale Object Recognition.},
  author = {Deng, J. and Satheesh, S. and Berg, A. C. and Fei-Fei, L.},
  booktitle = {NIPS},
  year = 2011
}

@inproceedings{conf/cvpr/ZhaoX13,
  author = {Zhao, B. and Xing, E. P.},
  booktitle = {CVPR},
  title = {Sparse Output Coding for Large-Scale Visual Recognition.},
  year = 2013
}

@article{JL2016,
  author    = {H. Daume and N. Karampatziakis and J. Langford and P. Mineiro},
  title     = {Logarithmic Time One-Against-Some},
  journal   = {CoRR},
  volume    = {abs/1606.04988},
  year      = {2016},
}

@inproceedings{DBLP:journals/corr/abs-0902-1284,
  author = {Hsu, D. and Kakade, S. and Langford, J. and Zhang, T.},
  title = {Multi-Label Prediction via Compressed Sensing.},
  booktitle = {NIPS},
  year = 2009
}

@inproceedings{DBLP:journals/corr/AgarwalKKSV13,
  author    = {A. Agarwal and
               S. M. Kakade and
               N. Karampatziakis and
               L. Song and
               G. Valiant},
  title     = {Least Squares Revisited: Scalable Approaches for Multi-class Prediction},
  booktitle   = {ICML},
  year      = {2014}
}

@inproceedings{icml2014c2_beijbom14,
    Author = {O. Beijbom and M. Saberian and D. Kriegman and N. Vasconcelos},
    Title = {Guess-Averse Loss Functions For Cost-Sensitive Multiclass Boosting},
  booktitle   = {ICML},
  year      = {2014}
   } 

@inproceedings{BeygelzimerLR09,
  author = {Beygelzimer, A. and Langford, J. and Ravikumar, P. D.},
  booktitle = {ALT},
  title = {Error-Correcting Tournaments.},
  year = 2009
}

@incollection{NIPS2015_5937,
title = {Logarithmic Time Online Multiclass prediction},
author = {\textbf{A. Choromanska} and Langford, J.},
booktitle = {NIPS},
year = {2015},
}

@inproceedings{BeygelzimerLLSS09,
  author = {Beygelzimer, A. and Langford, J. and Lifshits, Y. and Sorkin, G. B. and Strehl, A. L.},
  booktitle = {UAI},
  title = {Conditional Probability Tree Estimation Analysis and Algorithms.},
  year = 2009
}

@inproceedings{Agrawal:2013:MLM:2488388.2488391,
 author = {Agrawal, R. and Gupta, A. and Prabhu, Y. and Varma, M.},
 title = {Multi-label Learning with Millions of Labels: Recommending Advertiser Bid Phrases for Web Pages},
 booktitle = {WWW},
 year = {2013},
} 

@inproceedings{Prabhu:2014:FFA:2623330.2623651,
 author = {Prabhu, Y. and Varma, M.},
 title = {FastXML: A Fast, Accurate and Stable Tree-classifier for Extreme Multi-label Learning},
 booktitle = {KDD},
 year = {2014},
} 

@inproceedings{conf/icml/WestonMY13,
  author = {Weston, J. and Makadia, A. and Yee, H.},
  booktitle = {ICML},
  title = {Label Partitioning For Sublinear Ranking.},
  year = 2013
}

@article{Takimoto:2003:TDT:763058.763066,
 author = {Takimoto, E. and Maruoka, A.},
 title = {Top-down Decision Tree Learning As Information Based Boosting},
 journal = {Theor. Comput. Sci.},
 volume = {292},
 number = {2},
 month = jan,
 year = {2003},
 pages = {447--464},
 url = {http://dx.doi.org/10.1016/S0304-3975(02)00181-0},
} 

@inproceedings{conf/icml/BalasubramanianL12,
  author = {Balasubramanian, K. and Lebanon, G.},
  booktitle = {ICML},
  title = {The Landmark Selection Method for Multiple Output Prediction.},
  year = 2012
}

@incollection{NIPS2015_5969,
title = {Sparse Local Embeddings for Extreme Multi-label Classification},
author = {Bhatia, K. and Jain, H. and Kar, P. and Varma, M. and Jain, P.},
booktitle = {NIPS},
year = {2015},
}

@inproceedings{icml2013_bi13, 
    Title = {Efficient Multi-label Classification with Many Labels}, 
    Booktitle = {ICML}, 
    Author = {W. Bi and J. Kwok}, 
    Year = {2013}, 
   }

@incollection{NIPS2012_4561,
title = {Feature-aware Label Space Dimension Reduction for Multi-label Classification},
author = {Y{-}n. Chen and H{-}t. Lin},
booktitle = {NIPS},
year = {2012},
}

@incollection{NIPS2013_5083,
title = {Robust Bloom Filters for Large MultiLabel Classification Tasks},
author = {Cisse, M. M. and Usunier, N. and Arti\`{e}res, T. and Gallinari, P.},
booktitle = {NIPS},
year = {2013},
}

@INPROCEEDINGS{Ferng11multi-labelclassification,
    author = {C{-}s. Ferng and H{-}t. Lin},
    title = {Multi-label Classification with Error-correcting Codes},
    booktitle = {ACML},
    year = {2011},
}

@incollection{NIPS2009_3824,
title = {Multi-Label Prediction via Compressed Sensing},
author = {Hsu, D. J. and Kakade, S. M. and Langford, J. and Zhang, T.},
booktitle = {NIPS},
year = {2009},
}

@inproceedings{Ji:2008:ESS:1401890.1401939,
 author = {Ji, S. and Tang, L. and Yu, S. and Ye, J.},
 title = {Extracting Shared Subspace for Multi-label Classification},
 booktitle = {KDD},
 year = {2008},
} 

@article{DBLP:journals/corr/KarampatziakisM15,
  author    = {N. Karampatziakis and
               P. Mineiro},
  title     = {Multilabel Prediction via Calibration},
  journal   = {CoRR},
  volume    = {abs/1502.02710},
  year      = {2015},
  url       = {http://arxiv.org/abs/1502.02710},
}

@inproceedings{icml2014c2_linc14, 
    Title = {Multi-label Classification via Feature-aware Implicit Label Space Encoding}, 
    Url = {http://jmlr.org/proceedings/papers/v32/linc14.pdf}, 
    Author = {Z. Lin and G. Ding and M. Hu and J. Wang}, 
    Year = {2014}, 
    Booktitle = {ICML}, 
   }

@inproceedings{DBLP:journals/corr/MineiroK15,
  author    = {P. Mineiro and
               N. Karampatziakis},
  title     = {Fast Label Embeddings for Extremely Large Output Spaces},
 booktitle = {ECML},
  year      = {2015},
}

@inproceedings{NIPS2015_5770, 
   title = {Large-Scale Bayesian Multi-Label Learning via Topic-Based Label Embeddings}, 
   author = {Rai, P. and Hu, C. and Henao, R. and Carin, L.}, 
   booktitle = {NIPS},  
   year = {2015}, 
   }

@inproceedings{Tai:2012:MCP:2344802.2344812,
 author = {Tai, F. and Lin, H.-T.},
 title = {Multilabel Classification with Principal Label Space Transformation},
booktitle = {MLD},  
 year = {2010},
} 

@inproceedings{DBLP:conf/icml/Yu0KD14,
  author    = {H.{-}F. Yu and
               P. Jain and
               P. Kar and
               I. S. Dhillon},
  title     = {Large-scale Multi-label Learning with Missing Labels},
  booktitle = {ICML},
  year      = {2014},
}

@inproceedings{cca,
    author = {Zhang, Y. and Schneider, J.},
    booktitle = {AISTATS},
    title = {{Multi-label Output Codes using Canonical Correlation Analysis}},
    year = {2011}
}

@InProceedings{Jain16,
  author    = "Jain, H. and Prabhu, Y. and Varma, M.",
  title     = "Extreme Multi-label Loss Functions for Recommendation, Tagging, Ranking and Other Missing Label Applications",
  booktitle = "KDD",
  year      = "2016",
}

@incollection{DBLP:journals/corr/SaxeMG13,
  author    = {A. M. Saxe and
               J. L. McClelland and
               S. Ganguli},
  title     = {Exact solutions to the nonlinear dynamics of learning in deep linear
               neural networks},
  booktitle   = {ICLR},
  year      = {2014},
}

@article{PhysRevA.32.1007,
  title = {Spin-glass models of neural networks},
  author = {Amit, D. J. and Gutfreund, H. and Sompolinsky, H.},
  journal = {Phys. Rev. A},
  volume = {32},
  pages = {1007--1018},
  year = {1985},
}

@article{0305-4470-30-23-009,
  author={K. Nakanishi and H. Takayama},
  title={Mean-field theory for a spin-glass model of neural networks: TAP free energy and the paramagnetic to spin-glass transition},
  journal={Journal of Physics A: Mathematical and General},
  volume={30},
  pages={8085},
  year={1997},
}

@book{Dotsenko1995,
  author = {V. Dotsenko},
  title = {An Introduction to the Theory of Spin Glasses and Neural Networks},
  publisher = {World Scientific Lecture Notes in Physics},
  year = {1995},
}

@article{Baldi:1989:NNP:70359.70362,
	Author = {Baldi, P. and Hornik, K.},
	Journal = {Neural Networks},
	Pages = {53--58},
	Title = {Neural Networks and Principal Component Analysis: Learning from Examples Without Local Minima},
	Volume = {2},
	Year = {1989}}

@article{chaudhari2015trivializing,
	Author = {Chaudhari, P. and Soatto, S.},
	journal = {CoRR},
	volume = {abs/1511.06485},
	Title = {On the energy landscape of deep networks},
	Year = {2015}}

@article{soudry2016no,
	Author = {Soudry, D. and Carmon, Y.},
	Title = {No bad local minima: Data independent training error guarantees for multilayer neural networks},
	journal = {CoRR},
	volume = {abs/1605.08361},
	Year = {2016}}

@inproceedings{hardt2015train,
	Author = {Hardt, M. and Recht, B. and Singer, Y.},
  title     = {Train faster, generalize better: Stability of stochastic gradient
               descent},
  booktitle = {ICML},
  year      = {2016},
  url       = {http://jmlr.org/proceedings/papers/v48/hardt16.html},
}

@article{anandkumar2016efficient,
	Author = {Anandkumar, A. and Ge, R.},
	Title = {Efficient approaches for escaping higher order saddle points in non-convex optimization},
	Journal = {CoRR},
	volume = {abs/1602.05908},
	Year = {2016}}

@inproceedings{sutskever2013importance,
	Author = {Sutskever, I. and Martens, J. and Dahl, G. and Hinton, G.},
	Booktitle = {ICML},
	Title = {On the importance of initialization and momentum in deep learning},
	Year = {2013}}

@article{duchi2011adaptive,
	Author = {Duchi, J. and Hazan, E. and Singer, Y.},
	Journal = {JMLR},
	Numpages = {39},
	Pages = {2121--2159},
	Title = {Adaptive subgradient methods for online learning and stochastic optimization},
	Volume = {12},
	Year = {2011}}

@article{srivastava2014dropout,
	Author = {Srivastava, N. and Hinton, G. and Krizhevsky, A. and Sutskever, I. and Salakhutdinov, R.},
	Journal = {JMLR},
	Number = {1},
	Pages = {1929--1958},
	Title = {Dropout: a simple way to prevent neural networks from overfitting.},
	Volume = {15},
	Year = {2014}}

@article{sermanet2013overfeat,
  author = {Sermanet, P. and Eigen, D. and Zhang, X. and Mathieu, M. and Fergus, R. and LeCun, Y.},
  title = {OverFeat: Integrated Recognition, Localization and Detection using
  Convolutional Networks},
  journal = {CoRR},
  volume = {abs/1312.6229},
  year = 2013,
}

@article{ioffe2015batch,
	Author = {Ioffe, S. and Szegedy, C.},
	Title = {{Batch normalization: Accelerating deep network training by reducing internal covariate shift}},
	journal = {CoRR},
	volume = {abs/1502.03167},
	Year = {2015}}

@article{cooijmans2016recurrent,
	Author = {Cooijmans, T. and Ballas, N. and Laurent, C. and Courville, A.},
	journal = {CoRR},
	volume = {abs/1603.09025},
	Title = {Recurrent Batch Normalization},
	Year = {2016}}

@article{CHAHAL202065,
title = "A {Hitchhiker's} Guide On Distributed Training Of Deep Neural Networks",
journal = "Journal of Parallel and Distributed Computing",
volume = "137",
pages = "65 - 76",
year = "2020",
author = "K. S. Chahal and M. S. Grover and K. Dey and R. R. Shah",
}

@incollection{NIPS2003_2440,
title = {Online Learning of Non-stationary Sequences},
author = {C. Monteleoni and T. S. Jaakkola},
booktitle = {NIPS},
year = {2004},
}

@article{salimans2016weight,
	Author = {Salimans, T. and Kingma, D.},
	journal = {CoRR},
	volume = {abs/1602.07868},
	Title = {Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks},
	Year = {2016}}

@article{baldassi2015subdominant,
	Author = {Baldassi, C. and Ingrosso, A. and Lucibello, C. and Saglietti, L. and Zecchina, R.},
	Journal = {Physical review letters},
	Number = {12},
	Pages = {128101},
	Title = {Subdominant dense clusters allow for simple learning and high computational performance in neural networks with discrete synapses},
	Volume = {115},
	Year = {2015}}

@article{baldassi2016unreasonable,
	Title = {Unreasonable effectiveness of learning neural networks: {From} accessible states and robust ensembles to basic algorithmic schemes},
	Volume = {113},
	Number = {48},
	Journal = {PNAS},
	Author = {Baldassi, C. and Borgs, C. and Chayes, J. T. and Ingrosso, A. and Lucibello, C. and Saglietti, L. and Zecchina, R.},	
	Year = {2016},
	Pages = {E7655--E7662}}

@article{baldassi2016multilevel,
        Title = {Learning may need only a few bits of synaptic precision},
        Volume = {93},
        Number = {5},
        Journal = {Physical Review E},
        Author = {Baldassi, C. and Gerace, F. and Lucibello, C. and Saglietti, L. and Zecchina, R.},
        Year = {2016},
        Pages = {052313}}

@article{hochreiter1997flat,
  title={Flat minima},
  author={Hochreiter, S. and Schmidhuber, J.},
  journal={Neural Computation},
  volume={9},
  number={1},
  pages={1--42},
  year={1997},
  publisher={MIT Press}
}

@article{zhang2016understanding,
  title={Understanding deep learning requires rethinking generalization},
  author={Zhang, C. and Bengio, S. and Hardt, M. and Recht, B. and Vinyals, O.},
  journal = {CoRR},
  volume={abs/1611.03530},
  year={2016}
}

@article{Bartlett:2006:SCP:2263195.2265382,
 author = {Bartlett, P. L.},
 title = {The Sample Complexity of Pattern Classification with Neural Networks: The Size of the Weights is More Important Than the Size of the Network},
 journal = {IEEE Trans. Inf. Theor.},
 volume = {44},
 number = {2},
 month = sep,
 year = {2006},
 pages = {525--536},
 url = {http://dx.doi.org/10.1109/18.661502},
} 

@article{DBLP:journals/corr/NeyshaburTS14,
  author    = {B. Neyshabur and
               R. Tomioka and
               N. Srebro},
  title     = {In Search of the Real Inductive Bias: On the Role of Implicit Regularization
               in Deep Learning},
  journal   = {CoRR},
  volume    = {abs/1412.6614},
  year      = {2014},
  url       = {http://arxiv.org/abs/1412.6614},
}

@book{allgower2012numerical,
  title={Numerical continuation methods: an introduction},
  author={Allgower, E. L. and Georg, K.},
  volume={13},
  year={2012},
  publisher={Springer}
}

@inproceedings{mobahi2015link,
  title={{On the link between Gaussian homotopy continuation and convex envelopes}},
  author={Mobahi, H. and Fisher III, J.},
  booktitle={Workshop on Energy Minimization Methods in CVPR},
  pages={43--56},
  year={2015},
  organization={Springer}
}

@article{gulcehre2016mollifying,
  title={Mollifying Networks},
  author={Gulcehre, C. and Moczulski, M. and Visin, F. and Bengio, Y.},
  journal = {CoRR},
  volume={abs/1608.04980},
  year={2016}
}

@article{doi:10.1117/12.2176598,
author = {Ji, Z. and Ovsiannikov, I, and Wang, Y. and Shi, L. and Zhang, Q.},
title = {Reducing weight precision of convolutional neural networks towards large-scale on-chip image recognition},
journal = {Proc. SPIE},
year = {2015},
}

@article{DBLP:journals/corr/ChenWTWC15,
  author    = {W. Chen and
               J. T. Wilson and
               S. Tyree and
               K. Q. Weinberger and
               Y. Chen},
  title     = {Compressing Neural Networks with the Hashing Trick},
  journal   = {CoRR},
  volume    = {abs/1504.04788},
  year      = {2015},
}

@article{DBLP:journals/corr/CollinsK14,
  author    = {M. D. Collins and
               P. Kohli},
  title     = {Memory Bounded Deep Convolutional Networks},
  journal   = {CoRR},
  volume    = {abs/1412.1442},
  year      = {2014},
}

@incollection{NIPS2013_5025,
title = {Predicting Parameters in Deep Learning},
author = {Denil, M. and Shakibi, B. and Dinh, L. and Ranzato, M. A. and de Freitas, N.},
booktitle = {NIPS},
year = {2013},
}

@incollection{NIPS2014_5544,
    Author = {E. L. Denton and W. Zaremba and J. Bruna and Y. Lecun and R. Fergus},
    Booktitle = {NIPS},
    Title = {Exploiting Linear Structure Within Convolutional Networks for Efficient Evaluation},
    Year = {2014},
   } 

@incollection{NIPS2013_5025,
title = {Predicting Parameters in Deep Learning},
author = {Denil, M. and Shakibi, B. and Dinh, L. and Ranzato, M. A. and de Freitas, N.},
booktitle = {NIPS},
year = {2013},
}

@inproceedings{Hinton:1993:KNN:168304.168306,
 author = {Hinton, G. E. and van Camp, D.},
 title = {Keeping the Neural Networks Simple by Minimizing the Description Length of the Weights},
 booktitle = {COLT},
 year = {1993},
} 

@incollection{nowlan:adaptive,
  author = {Nowlan, S. J. and Hinton, G. E.},
  booktitle = {NIPS},
  title = {Adaptive Soft Weight Tying Using Gaussian Mixtures},
  year = 1992
}

@incollection{Kim2015,
  author    = {M. Kim and P. Smaragdis},
  title     = {Bitwise neural networks},
  booktitle = {ICML Workshop on Resource-Efficient Machine Learning},
  year      = {2015},
}

@article{DBLP:journals/corr/LinCMB15,
  author    = {Z. Lin and
               M. Courbariaux and
               R. Memisevic and
               Y. Bengio},
  title     = {Neural Networks with Few Multiplications},
  journal   = {CoRR},
  volume    = {abs/1510.03009},
  year      = {2015},
}

@article{journals/tnn/MarchesiOPU93,
  author = {Marchesi, M. and Orlandi, G. and Piazza, F. and Uncini, A.},
  journal = {IEEE Transactions on Neural Networks},
  keywords = {dblp},
  number = 1,
  pages = {53-62},
  title = {Fast neural networks without multipliers.},
  volume = 4,
  year = 1993
}

@incollection{NIPS1993_833,
title = {Backpropagation without Multiplication},
author = {P. Y. Simard and H. P. Graf},
booktitle = {NIPS},
year = {1994},
}

@incollection{Burge:1999:SBN:296533.296552,
 author = {Burge, P. S. and van Daalen, M. R. and Rising, B. J. P. and Shawe-Taylor, J. S.},
 chapter = {Stochastic Bit-stream Neural Networks},
 title = {Pulsed Neural Networks},
 year = {1999},
 pages = {337--352},
 numpages = {16},
 publisher = {MIT Press},
} 

@article{DBLP:journals/corr/Courbariaux2015,
  author    = {M. Courbariaux and Y. Bengio and J.-P. David},
  title     = {BinaryConnect: Training Deep Neural Networks with binary weights during propagations},
  journal   = {CoRR},
  volume    = {abs/1511.00363},
  year      = {2015},
}

@unpublished{LSZ2017,
  author    = {L. Shi and S. Feng and Z. Zhu},
  title     = {HFH: Homologically Functional Hashing for Compressing Deep Neural Networks},
  note = {Preprint at \url{https://openreview.net/pdf?id=Hyanrrqlg}},
  year      = {2017},
}

@inproceedings{DBLP:journals/corr/KimPYCYS15,
  author    = {Y.{-}D. Kim and
               E. Park and
               S. Yoo and
               T. Choi and
               L. Yang and
               D. Shin},
  title     = {Compression of Deep Convolutional Neural Networks for Fast and Low
               Power Mobile Applications},
  booktitle = {ICLR},
  year      = {2016},
}

@inproceedings{Yang2015,
    author = "Yang, Z. and Moczulski, M. and Denil, M. and de Freitas, N. and Smola, A. and Song, L. and Wang, Z.",
    title = "Deep Fried Convnets",
    year = "2015",
    booktitle = "ICCV",
}

@article{DBLP:journals/corr/HintonVD15,
  author    = {G. E. Hinton and
               O. Vinyals and
               J. Dean},
  title     = {Distilling the Knowledge in a Neural Network},
  journal   = {CoRR},
  volume    = {abs/1503.02531},
  year      = {2015},
  url       = {http://arxiv.org/abs/1503.02531},
}

@inproceedings{41466,
title = {Fastfood - Approximating Kernel Expansions in Loglinear Time},
author  = {Q. Le and T. Sarlos and A. Smola},
year  = 2013,
URL = {http://jmlr.org/proceedings/papers/v28/le13.html},
booktitle = {ICML}
}

@inproceedings{DBLP:conf/nips/HanPTD15,
  author    = {S. Han and
               J. Pool and
               J. Tran and
               W. J. Dally},
  title     = {Learning both Weights and Connections for Efficient Neural Network},
  booktitle = {NIPS},
  year      = {2015},
  url       = {http://papers.nips.cc/paper/5784-learning-both-weights-and-connections-for-efficient-neural-network},
}

@inproceedings{DBLP:journals/corr/GuoYC16,
  author    = {Y. Guo and
               A. Yao and
               Y. Chen},
  title     = {Dynamic Network Surgery for Efficient DNNs},
  booktitle   = {NIPS},
  year      = {2016},
}

@inproceedings{DBLP:journals/corr/C,
  author    = {W. Chen and
               J. T. Wilson and
               S. Tyree and
               K. Q. Weinberger and
               Y. Chen},
  title     = {Compressing Convolutional Neural Networks},
  booktitle   = {NIPS},
  year      = {2015},
  url       = {http://arxiv.org/abs/1506.04449},
}

@inproceedings{SWS17,
  author    = {K. Ullrich, E. Meeds, M. Welling},
  title     = {Soft Weight-Sharing for Neural Network Compression},
  booktitle   = {ICLR},
  year      = {2017},
}

@inproceedings{DBLP:conf/nips/WenWWCL16,
  author    = {W. Wen and
               C. Wu and
               Y. Wang and
               Y. Chen and
               H. Li},
  title     = {Learning Structured Sparsity in Deep Neural Networks},
  booktitle = {NIPS},
  year      = {2016},
  url       = {http://papers.nips.cc/paper/6504-learning-structured-sparsity-in-deep-neural-networks},
}

@article{DBLP:journals/corr/IandolaMAHDK16,
  author    = {F. N. Iandola and
               M. W. Moskewicz and
               K. Ashraf and
               S. Han and
               W. J. Dally and
               K. Keutzer},
  title     = {SqueezeNet: AlexNet-level accuracy with 50x fewer parameters and {\textless}1MB
               model size},
  journal   = {CoRR},
  volume    = {abs/1602.07360},
  year      = {2016},
  url       = {http://arxiv.org/abs/1602.07360},
}

@article{DBLP:journals/corr/CourbariauxB16,
  author    = {M. Courbariaux and
               Y. Bengio},
  title     = {BinaryNet: Training Deep Neural Networks with Weights and Activations
               Constrained to +1 or -1},
  journal   = {CoRR},
  volume    = {abs/1602.02830},
  year      = {2016},
}

@article{DBLP:journals/corr/VenkateshNM16,
  author    = {G. Venkatesh and
               E. Nurvitadhi and
               D. Marr},
  title     = {Accelerating Deep Convolutional Networks using low-precision and sparsity},
  journal   = {CoRR},
  volume    = {abs/1610.00324},
  year      = {2016},
  url       = {http://arxiv.org/abs/1610.00324},
}

@article{DBLP:journals/corr/GuptaAGN15,
  author    = {S. Gupta and
               A. Agrawal and
               K. Gopalakrishnan and
               P. Narayanan},
  title     = {Deep Learning with Limited Numerical Precision},
  journal   = {CoRR},
  volume    = {abs/1502.02551},
  year      = {2015},
  url       = {http://arxiv.org/abs/1502.02551},
}

@article{Molchanov2017VariationalDS,
  title={Variational Dropout Sparsifies Deep Neural Networks},
  author={D. Molchanov and A. Ashukha and D. Vetrov},
  journal={CoRR},
  year={2017},
  volume={abs/1701.05369}
}

@article{DBLP:journals/corr/LinTA15,
  author    = {D. Dexu Lin and
               S. S. Talathi and
               V. {Sreekanth Annapureddy}},
  title     = {Fixed Point Quantization of Deep Convolutional Networks},
  journal   = {CoRR},
  volume    = {abs/1511.06393},
  year      = {2015},
  url       = {http://arxiv.org/abs/1511.06393},
}

@article{DBLP:journals/corr/WuLWHC15,
  author    = {J. Wu and
               C. Leng and
               Y. Wang and
               Q. Hu and
               J. Cheng},
  title     = {Quantized Convolutional Neural Networks for Mobile Devices},
  journal   = {CoRR},
  volume    = {abs/1512.06473},
  year      = {2015},
  url       = {http://arxiv.org/abs/1512.06473},
}

@article{DBLP:journals/corr/MerollaAAEM16,
  author    = {P. Merolla and
               R. Appuswamy and
               J. V. Arthur and
               S. K. Esser and
               D. S. Modha},
  title     = {Deep neural networks are robust to weight binarization and other non-linear
               distortions},
  journal   = {CoRR},
  volume    = {abs/1606.01981},
  year      = {2016},
  url       = {http://arxiv.org/abs/1606.01981},
}

@article{DBLP:journals/corr/JaderbergVZ14,
  author    = {M. Jaderberg and
               A. Vedaldi and
               A. Zisserman},
  title     = {Speeding up Convolutional Neural Networks with Low Rank Expansions},
  journal   = {CoRR},
  volume    = {abs/1405.3866},
  year      = {2014},
  url       = {http://arxiv.org/abs/1405.3866},
}

@article{DBLP:journals/corr/IoannouRSCC15,
  author    = {Y. Ioannou and
               D. P. Robertson and
               J. Shotton and
               R. Cipolla and
               A. Criminisi},
  title     = {Training CNNs with Low-Rank Filters for Efficient Image Classification},
  journal   = {CoRR},
  volume    = {abs/1511.06744},
  year      = {2015},
  url       = {http://arxiv.org/abs/1511.06744},
}

@article{DBLP:journals/corr/TaiXWE15,
  author    = {C. Tai and
               T. Xiao and
               X. Wang and
               W. E},
  title     = {Convolutional neural networks with low-rank regularization},
  journal   = {CoRR},
  volume    = {abs/1511.06067},
  year      = {2015},
  url       = {http://arxiv.org/abs/1511.06067},
}

@inproceedings{conf/cvpr/LiuWFTP15,
  author = {Liu, B. and Wang, M. and Foroosh, H. and Tappen, M. F. and Pensky, M.},
  booktitle = {CVPR},
  title = {Sparse Convolutional Neural Networks.},
  url = {http://dblp.uni-trier.de/db/conf/cvpr/cvpr2015.html#LiuWFTP15},
  year = 2015
}

@inproceedings{WCBN17,
  author    = {S. Wang, H. Cai, J. Bilmes, W. Noble},
  title     = {Training Compressed Fully-Connected Networks with a Density-Diversity Penalty},
  booktitle   = {ICLR},
  year      = {2017},
}

@inproceedings{DBLP:conf/icassp/SainathKSAR13,
  author    = {T. N. Sainath and
               B. Kingsbury and
               V. Sindhwani and
               E. Arisoy and
               B. Ramabhadran},
  title     = {Low-rank matrix factorization for Deep Neural Network training with
               high-dimensional output targets},
  booktitle = {ICASSP},
  year      = {2013},
}

@inproceedings{Li13,
author = {J. Li},
title = {Restructuring of Deep Neural Network Acoustic Models with Singular Value Decomposition},
booktitle = {Interspeech},
year = {2013},
}

@inproceedings{sindhwani_15,
  author    = {V. Sindhwani and
               T. N. Sainath and
               S. Kumar},
  title     = {Structured Transforms for Small-Footprint Deep Learning},
  booktitle = {NIPS},
  year      = {2015},
  url       = {http://papers.nips.cc/paper/5869-structured-transforms-for-small-footprint-deep-learning},
}

@inproceedings{YAD2017,
    author = {Y. Jernite and \textbf{A. Choromanska} and D. Sontag},
    title = {Simultaneous Learning of Trees and Representations for Extreme Classification and Density Estimation},
    booktitle = {ICML},
    year      = {2017},
}

@article{Kearns95,
author = {Kearns, M. and Mansour, Y.},
year = {1995},
title = {On the Boosting Ability of Top-Down Decision Tree Learning Algorithms},
volume = {58},
journal = {Journal of Computer and System Sciences},
}

@inproceedings{AALWork2013,
  author    = {\textbf{A. Choromanska} and A. Agarwal and J. Langford},
  title     = {Extreme Multi Class Classification},
  booktitle = {NIPS Workshop: eXtreme Classification},
  year      = {2013},
}

@inproceedings{AISTATS2012_ChoromanskaM12, 
    Title = {Online Clustering with Experts}, 
    Url = {http://jmlr.csail.mit.edu/proceedings/papers/v22/choromanska12/choromanska12.pdf}, 
    Author = {\textbf{A. Choromanska} and C. Monteleoni}, 
    Year = {2012}, 
    Booktitle = {AISTATS} 
   }

@inproceedings{conf/alt/ChoromanskaJKMM13,
  author = {\textbf{A. Choromanska} and Jebara, T. and Kim, H. and Mohan, M. and Monteleoni, C.},
  booktitle = {ALT},
  title = {Fast Spectral Clustering via the Nyström Method.},
  url = {http://dblp.uni-trier.de/db/conf/alt/alt2013.html#ChoromanskaJKMM13},
  year = 2013
}

@inproceedings{DBLP:conf/aistats/ChoromanskaHMAL15,
  author    = {\textbf{A. Choromanska} and
               M. Henaff and
               M. Mathieu and
               G. {Ben Arous} and
               Y. LeCun},
  title     = {The Loss Surfaces of Multilayer Networks},
  booktitle = {AISTATS},
  year      = {2015},
}

@inproceedings{DBLP:conf/colt/ChoromanskaLA15,
  author    = {\textbf{A. Choromanska} and
               Y. LeCun and
               G. {Ben Arous}},
  title     = {Open Problem: The landscape of the loss surfaces of multilayer networks},
  booktitle = {COLT},
  year      = {2015},
}

@inproceedings{DBLP:conf/nips/JebaraC12,
  author    = {T. Jebara and
               \textbf{A. Choromanska}},
  title     = {Majorization for CRFs and Latent Likelihoods},
  booktitle = {NIPS},
  year      = {2012},
}

@inproceedings{DBLP:journals/corr/AravkinCKJ13,
  author    = {A. Y. Aravkin and
               \textbf{A. Choromanska} and
               D. Kanevsky and
               T. Jebara},
  title     = {Semistochastic Quadratic Bound Methods},
  booktitle   = {ICLR Workshop},
  year      = {2014},
  url       = {http://arxiv.org/abs/1309.1369},
}

@inproceedings{DBLP:conf/icml/ChoromanskaCBJK16,
  author    = {\textbf{A. Choromanska} and
               K. Choromanski and
               M. Bojarski and
               T. Jebara and
               S. Kumar and
               Y. LeCun},
  title     = {Binary embeddings with structured hashed projections},
  booktitle = {ICML},
  year      = {2016},
}

@inproceedings{DBLP:journals/corr/BojarskiCCFGMSS16,
  author    = {M. Bojarski and
               \textbf{A. Choromanska} and
               K. Choromanski and
               F. Fagan and
               C. Gouy{-}Pailler and
               A. Morvan and
               N. Sakr and
               T. Sarl{\'{o}}s and
               J. Atif},
  title     = {Structured adaptive and random spinners for fast machine learning
               computations},
  booktitle = {AISTATS},
  year      = {2017},
}

@inproceedings{DBLP:conf/alt/ChoromanskaCJM13,
  author    = {\textbf{A. Choromanska} and
               K. Choromanski and
               G. Jagannathan and
               C. Monteleoni},
  title     = {Differentially-Private Learning of Low Dimensional Manifolds},
  booktitle = {ALT},
  year      = {2013},
}

@article{DBLP:journals/tcs/ChoromanskaCJM16,
  author    = {\textbf{A. Choromanska} and
               K. Choromanski and
               G. Jagannathan and
               C. Monteleoni},
  title     = {Differentially-private learning of low dimensional manifolds},
  volume = {620},
  pages = {91-104},
  journal   = {Theor. Comput. Sci.},
  year      = {2016},
}

@inproceedings{CKJ12,
  author    = {\textbf{A. Choromanska} and D. Kanevsky and T. Jebara},
  title     = {Majorization for Deep Belief Networks},
  booktitle = {NIPS Workshop: Log-linear models},
  year      = {2012},
}

@article{FNC12:GraphTracking, 
   Author = {\textbf{A. Choromanska} and Chang, S.-F. and Yuste, R.},
   Title = {Automatic reconstruction of neural morphologies with multi-scale graph-based tracking},
   Journal = {Frontiers in Neural Circuits},
   Volume = {6:25},
   Year = {2012}
} 

@article{journals/jmlr/MukherjeeS13,
  author = {Mukherjee, I. and Schapire, R. E.},
  journal = {Journal of Machine Learning Research},
  pages = {437-497},
  title = {A theory of multiclass boosting.},
  volume = 14,
  year = 2013
}

@article{haeffele2015global,
	Author = {Haeffele, B. and Vidal, R.},
	journal = {CoRR},
	volume = {abs/1506.07540},
	Title = {Global Optimality in Tensor Factorization, Deep Learning, and Beyond},
	Year = {2015}}

@article{janzamin2015beating,
	Author = {Janzamin, M. and Sedghi, H. and Anandkumar, A.},
	Title = {{Beating the Perils of Non-Convexity: Guaranteed Training of Neural Networks using Tensor Methods}},
	journal = {CoRR},
	volume = {abs/1506.08473},
	Year = {2015}}

@inproceedings{mikolov2013distributed,
  title={Distributed representations of words and phrases and their compositionality},
  author={Mikolov, T. and Sutskever, I. and Chen, K. and Corrado, G. S. and Dean, J.},
  booktitle={NIPS},
  year={2013}
}

@article{ShaiSS2012,
 author = {Shalev-Shwartz, S.},
 title = {Online Learning and Online Convex Optimization},
 journal = {Found. Trends Mach. Learn.},
 volume = {4},
 number = {2},
 year = {2012},
 pages = {107--194}
} 

@ARTICLE{Bengio03aneural,
    author = {Y. Bengio and R. Ducharme and P. Vincent and C. Jauvin},
    title = {A Neural Probabilistic Language Model},
    journal = {Journal of Machine Learning Research},
    year = {2003},
    volume = {3},
    pages = {1137--1155}
}

@inproceedings{morin2005hierarchical,
    author = {Morin, F. and Bengio, Y.},
    booktitle = {AISTATS},
    title = {{Hierarchical probabilistic neural network language model}},
    year = {2005}
}

@incollection{NIPS2008_3583,
title = {A Scalable Hierarchical Distributed Language Model},
author = {Mnih, A. and G. E. Hinton},
booktitle = {NIPS},
year = {2009},
}

@inproceedings{Chen:1996:ESS:981863.981904,
 author = {Chen, S. F. and Goodman, J.},
 title = {An Empirical Study of Smoothing Techniques for Language Modeling},
 booktitle = {ACL},
 year = {1996}
} 

@article{Brown:1992:CNG:176313.176316,
 author = {Brown, P. F. and deSouza, P. V. and Mercer, R. L. and Pietra, V. J. Della and Lai, J. C.},
 title = {Class-based N-gram Models of Natural Language},
 journal = {Comput. Linguist.},
 volume = {18},
 number = {4},
 year = {1992},
 pages = {467--479},
 publisher = {MIT Press}
} 

@inproceedings{conf/interspeech/KneserN93,
  author = {Kneser, R. and Ney, H.},
  booktitle = {EUROSPEECH},
  title = {Improved clustering techniques for class-based statistical language modelling.},
  year = 1993
}

@INPROCEEDINGS{Niesler98comparisonof,
    author = {T.R. Niesler and E. W. D. Whittaker and P.C. Woodland},
    title = {Comparison Of Part-Of-Speech And Automatically Derived Category-Based Language Models For Speech Recognition},
    booktitle = {ICASSP},
    year = {1998},
}

@Inproceedings {export:163344,
author       = {G. Zweig and J. C. Platt and C. Meek and C. J.C. Burges
                and A. Yessenalina and Q. Liu},
title = {Computational approaches to sentence completion},
booktitle    = {ACL},
year         = {2012},
}

@inproceedings{conf/emnlp/GubbinsV13,
  author = {Gubbins, J. and Vlachos, A.},
  booktitle = {EMNLP},
  title = {Dependency Language Models for Sentence Completion.},
  year = 2013
}

@article{Bengio:2003:NPL:944919.944966,
 author = {Bengio, Y. and Ducharme, R. and V., Pascal and Janvin, C.},
 title = {A Neural Probabilistic Language Model},
 journal = {J. Mach. Learn. Res.},
 volume = {3},
 year = {2003},
 pages = {1137--1155},
 publisher = {JMLR.org}
} 

@INPROCEEDINGS{Mnih12afast,
    author = {A. Mnih and Y. W. Teh},
    title = {A fast and simple algorithm for training neural probabilistic language models},
    booktitle = {ICML},
    year = {2012}
}

@Inproceedings {export:175560,
author       = {T. Mikolov and A. Deoras and S. Kombrink and L. Burget and J. Honza
                Cernocky},
booktitle    = {INTERSPEECH},
title        = {Empirical Evaluation and Combination of Advanced Language Modeling Techniques},
year         = {2011}
}

@inproceedings{conf/icml/SocherLNM11,
  author = {Socher, R. and Lin, C. C.-Y. and Ng, A. Y. and Manning, C. D.},
  booktitle = {ICML},
  title = {Parsing Natural Scenes and Natural Language with Recursive Neural Networks.},
  year = 2011,
}

@inproceedings{Turian:2010:WRS:1858681.1858721,
 author = {Turian, J. and Ratinov, L. and Bengio, Y.},
 title = {Word Representations: A Simple and General Method for Semi-supervised Learning},
 booktitle = {ACL},
 year = {2010}
}

@inproceedings{Collobert:2008:UAN:1390156.1390177,
 author = {Collobert, R. and Weston, J.},
 title = {A Unified Architecture for Natural Language Processing: Deep Neural Networks with Multitask Learning},
 booktitle = {ICML},
 year = {2008}
} 

@inproceedings{maas2010probabilistic,
  author = {Maas, A. L. and Ng, A. Y.},
  booktitle = {Workshop on Deep Learning and Unsupervised Feature Learning, NIPS},
  title = {A probabilistic model for semantic word vectors},
  year = 2010
}

@inproceedings{conf/www/DjuricWRGB15,
  author = {Djuric, N. and Wu, H. and Radosavljevic, V. and Grbovic, M. and Bhamidipati, N.},
  booktitle = {WWW},
  title = {Hierarchical Neural Language Models for Joint Representation of Streaming Documents and their Content},
  year = 2015
}

@article{DBLP:journals/corr/MirowskiV15,
  author    = {P. Mirowski and
               A. Vlachos},
  title     = {Dependency Recurrent Neural Language Models for Sentence Completion},
  journal   = {CoRR},
  volume    = {abs/1507.01193},
  year      = {2015}
}

@inproceedings{conf/interspeech/MikolovKBCK10,
  author = {Mikolov, T. and Karafiát, M. and Burget, L. and Cernocký, J. and Khudanpur, S.},
  booktitle = {INTERSPEECH},
  title = {Recurrent neural network based language model.},
  year = 2010
}

@article{DBLP:journals/corr/TaiSM15,
  author    = {K. S. Tai and
               R. Socher and
               C. D. Manning},
  title     = {Improved Semantic Representations From Tree-Structured Long Short-Term
               Memory Networks},
  journal   = {CoRR},
  volume    = {abs/1503.00075},
  year      = {2015}
}

@article{DBLP:journals/corr/KumarISBEPOGS15,
  author    = {A. Kumar and
               O. Irsoy and
               J. Su and
               J. Bradbury and
               R. English and
               B. Pierce and
               P. Ondruska and
               I. Gulrajani and
               R. Socher},
  title     = {Ask Me Anything: Dynamic Memory Networks for Natural Language Processing},
  journal   = {CoRR},
  volume    = {abs/1506.07285},
  year      = {2015}
}

@inproceedings{Schwenk:2005:TNN:1220575.1220601,
 author = {Schwenk, H. and Gauvain, J.-L.},
 title = {Training Neural Network Language Models on Very Large Corpora},
 booktitle = {HLT},
 year = {2005}
} 

@INPROCEEDINGS{Bengio+Senecal-2003,
     author = {Bengio, Y. and S{\'{e}}n{\'{e}}cal, J.-S.},
      title = {Quick Training of Probabilistic Neural Nets by Importance Sampling},
  booktitle = {AISTATS},
       year = {2003}
}

@article{journals/tnn/BengioS08,
  author = {Bengio, Y. and Senecal, J.-S.},
  journal = {IEEE Trans. Neural Networks},
  pages = {713-722},
  title = {Adaptive Importance Sampling to Accelerate Training of a Neural Probabilistic Language Model.},
  volume = 19,
  year = 2008
}

@article{Breiman:2001:RF:570181.570182,
 author = {Breiman, L.},
 title = {Random Forests},
 journal = {Mach. Learn.},
 volume = {45},
 year = {2001},
 pages = {5--32},
 publisher = {Kluwer Academic Publishers}
} 

@book{ig,
  author = {Breiman, L. and Friedman, J. H. and Olshen, R. A. and Stone, C. J.},
  publisher = {CRC Press LLC, Boca Raton, Florida},
  title = {Classification and Regression Trees},
  year = 1984
}

@inproceedings{mnih2007three,
 author = {Mnih, A. and Hinton, G.},
 title = {Three New Graphical Models for Statistical Language Modelling},
 booktitle = {ICML},
 year = {2007}
} 

@incollection{JelMer80,
  author = {Jelinek, F. and Mercer, R. L.},
  booktitle = {Proceedings, Workshop on Pattern Recognition in Practice},
  pages = {381--397},
  publisher = {North Holland},
  title = {Interpolated estimation of {M}arkov source parameters from sparse
  data},
  year = 1980,
}

@inproceedings{Katz1987,
  author = {Katz, S. M.},
  booktitle = {IEEE Trans. on Acoustics, Speech and Singal proc.},
  pages = {400-401},
  title = {Estimation of probabilities from sparse data for the language model
	component of a speech recognizer},
  volume = {ASSP-35},
  year = 1987
}

@inproceedings{Pereira+Tishby+Lee:93a,
   author = {F. Pereira and N. Tishby and L. Lee},
   title = {Distributional Clustering Of {English} Words},
   year = {1993},
   booktitle = {ACL}
}

@inproceedings{conf/interspeech/KneserN93,
  author = {Kneser, R. and Ney, H.},
  booktitle = {EUROSPEECH},
  title = {Improved clustering techniques for class-based statistical language modelling.},
  year = 1993
}

@inproceedings{Baker:1998:DCW:290941.290970,
 author = {Baker, L. Douglas and McCallum, Andrew Kachites},
 title = {Distributional Clustering of Words for Text Classification},
 booktitle = {SIGIR},
 year = {1998},
} 

@inproceedings{conf/icassp/SchwenkG02,
  author = {Schwenk, H. and Gauvain, J.-L.},
  booktitle = {ICASSP},
  title = {Connectionist language modeling for large vocabulary continuous speech recognition},
  year = 2002
}

@inproceedings{SchwenkH04,
  author = {Schwenk, H.},
  booktitle = {IJCNN},
  title = {Efficient training of large neural networks for language modeling},
  year = 2004
}

@inproceedings{Xu03,
  author = {Xu, P. and Emami, A. and Jelinek, F.},
  booktitle = {EMNLP},
  title = {Training connectionist models for the structured language model},
  year = 2003
}

@misc{VowpalWabbit,
  author = {Langford, J. and Li, L. and Strehl, A.},
  howpublished = {\url{ http://hunch.net/~vw}},
  year = 2007
}
	
@article{AKM2016,
  author    = {\textbf{A. Choromanska} and K. Choromanski and M. Bojarski},
  title     = {On the boosting ability of top-down decision tree learning algorithm for multiclass classification},
  journal   = {CoRR},
  volume    = {abs/1605.05223},
  year      = {2016},
}

@article{AGNS2011,
  author    = {A. Azocar and J. Gimenez and K. Nikodem and J. L. Sanchez},
  title     = {On strongly midconvex functions},
  journal = {Opuscula Math.},
  number = 1,
  pages = {15-26},
  volume = 31,
  year = 2011
}

@inproceedings{Ram:2011:DET:2020408.2020507,
 author = {Ram, P. and Gray, A. G.},
 title = {Density Estimation Trees},
 booktitle = {KDD},
 year = {2011}
} 

@Book{           bishop:2006:PRML,
  author = 	 "C. M. Bishop",
  title = 	 "Pattern Recognition and Machine Learning",
  publisher = 	 "Springer",
  year = 	 "2006",
}

@inproceedings{37180,
title = {Wsabie: Scaling Up To Large Vocabulary Image Annotation},
author  = {J. Weston and S. Bengio and N. Usunier},
year  = 2011,
booktitle = {IJCAI}
}

@incollection{NIPS2015_5969,
title = {Sparse Local Embeddings for Extreme Multi-label Classification},
author = {Bhatia, K. and Jain, H. and Kar, P. and Varma, M. and Jain, P.},
booktitle = {NIPS},
year = {2015},
}

@inproceedings{collobert2011torch7,
  title={Torch7: A matlab-like environment for machine learning},
  author={Collobert, R. and Kavukcuoglu, K. and Farabet, C.},
  booktitle={BigLearn, NIPS Workshop},
  number={EPFL-CONF-192376},
  year={2011},
}

@article{gutmann2010noise,
 author = {Gutmann, M. U. and Hyv\"{a}rinen, A.},
 title = {Noise-contrastive Estimation of Unnormalized Statistical Models, with Applications to Natural Image Statistics},
 journal = {J. Mach. Learn. Res.},
 volume = {13},
 number = {1},
 year = {2012},
 pages = {307--361},
} 

@inproceedings{DBLP:conf/naacl/AndreasK15,
  author    = {J. Andreas and
               D. Klein},
  title     = {When and why are log-linear models self-normalizing?},
  booktitle = {{NAACL} {HLT}},
  year      = {2015},
}

@article{DBLP:journals/corr/GraveJCGJ16,
  author    = {E. Grave and
               A. Joulin and
               M. Ciss{\'{e}} and
               D. Grangier and
               H. J{\'{e}}gou},
  title     = {Efficient softmax approximation for GPUs},
  journal   = {CoRR},
  volume    = {abs/1609.04309},
  year      = {2016},
}

@inproceedings{export:255952,
    author = {Kontschieder, P. and Fiterau, M. and Criminisi, A. and Bul{\`o}, S. Rota},
    booktitle = {ICCV},
    title = {{Deep Neural Decision Forests}},
    year = {2015},
}

@inproceedings{DBLP:conf/nips/VincentBB15,
  author    = {P. Vincent and
               A. de Br{\'{e}}bisson and
               X. Bouthillier},
  title     = {Efficient Exact Gradient Update for training Deep Networks with Very
               Large Sparse Targets},
  booktitle = {NIPS},
  year      = {2015},
}

@inproceedings{DBLP:journals/corr/BrebissonV15,
  author    = {A. de Br{\'{e}}bisson and
               P. Vincent},
  title     = {An Exploration of Softmax Alternatives Belonging to the Spherical
               Loss Family},
  booktitle = {ICLR},
  year      = {2016},
}

@article{DBLP:journals/cacm/ThomeeSFENPBL16,
  author    = {Bart Thomee and
               David A. Shamma and
               Gerald Friedland and
               Benjamin Elizalde and
               Karl Ni and
               Douglas Poland and
               Damian Borth and
               Li{-}Jia Li},
  title     = {{YFCC100M:} the new data in multimedia research},
  journal   = {Commun. {ACM}},
  volume    = {59},
  number    = {2},
  pages     = {64--73},
  year      = {2016}
}

@article{DBLP:journals/corr/JoulinGBM16,
  author    = {Armand Joulin and
               Edouard Grave and
               Piotr Bojanowski and
               Tomas Mikolov},
  title     = {Bag of Tricks for Efficient Text Classification},
  journal   = {CoRR},
  volume    = {abs/1607.01759},
  year      = {2016}
}

@incollection{NIPS2015_5797, 
   title = {Path-SGD: Path-Normalized Optimization in Deep Neural Networks}, 
   author = {Neyshabur, B. and Salakhutdinov, R. R. and Srebro, N.}, 
   booktitle = {NIPS}, 
   year = {2015}, 
   url = {http://papers.nips.cc/paper/5797-path-sgd-path-normalized-optimization-in-deep-neural-networks.pdf} 
   }

@inproceedings{DBLP:journals/corr/KeskarMNST16,
  author    = {N. S. Keskar and
               D. Mudigere and
               J. Nocedal and
               M. Smelyanskiy and
               P. T. P. Tang},
  title     = {On Large-Batch Training for Deep Learning: Generalization Gap and
               Sharp Minima},
  booktitle = {ICLR},
  year      = {2017},
  url       = {http://arxiv.org/abs/1609.04836},
}

@InProceedings{SB2017,
  title = 	 {Sharp Minima Can Generalize For Deep Nets},
  author = 	 {L. Dinh, R. Pascanu, S. Bengio, Y. Bengio},
  booktitle = 	 {ICML},
  year = 	 {2017},
}

@misc{mnistlecun,
    author = {LeCun, Y. and Cortes, C.},
    title = {{The MNIST database of handwritten digits}},
    howpublished = {\url{http://yann.lecun.com/exdb/mnist/}},
}

@article{physics2007,
   author = "Berthier, L. and Biroli, G. and Bouchaud, J.-P. and Kob, W. and Miyazaki, K. and Reichman, D. R.",
   title = "Spontaneous and induced dynamic fluctuations in glass formers. I. General results and dependence on ensemble and dynamics",
   journal = "The Journal of Chemical Physics",
   year = "2007",
   volume = "126",
   number = "18", 
}

@book{citeulike:3359427,
    author = {van Laarhoven, P. J. and Aarts, E. H.},
    month = jun,
    publisher = {Springer},
    title = {{Simulated Annealing: Theory and Applications (Mathematics and Its Applications)}},
    url = {http://www.amazon.com/exec/obidos/redirect?tag=citeulike07-20\&path=ASIN/9027725136},
    year = {1987}
}

@incollection{2012distbelief,
 title ={Large Scale Distributed Deep Networks},
 author={Dean, J. and Corrado, G. and Monga, R. and Chen, K. and Devin, M. and Le, Q. and Mao, M. and Ranzato, M. and Senior, A. and Tucker, P. and Yang, K. and Ng, A.},
 booktitle = {NIPS},
 year = {2012},
}

@article{DBLP:journals/corr/BalakrishnanWY14,
  author    = {S. Balakrishnan and
               M. J. Wainwright and
               B. Yu},
  title     = {Statistical guarantees for the {EM} algorithm: From population to
               sample-based analysis},
  journal   = {The Annals of Statistics},
   volume = "45",
   number = "1", 
 pages = {77--120},
  year      = {2015},
}

@article{csiszar_information_1984,
    author = {Csisz\'{a}r, I. and Tusn\'{a}dy, G.},
    journal = {Statistics and Decisions},
    title = {{Information Geometry and Alternating minimization procedures}},
    volume = {Supplement Issue 1},
    year = {1984}
}

@article{celisse2016stability,
title={Stability revisited: new generalisation bounds for the Leave-one-Out},
author={Celisse, A. and Guedj, B.},
journal   = {CoRR},
volume = "abs/1608.06412",
year={2016}
}

@inproceedings{DBLP:conf/icml/Dziugaite018,
  author    = {G. K. Dziugaite and
               D. M. Roy},
  title     = {Entropy-SGD optimizes the prior of a PAC-Bayes bound: Generalization
               properties of Entropy-SGD and data-dependent priors},
  booktitle = {ICML},
  year      = {2018},
}

@inproceedings{DBLP:conf/uai/DziugaiteR17,
  author    = {G. K. Dziugaite and
               D. M. Roy},
  title     = {Computing Nonvacuous Generalization Bounds for Deep (Stochastic) Neural
               Networks with Many More Parameters than Training Data},
  booktitle = {UAI},
  year      = {2017},
}

@inproceedings{Yang2018,
author = {Yang, T. and Lin, Q. and Li, Z.},
year = {2018},
booktitle = {IJCAI},
title = {Unified Convergence Analysis of Stochastic Momentum Methods for Convex and Non-convex Optimization}
}

@incollection{NIPS2015_5751,
title = {Asynchronous Parallel Stochastic Gradient for Nonconvex Optimization},
author = {Lian, X. and Huang, Y. and Li, Y. and Liu, J.},
booktitle = {NIPS},
year = {2015},
}

@inproceedings{DBLP:journals/corr/abs-1808-10396,
  author    = {Y. Yan and
               T. Yang and
               Z. Li and
               Q. Lin and
               Y. Yang},
  title     = {A Unified Analysis of Stochastic Momentum Methods for Deep Learning},
year = {2018},
booktitle = {IJCAI},
}

@article{DBLP:journals/corr/abs-1809-04564,
  author    = {A. Ramezani{-}Kebrya and
               A. Khisti and
               B. Liang},
  title     = {On the Stability and Convergence of Stochastic Gradient Descent with
               Momentum},
  journal   = {CoRR},
  volume    = {abs/1809.04564},
  year      = {2018},
}

@article{doi:10.1137/120880811,
author = {Ghadimi, S. and Lan, G.},
title = {Stochastic First- and Zeroth-Order Methods for Nonconvex Stochastic Programming},
journal = {SIAM Journal on Optimization},
volume = {23},
number = {4},
pages = {2341-2368},
year = {2013},
}

@book{10.5555/2621980,
author = {Shalev-Shwartz, S. and Ben-David, S.},
title = {Understanding Machine Learning: From Theory to Algorithms},
year = {2014},
isbn = {1107057132},
publisher = {Cambridge University Press},
address = {USA}
}

@article{DBLP:journals/corr/MirzaO14,
  author    = {M. Mirza and S. Osindero},
  title     = {Conditional Generative Adversarial Nets},
  journal   = {CoRR},
  volume    = {abs/1411.1784},
  year      = {2014},
  url       = {http://arxiv.org/abs/1411.1784},
}


@article{bousquet2002stability,
  title={Stability and generalization},
  author={Bousquet, O. and Elisseeff, A.},
  journal={JMLR},
  volume={2},
  number={Mar},
  pages={499--526},
  year={2002}
}

@article{Boyd2011,
 author = {Boyd, S. and Parikh, N. and Chu, E. and Peleato, B. and Eckstein, J.},
 title = {Distributed Optimization and Statistical Learning via the Alternating Direction Method of Multipliers},
 journal = {Found. Trends Mach. Learn.},
 volume = {3},
 number = {1},
 year = {2011},
 pages = {1--122},
 numpages = {122},
}

@article{Shalev-Shwartz:2010:LSU:1756006.1953019,
 author = {Shalev-Shwartz, S. and Shamir, O. and Srebro, N. and Sridharan, K.},
 title = {Learnability, Stability and Uniform Convergence},
 journal = {J. Mach. Learn. Res.},
 volume = {11},
 year = {2010},
 pages = {2635--2670},
 url = {http://dl.acm.org/citation.cfm?id=1756006.1953019},
} 

@InProceedings{pmlr-v40-Ge15,
  title = 	 {Escaping From Saddle Points --- Online Stochastic Gradient for Tensor Decomposition},
  author = 	 {R. Ge and F. Huang and C. Jin and Y. Yuan},
  booktitle = 	 {COLT},
  year = 	 {2015},
  url = 	 {http://proceedings.mlr.press/v40/Ge15.html},
}

@InProceedings{pmlr-v49-anandkumar16,
  title = 	 {Efficient approaches for escaping higher order saddle points  in non-convex optimization},
  author = 	 {A. Anandkumar and R. Ge},
  booktitle = 	 {COLT},
  year = 	 {2016},
  url = 	 {http://proceedings.mlr.press/v49/anandkumar16.html},
}

@article{JGNKJ2017,
  author    = {C. Jin, R. Ge, P. Netrapalli, S. M. Kakade, M. I. Jordan},
  title     = {How to Escape Saddle Points Efficiently},
  journal   = {CoRR},
  volume    = {abs/1703.00887},
  year      = {2017},
}

@InProceedings{DBLP:journals/corr/GoodfellowSS14,
  author    = {I. J. Goodfellow and
               J. Shlens and
               C. Szegedy},
  title     = {Explaining and Harnessing Adversarial Examples},
  booktitle   = {ICLR},
  year      = {2015},
}

@article{Rumelhart:1986we,
  author = {Rumelhart, D. E. and Hinton, G. E. and Williams, R. J.},
  journal = {Nature},
  number = 6088,
  pages = {533--536},
  title = {{Learning Representations by Back-propagating Errors}},
  volume = 323,
  year = 1986
}

@article{DBLP:journals/corr/abs-1212-5701, 
author = {M. D. Zeiler}, 
title = {{ADADELTA:} An Adaptive Learning Rate Method}, journal = {CoRR}, 
volume = {abs/1212.5701}, 
year = {2012}, 
}

@article{journals/nn/Qian99,
 author = {Qian, N.},
 journal = {Neural Networks},
 number = 1,
 pages = {145-151},
 title = {On the momentum term in gradient descent learning algorithms.},
 volume = 12,
 year = 1999
}

@article{Nesterov2005SmoothMO,
  title={Smooth minimization of non-smooth functions},
  author={Y. Nesterov},
  journal={Mathematical Programming},
  year={2005},
  volume={103},
  pages={127-152}
}

@InProceedings{ICML2011Saxe_551,
  author =    {A. Saxe and P. W. Koh and Z. Chen and M. Bhand and B. Suresh and A. Ng},
  title =     {On Random Weights and Unsupervised Feature Learning },
  booktitle = {ICML},
  year =      {2011},
}

@inproceedings{conf/iccv/JarrettKRL09,
  author = {Jarrett, K. and Kavukcuoglu, K. and Ranzato, M. and LeCun, Y.},
  booktitle = {ICCV},
  title = {What is the best multi-stage architecture for object recognition?},
  year = 2009
}

@article{journals/ploscb/PintoDDC09,
  author = {Pinto, N. and Doukhan, D. and DiCarlo, J. J. and Cox, D. D.},
  journal = {PLoS Computational Biology},
  number = 11,
  title = {A High-Throughput Screening Approach to Discovering Good Forms of Biologically Inspired Visual Representation.},
  volume = 5,
  year = 2009
}

@inproceedings{ pinto:bionetics_2010,
    author={Pinto, N. and Cox, D. D.},
    title={{An Evaluation of the Invariance Properties of a Biologically-Inspired System for Unconstrained Face Recognition}},
    booktitle={BIONETICS},
    year={2010}
}

@article{Huang2006489,
title = "Extreme learning machine: Theory and applications ",
journal = "Neurocomputing ",
volume = "70",
number = "1–3",
pages = "489 - 501",
year = "2006",
author = "G.-B. Huang and Q.-Y. Zhu and C.-K. Siew",
}

@inproceedings{Moczulski2016,
  title={ACDC: A Structured Efficient Linear Layer},
  author={Moczulski, M. and Denil, M. and Appleyard, J. and de Freitas, N.},
  booktitle= {ICLR},
  year={2016},
}

@article{tyree,
  author    = {W. Chen and
                 J. T. Wilson and
                  S. Tyree and
                  K. Q. Weinberger and 
                 Y. Chen},
  title     = {Compressing Neural Networks with the Hashing Trick},
  journal   = {CoRR},
  volume    = {abs/1504.04788},
  year      = {2015},
}

@inproceedings{felix,
  author    = {F. X. Yu and
               S. Kumar and
               Y. Gong and
               S.{-}F. Chang},
  title     = {Circulant Binary Embedding},
  booktitle = {ICML},
  year      = {2014},
}

@inproceedings{indyk2015,
  author    = {A. Andoni and
               P. Indyk and
               T. Laarhoven and
               I. P. Razenshteyn and
               L. Schmidt},
  title     = {Practical and Optimal {LSH} for Angular Distance},
  booktitle = {NIPS},
  year      = {2015},
}

@misc{ARL,
author = "{US Army Research Laboratory}",
title = "{Technical Implementation Plan 2016 - 2020}",
year =  {2016},
howpublished = {\url{https://defenseinnovationmarketplace.dtic.mil/wp-content/uploads/2018/02/ARL_Technical_Implementation_Plan_2016-2020_FINAL.pdf}},
}

@misc{AChoromaweb,
title = "{\textbf{A. Choromanska} website}",
howpublished = {\url{http://cims.nyu.edu/~achoroma}},
}


@inproceedings{DBLP:journals/corr/BojarskiCCFJMZ16,
  author    = {M. Bojarski and
               \textbf{A. Choromanska} and
               K. Choromanski and
               B. Firner and
               L. D. Jackel and
               U. Muller and
               K. Zieba},
  title     = {VisualBackProp: efficient visualization of CNNs},
  booktitle   = {ICRA},
  year      = {2018},
}

@inproceedings{lalala,
author = {S. Minaee and Y. Wang and \textbf{A. Choromanska} and S. Chung and X. Wang and E. Fieremans and S. Flanagan and J. Rath and Y. W. Lui},
title = "{A Deep Unsupervised Learning Approach Toward MTBI Identification Using Diffusion MRI}", booktitle = {EMBC}, 
year = {2018},
}

@inproceedings{lalalala,
author = {N. Patel and A. Nandini Saridena and \textbf{A. Choromanska} and P. Krishnamurthy and F. Khorrami},
title = "{Adversarial Learning Based On-Line Anomaly Monitoring for Assured Autonomy}",
booktitle = {IROS}, 
year = {2018},
}

@inproceedings{SMF17,
  author    = {N. Patel and \textbf{A. Choromanska} and P. Krishnamurthy and F. Khorrami},
  title     = "{Sensor Modality Fusion with CNNs for UGV Autonomous Driving in Indoor Environments}",
  booktitle = {IROS},
  year      = {2017},
  howpublished = {\url{http://cims.nyu.edu/~achoroma/NonFlash/Papers/SMF_CNN.pdf}},
}

@article{DBLP:journals/corr/MirzaO14,
  author    = {M. Mirza and S. Osindero},
  title     = {Conditional Generative Adversarial Nets},
  journal   = {CoRR},
  volume    = {abs/1411.1784},
  year      = {2014},
  url       = {http://arxiv.org/abs/1411.1784},
}

@article{DBLP:journals/corr/abs-1708-06374,
  author    = {S. Shalev{-}Shwartz and S. Shammah and A. Shashua},
  title     = {On a Formal Model of Safe and Scalable Self-driving Cars},
  journal   = {CoRR},
  volume    = {abs/1708.06374},
  year      = {2017},
  url       = {http://arxiv.org/abs/1708.06374},
}

@InProceedings{DBLP:journals/corr/abs-1810-02281,
  author    = {S. Arora and
               N. Cohen and
               N. Golowich and
               W. Hu},
  title     = {A Convergence Analysis of Gradient Descent for Deep Linear Neural
               Networks},
  booktitle   = {ICLR},
  year      = {2019}
}

@InProceedings{pmlr-v80-du18a,
  title = 	 {On the Power of Over-parametrization in Neural Networks with Quadratic Activation},
  author = 	 {Du, S. and Lee, J.},
  booktitle = 	 {ICML},
  year = 	 {2018},
}

@inproceedings{Safran:2016:QIB:3045390.3045473,
 author = {Safran, I. and Shamir, O.},
 title = {On the Quality of the Initial Basin in Overspecified Neural Networks},
 booktitle = {ICML},
 year = {2016},
} 


@InProceedings{pmlr-v70-ge17a,
  title = 	 {No Spurious Local Minima in Nonconvex Low Rank Problems: A Unified Geometric Analysis},
  author = 	 {R. Ge and C. Jin and Y. Zheng},
  booktitle = 	 {ICML},
  year = 	 {2017},
}

@inproceedings{DBLP:journals/corr/YunSJ17,
  author    = {C. Yun and
               S. Sra and
               A. Jadbabaie},
  title     = {Global optimality conditions for deep neural networks},
  booktitle = 	 {ICLR},
  year = 	 {2018},
}

@InProceedings{pmlr-v80-draxler18a,
  title = 	 {Essentially No Barriers in Neural Network Energy Landscape},
  author = 	 {Draxler, F. and Veschgini, K. and Salmhofer, M. and Hamprecht, F.},
  booktitle = 	 {ICML},
  year = 	 {2018},
}

@InProceedings{pmlr-v80-baity-jesi18a,
  title = 	 {Comparing Dynamics: Deep Neural Networks versus Glassy Systems},
  author = 	 {Baity-Jesi, M. and Sagun, L. and Geiger, M. and Spigler, S. and {Ben Arous}, G. and Cammarota, C. and LeCun, Y. and Wyart, M. and Biroli, G.},
  booktitle = 	 {ICML},
  year = 	 {2018},
}

@inproceedings{MoosaviDezfooli2017UniversalAP,
  title={Universal Adversarial Perturbations},
  author={S.-M. Moosavi-Dezfooli and A. Fawzi and O. Fawzi and P. Frossard},
  booktitle={CVPR},
  year={2017},
}

@article{DBLP:journals/corr/Moosavi-Dezfooli17,
  author    = {S.{-}M. Moosavi{-}Dezfooli and
               A. Fawzi and
               O. Fawzi and
               P. Frossard and
               S. Soatto},
  title     = {Analysis of universal adversarial perturbations},
  journal   = {CoRR},
  volume    = {abs/1705.09554},
  year      = {2017}
}

@article{DBLP:journals/corr/abs-1901-06053,
  author    = {U. Simsekli and
               L. Sagun and
               M. G{\"{u}}rb{\"{u}}zbalaban},
  title     = {A Tail-Index Analysis of Stochastic Gradient Noise in Deep Neural
               Networks},
  journal   = {CoRR},
  volume    = {abs/1901.06053},
  year      = {2019}
}

@inproceedings{DBLP:journals/corr/SagunEGDB17,
  author    = {L. Sagun and
               U. Evci and
               V. Ugur G{\"{u}}ney and
               Y. Dauphin and
               L. Bottou},
  title     = {Empirical Analysis of the Hessian of Over-Parametrized Neural Networks},
  booktitle = {ICLR Workshop},
  year      = {2018}
}

@inproceedings{DBLP:journals/corr/abs-1810-09665,
  author    = {S. Spigler and
               M. Geiger and
               S. d'Ascoli and
               L. Sagun and
               G. Biroli and
               M. Wyart},
  title     = {A jamming transition from under- to over-parametrization affects loss
               landscape and generalization},
  booktitle = {Integration of Deep Learning Theories, NeurIPS Workshop},
  year      = {2018},
}

@article{DBLP:journals/corr/abs-1809-09349,
  author    = {M. Geiger and
               S. Spigler and
               S. d'Ascoli and
               L. Sagun and
               M. Baity{-}Jesi and
               G. Biroli and
               M. Wyart},
  title     = {The jamming transition as a paradigm to understand the loss landscape
               of deep neural networks},
  journal   = {CoRR},
  year      = {2018},
}

@Article{C7CP01108C,
author ="Ballard, A. J. and Das, R. and Martiniani, S. and Mehta, D. and Sagun, L. and Stevenson, J. D. and Wales, D. J.",
title  ="Energy landscapes for machine learning",
journal  ="Phys. Chem. Chem. Phys.",
year  ="2017",
volume  ="19",
pages  ="12585-12603",}


@inproceedings{DBLP:journals/corr/HardtM16,
  author    = {M. Hardt and
               T. Ma},
  title     = {Identity Matters in Deep Learning},
  booktitle = {ICLR},
  year      = {2017}
}

@article{DBLP:journals/corr/abs-1712-08968,
  author    = {I. Safran and
               O. Shamir},
  title     = {Spurious Local Minima are Common in Two-Layer ReLU Neural Networks},
  journal   = {CoRR},
  volume    = {abs/1712.08968},
  year      = {2017}
}

@article{DBLP:journals/corr/abs-1712-04741,
  author    = {R. Vidal and
               J Bruna and
               R. Giryes and
               S. Soatto},
  title     = {Mathematics of Deep Learning},
  journal   = {CoRR},
  volume    = {abs/1712.04741},
  year      = {2017}
}

@inproceedings{Nguyen2017TheLS,
  title={The loss surface of deep and wide neural networks},
  author={Q. N. Nguyen and M. Hein},
  booktitle={ICML},
  year={2017}
}

@InProceedings{pmlr-v70-brutzkus17a,
  title = 	 {Globally Optimal Gradient Descent for a {C}onv{N}et with {G}aussian Inputs},
  author = 	 {A. Brutzkus and A. Globerson},
  booktitle = 	 {ICML},
  year = 	 {2017},}
}

@incollection{NIPS2017_6662,
title = {Convergence Analysis of Two-layer Neural Networks with ReLU Activation},
author = {Li, Y. and Yuan, Y.},
booktitle = {NIPS},
year = {2017},
}

@incollection{NIPS2018_7321,
title = {Algorithmic Regularization in Learning Deep Homogeneous Models: Layers are Automatically Balanced},
author = {Du, S. S. and Hu, W. and Lee, J. D.},
booktitle = {NIPS},
year = {2018},
}


@inproceedings{Li2018AlgorithmicRI,
  title={Algorithmic Regularization in Over-parameterized Matrix Sensing and Neural Networks with Quadratic Activations},
  author={Y. Li and T. Ma and H. Zhang},
  booktitle={COLT},
  year={2018}
}

@InProceedings{DBLP:journals/corr/abs-1710-10174,
  author    = {A. Brutzkus and
               A. Globerson and
               E. Malach and
               S. Shalev{-}Shwartz},
  title     = {{SGD} Learns Over-parameterized Networks that Provably Generalize
               on Linearly Separable Data},
booktitle = {ICLR},
year = {2018},
}

@inproceedings{Tian2017AnAF,
  title={An Analytical Formula of Population Gradient for two-layered ReLU network and its Applications in Convergence and Critical Point Analysis},
  author={Y. Tian},
  booktitle={ICML},
  year={2017}
}

@InProceedings{DBLP:journals/corr/ZhongS0BD17,
  author    = {K. Zhong and
               Z. Song and
               P. Jain and
               P. L. Bartlett and
               I. S. Dhillon},
  title     = {Recovery Guarantees for One-hidden-layer Neural Networks},
  booktitle = {ICML},
  year      = {2017}
}

@article{jacobs1991adaptive,
  title={Adaptive mixtures of local experts},
  author={Jacobs, Robert A and Jordan, Michael I and Nowlan, Steven J and Hinton, Geoffrey E},
  journal={Neural computation},
  volume={3},
  number={1},
  pages={79--87},
  year={1991},
  publisher={MIT Press}
}

@inproceedings{collobert2002parallel,
  title={A parallel mixture of SVMs for very large scale problems},
  author={Collobert, Ronan and Bengio, Samy and Bengio, Yoshua},
  booktitle={NIPS},
  year={2002}
}

@inproceedings{schlosser2016fusing,
  title={Fusing lidar and images for pedestrian detection using convolutional neural networks},
  author={Schlosser, Joel and Chow, Christopher K and Kira, Zsolt},
  booktitle={ICRA},
  year={2016},
}

@inproceedings{mees2016choosing,
  title={Choosing smartly: Adaptive multimodal fusion for object detection in changing environments},
  author={Mees, Oier and Eitel, Andreas and Burgard, Wolfram},
  booktitle={IROS},
  year={2016},
}

@article{huval2015empirical,
  title={An empirical evaluation of deep learning on highway driving},
  author={Huval, Brody and Wang, Tao and Tandon, Sameep and Kiske, Jeff and Song, Will and Pazhayampallil, Joel and Andriluka, Mykhaylo and Rajpurkar, Pranav and Migimatsu, Toki and Cheng-Yue, Royce and others},
  journal={arXiv:1504.01716},
  year={2015}
}

@inproceedings{chen2015deepdriving,
  title={Deepdriving: Learning affordance for direct perception in autonomous driving},
  author={Chen, Chenyi and Seff, Ari and Kornhauser, Alain and Xiao, Jianxiong},
  booktitle={ICCV},
  year={2015},
}

@article{eigen2013learning,
  title={Learning factored representations in a deep mixture of experts},
  author={Eigen, David and Ranzato, Marc'Aurelio and Sutskever, Ilya},
  journal={arXiv preprint arXiv:1312.4314},
  year={2013}
}

@article{jordan1994hierarchical,
  title={Hierarchical mixtures of experts and the EM algorithm},
  author={Jordan, Michael I and Jacobs, Robert A},
  journal={Neural computation},
  volume={6},
  number={2},
  pages={181--214},
  year={1994},
  publisher={MIT Press}
}

@inproceedings{pomerleau1989alvinn,
  title={Alvinn: An autonomous land vehicle in a neural network},
  author={Pomerleau, Dean A},
  booktitle={NIPS},
  year={1989}
}

@article{shazeer2017outrageously,
  title={Outrageously large neural networks: The sparsely-gated mixture-of-experts layer},
  author={Shazeer, Noam and Mirhoseini, Azalia and Maziarz, Krzysztof and Davis, Andy and Le, Quoc and Hinton, Geoffrey and Dean, Jeff},
  journal={arXiv:1701.06538},
  year={2017}
}

@article{lecun1998gradient,
  title={Gradient-based learning applied to document recognition},
  author={LeCun, Yann and Bottou, L{\'e}on and Bengio, Yoshua and Haffner, Patrick},
  journal={Proceedings of the IEEE},
  volume={86},
  number={11},
  pages={2278--2324},
  year={1998},
  publisher={IEEE}
}


@inproceedings{muller2006off,
  title={Off-road obstacle avoidance through end-to-end learning},
  author={Muller, Urs and Ben, Jan and Cosatto, Eric and Flepp, Beat and Cun, Yann L},
  booktitle={NIPS},
  year={2006}
}

@InProceedings{pmlr-v97-tan19a,
  title = {{E}fficient{N}et: Rethinking Model Scaling for Convolutional Neural Networks},
  author = {Tan, M. and Le, Q.},
  booktitle = {ICML},
  year = {2019},
}

@article{bojarski2017explaining,
  title={Explaining how a deep neural network trained with end-to-end learning steers a car},
  author={Bojarski, M. and Yeres, P. and \textbf{A. Choromanska} and Choromanski, K. and Firner, B. and Jackel, L. and Muller, U.},
  journal = {CoRR},
  volume = {abs/1704.07911},
  year={2017}
}

@article{bojarski2016end,
  title={End to end learning for self-driving cars},
  author={Bojarski, Mariusz and Del Testa, Davide and Dworakowski, Daniel and Firner, Bernhard and Flepp, Beat and Goyal, Prasoon and Jackel, Lawrence D and Monfort, Mathew and Muller, Urs and Zhang, Jiakai and others},
  journal={arXiv preprint arXiv:1604.07316},
  year={2016}
}

@inproceedings{patel2017sensor,
  title={Sensor Modality Fusion with CNNs for UGV Autonomous Driving in Indoor Environments},
  author={Patel, Naman and Choromanska, Anna and Krishnamurthy, Prashanth and Khorrami, Farshad},
  booktitle={IROS},
  year={2017}
}

@article{stiller2011information,
  title={Information fusion for automotive applications--An overview},
  author={Stiller, Christoph and Le{\'o}n, Fernando Puente and Kruse, Marco},
  journal={Information fusion},
  volume={12},
  number={4},
  pages={244--252},
  year={2011},
  publisher={Elsevier}
}

@inproceedings{chen2017multi,
  title={Multi-view 3d object detection network for autonomous driving},
  author={Chen, Xiaozhi and Ma, Huimin and Wan, Ji and Li, Bo and Xia, Tian},
  booktitle={CVPR},
  year={2017}
}
@inproceedings{szegedy2015going,
  title={Going deeper with convolutions},
  author={Szegedy, Christian and Liu, Wei and Jia, Yangqing and Sermanet, Pierre and Reed, Scott and Anguelov, Dragomir and Erhan, Dumitru and Vanhoucke, Vincent and Rabinovich, Andrew and others},
  year={2015},
  organization={CVPR}
}

@article{lin2013network,
  title={Network in network},
  author={Lin, Min and Chen, Qiang and Yan, Shuicheng},
  journal={arXiv:1312.4400},
  year={2013}
}

@article{bengio2015conditional,
  title={Conditional computation in neural networks for faster models},
  author={Bengio, Emmanuel and Bacon, Pierre-Luc and Pineau, Joelle and Precup, Doina},
  journal={arXiv:1511.06297},
  year={2015}
}

@inproceedings{bolukbasi2017adaptive,
  title={Adaptive neural networks for efficient inference},
  author={Bolukbasi, Tolga and Wang, Joseph and Dekel, Ofer and Saligrama, Venkatesh},
  booktitle={ICML},
  year={2017}
}

@inproceedings{MR14,
  author    = {M. D. Zeiler and
               R. Fergus},
  title     = {Visualizing and Understanding Convolutional Networks},
  booktitle = {ECCV},
  year      = {2014},
}

@inproceedings{MGR11,
  author    = {M. D. Zeiler and
               G. W. Taylor and
               R. Fergus},
  title     = {Adaptive deconvolutional networks for mid and high level feature learning},
  booktitle = {ICCV},
  year      = {2011},
}

@article{BachPLOS15,
    author = {Bach, S. and Binder, A. and Montavon, G. and Klauschen, F. and M{\"u}ller, K.-R. and Samek, W},
    title = "On Pixel-wise Explanations for Non-Linear Classifier Decisions by Layer-wise Relevance Propagation",
    journal = "PLOS ONE",
    number = "7",
    volume = "10",
    year = "2015",
    pages = "e0130140"
}

@inproceedings{DBLP:journals/corr/ArrasHMMS16,
  author    = {L. Arras and
               F. Horn and
               G. Montavon and
               K.{-}R. M{\"{u}}ller and
               W. Samek},
  title     = {Explaining Predictions of Non-Linear Classifiers in {NLP}},
  booktitle = {ACL Workshop on Representation Learning for NLP},
  year      = {2016},
}

@TECHREPORT{Erhan-vis-techreport-2010,
       author = {Erhan, D. and Courville, A. and Bengio, Y.},
        title = {Understanding Representations Learned in Deep Architectures},
       number = {1355},
         year = {2010},
  institution = {University of Montreal},
}

@TECHREPORT{visualization_techreport,
       author = {Erhan, D. and Bengio, Y. and Courville, A. and Vincent, P.},
        title = {Visualizing Higher-Layer Features of a Deep Network},
       number = {1341},
         year = {2009},
  institution = {University of Montreal},
}

@article{Hinton:2006:FLA:1161603.1161605,
 author = {Hinton, G. E. and Osindero, S. and Teh, Y.-W.},
 title = {A Fast Learning Algorithm for Deep Belief Nets},
 journal = {Neural Comput.},
 volume = {18},
 number = {7},
 year = {2006},
 pages = {1527--1554},
 numpages = {28},
} 

@article{DBLP:journals/neco/BerkesW06,
  author    = {P. Berkes and
               L. Wiskott},
  title     = {On the Analysis and Interpretation of Inhomogeneous Quadratic Forms
               as Receptive Fields},
  journal   = {Neural Computation},
  volume    = {18},
  number    = {8},
  pages     = {1868--1895},
  year      = {2006},
}

@inproceedings{DBLP:journals/corr/SimonyanVZ13,
  author    = {K. Simonyan and
               A. Vedaldi and
               A. Zisserman},
  title     = {Deep Inside Convolutional Networks: Visualising Image Classification
               Models and Saliency Maps},
  booktitle = {Workshop Proc. ICLR},
  year      = {2014},
}

@inproceedings{bach-arxiv15,
  author = {S. Bach and A. Binder and G. Montavon and K.-R. M{\"u}ller and W. Samek},
  title = {Analyzing Classifiers: Fisher Vectors and Deep Neural Networks},
booktitle = {CVPR},
         year = 2016,
}

@article{DBLP:journals/corr/SamekBMBM15,
  author    = {W. Samek and
               A. Binder and
               G. Montavon and
               S. Bach and
               K.{-}R. M{\"{u}}ller},
  title = {Evaluating the visualization of what a Deep Neural Network has learned},
  journal = {IEEE Transactions on Neural Networks and Learning Systems},
  year = {2016},
}

@inproceedings{girshick2014rcnn,
  Author    = {R. Girshick and
               J. Donahue and
               T. Darrell and
               J. Malik},
  Title     = {Rich feature hierarchies for accurate
               object detection and semantic segmentation},
  Booktitle = {CVPR},
  Year      = {2014}}

@inproceedings{journals/corr/SzegedyZSBEGF13,
  added-at = {2014-01-06T00:00:00.000+0100},
  author = {Szegedy, C. and Zaremba, W. and Sutskever, I. and Bruna, J. and Erhan, D. and Goodfellow, I. J. and Fergus, R.},
  title = {Intriguing properties of neural networks.},
  booktitle = {ICLR},
  year = 2014
}

@article{DBLP:journals/corr/GoodfellowSS14,
  author    = {I. J. Goodfellow and
               J. Shlens and
               C. Szegedy},
  title     = {Explaining and Harnessing Adversarial Examples},
  journal   = {CoRR},
  volume    = {abs/1412.6572},
  year      = {2014},
}

@inproceedings{DBLP:journals/corr/abs-1805-12076,
  author    = {B. Neyshabur and
               Z. Li and
               S. Bhojanapalli and
               Y. LeCun and
               N. Srebro},
  title     = {Towards Understanding the Role of Over-Parametrization in Generalization
               of Neural Networks},
  booktitle   = {ICLR},
  year      = {2018},
}

@article{DBLP:journals/corr/abs-1709-05011,
  author    = {Y. You and
               Z. Zhang and
               C.{-}J. Hsieh and
               J. Demmel},
  title     = {100-epoch ImageNet Training with AlexNet in 24 Minutes},
  journal   = {CoRR},
  volume    = {abs/1709.05011},
  year      = {2017},
}

@article{DBLP:journals/corr/NeyshaburBMS17,
  author    = {B. Neyshabur and
               S. Bhojanapalli and
               D. McAllester and
               N. Srebro},
  title     = {Exploring Generalization in Deep Learning},
  journal   = {CoRR},
  volume    = {abs/1706.08947},
  year      = {2017},
}

@InProceedings{DB16,
  author       = "A. Dosovitskiy and T.Brox",
  title        = "Inverting Visual Representations with Convolutional Networks",
  booktitle    = "CVPR",
  year         = "2016",
}

@article{Baehrens:2010:EIC:1756006.1859912,
 author = {Baehrens, D. and Schroeter, T. and Harmeling, S. and Kawanabe, M.i and Hansen, K. and M\"{u}ller, K.-R.},
 title = {How to Explain Individual Classification Decisions},
 journal = {J. Mach. Learn. Res.},
 volume = {11},
 year = {2010},
 pages = {1803--1831},
 numpages = {29},
} 

@article{Rasmussen_visualizationof,
    author = {P. M. Rasmussen and T. Schmah and K. H. Madsen and T. E. Lund and S. C. Strother and L. K. Hansen},
    title = {Visualization of nonlinear classification models in neuroimaging - signed sensitivity maps},
    journal = {BIOSIGNALS},
    pages = {254--263},
    year = {2012},
}

@article{DBLP:journals/corr/BojarskiTDFFGJM16,
  author    = {M. Bojarski and
               D. Del Testa and
               D. Dworakowski and
               B. Firner and
               B. Flepp and
               P. Goyal and
               L. D. Jackel and
               M. Monfort and
               U. Muller and
               J. Zhang and
               X. Zhang and
               J. Zhao and
               K. Zieba},
  title     = {End to End Learning for Self-Driving Cars},
  journal   = {CoRR},
  volume    = {abs/1604.07316},
  year      = {2016},
}

@inproceedings{conf/iccv/ChenSKX15,
  author = {Chen, C. and Seff, A. and Kornhauser, A. L. and Xiao, J.},
  booktitle = {ICCV},
  title = {DeepDriving: Learning Affordance for Direct Perception in Autonomous Driving.},
   year = 2015
}

@inproceedings{43850,
title = {Real-Time Pedestrian Detection With Deep Network Cascades},
author  = {A. Angelova and A. Krizhevsky and V. Vanhoucke and A. Ogale and D. Ferguson},
year  = 2015,
booktitle = {BMVC}
}

@article{LeCun:1989:BAH:1351079.1351090,
 author = {LeCun, Y. and Boser, B. and Denker, J. S. and Henderson, D. and Howard, R. E. and Hubbard, W. and Jackel, L. D.},
 title = {Backpropagation Applied to Handwritten Zip Code Recognition},
 journal = {Neural Comput.},
 volume = {1},
 number = {4},
 year = {1989},
 pages = {541--551},
 numpages = {11},
} 

@inproceedings{cvpr2016_zhou,
Author = {B. Zhou and A. Khosla and A. Lapedriza and A. Oliva and A. Torralba},
Title = {Learning Deep Features for Discriminative Localization},
Year = {2016},
Booktitle = {CVPR}
}

@article{DBLP:journals/corr/SelvarajuDVCPB16,
  author    = {R. R. Selvaraju and
               A. Das and
               R. Vedantam and
               M. Cogswell and
               D. Parikh and
               D. Batra},
  title     = {Grad-CAM: Why did you say that? Visual Explanations from Deep Networks
               via Gradient-based Localization},
  journal   = {CoRR},
  volume    = {abs/1610.02391},
  year      = {2016},
}

@INPROCEEDINGS{SprDosBroRied15,
  AUTHOR =       {J. T. Springenberg and A. Dosovitskiy and T. Brox and M. Riedmiller},
  TITLE =        {Striving for Simplicity: The All Convolutional Net},
  BOOKTITLE =    {ICLR 2015 Workshop Track},
  YEAR =         {2015},
}

@article{DBLP:journals/tkde/Robnik-SikonjaK08,
  author    = {M. Robnik{-}Sikonja and
               I. Kononenko},
  title     = {Explaining Classifications For Individual Instances},
  journal   = {{IEEE} Trans. Knowl. Data Eng.},
  volume    = {20},
  number    = {5},
  pages     = {589--600},
  year      = {2008},
}

@inproceedings{BinBacMonMueSam16,
    author = {Binder, A. and Bach, S. and Montavon, G. and M{\"u}ller, K.-R. and Samek, W.},
    title = "Layer-Wise Relevance Propagation for Deep Neural Network Architectures",
    booktitle = ICISA,
    year = "2016",
    pages = "913--922"
}

@inproceedings{DBLP:journals/corr/ZintgrafCW16,
  author    = {L. M. Zintgraf and
               T. S. Cohen and T. Adel and 
               M. Welling},
  title     = {Visualizing Deep Neural Network Decisions: Prediction Difference Analysis},
  booktitle = {ICLR},
  year      = {2017},
}

@inproceedings{He2016IdentityMI,
  title={Identity Mappings in Deep Residual Networks},
  author={K. He and X. Zhang and S. Ren and J. Sun},
  booktitle={ECCV},
  year={2016}
}

@INPROCEEDINGS{Deng09imagenet:a,
    author = {J. Deng and W. Dong and R. Socher and L.-j. Li and K. Li and L. Fei-fei},
    title = {Imagenet: A large-scale hierarchical image database},
    booktitle = {CVPR},
    year = {2009}
}

@article{Lin2014ARXIV,
    author = {Lin, T.-Y. and Maire, M. and Belongie, S. and Hays, J. and Perona, P. and Ramanan, D. and Doll\'{a}r, P. and Zitnick, C. L.},
    title = {Microsoft {COCO:} Common Objects in Context},
    url = {http://arxiv.org/abs/1405.0312},
    journal   = {CoRR},
    volume = {abs/1405.0312},
    year = {2014}
}

@Article{Everingham10, 
   author = "Everingham, M. and Van~Gool, L. and Williams, C. K. I. and Winn, J. and Zisserman, A.", 
   title = "The Pascal Visual Object Classes (VOC) Challenge", 
   journal = "International Journal of Computer Vision", 
   volume = "88", 
   year = "2010", 
   number = "2", 
   month = jun, 
   pages = "303--338", 
} 

@inproceedings{conf/cvpr/XiaoHEOT10,
  author = {Xiao, J. and Hays, J. and Ehinger, K. A. and Oliva, A. and Torralba, A.},
  booktitle = {CVPR},
  title = {SUN database: Large-scale scene recognition from abbey to zoo.},
  url = {http://dblp.uni-trier.de/db/conf/cvpr/cvpr2010.html#XiaoHEOT10},
  year = 2010
}

@misc{ODP,
  title = "{Netscape Communication Corporation. Open Directory
Project.}",
  howpublished = {\url{http://www.dmoz.org}},
}

@inproceedings{conf/sigir/BennettN09,
  author = {Bennett, P. N. and Nguyen, N.},
  booktitle = {SIGIR},
  title = {Refined experts: improving classification in large taxonomies.},
  year = 2009
}

@article{Thomee:2016:YND:2886013.2812802,
 author = {Thomee, B. and Shamma, D. A. and Friedland, G. and Elizalde, B. and Ni, K.l and Poland, D. and Borth, D. and Li, L.-J.},
 title = "{YFCC100M: The New Data in Multimedia Research}",
 journal = {Commun. ACM},
 issue_date = {February 2016},
 volume = {59},
 number = {2},
 month = jan,
 year = {2016},
 pages = {64--73},
 url = {http://doi.acm.org/10.1145/2812802},
} 

@misc{Gutenberg,
  title = "{Project Gutenberg}",
  howpublished = {\url{http://www.gutenberg.org/}},
}

@misc{Wikipedia,
  title = "{Wikipedia Database}",
  howpublished = {\url{https://en.wikipedia.org/wiki/Wikipedia:Database_download}},
}

@article{citeulike:13779585,
    author = {The 1000 Genomes Project Consortium},
    journal = {Nature},
    number = {7571},
    pages = {68--74},
    title = {{A global reference for human genetic variation}},
    url = {http://dx.doi.org/10.1038/nature15393},
    volume = {526},
    year = {2015}
}

@article{hinton12,
title = {Deep Neural Networks for Acoustic Modeling in Speech Recognition},
author  = {G. Hinton and L. Deng and D. Yu and G. Dahl and A.-r. Mohamed and N. Jaitly and A. Senior and V. Vanhoucke and P. Nguyen and T. Sainath and B. Kingsbury},
year  = 2012,
url = {https://www.microsoft.com/en-us/research/publication/deep-neural-networks-for-acoustic-modeling-in-speech-recognition/},
month = {November},
pages = {82-97},
journal = {IEEE Signal Processing Magazine},
volume = {29},
}

@inproceedings{DBLP:journals/corr/abs-1303-5778,
  author    = {A. Graves and
               A.l{-}r. M. and
               G. E. Hinton},
  title     = {Speech Recognition with Deep Recurrent Neural Networks},
  booktitle = {ICASSP},
  year      = {2013},
}

@inproceedings{Karpathy:2014:LVC:2679600.2680211,
 author = {Karpathy, A. and Toderici, G. and Shetty, S. and Leung, T. and Sukthankar, R. and Fei-Fei, L.},
 title = {Large-Scale Video Classification with Convolutional Neural Networks},
 booktitle = {CVPR},
 year = {2014},
 url = {http://dx.doi.org/10.1109/CVPR.2014.223},
} 

@misc{ILSVRC,
  title = "{ImageNet Large Scale Visual Recognition Challenge}",
  howpublished = {\url{http://image-net.org/challenges/LSVRC/2016/}},
}

@misc{COCO_comp,
  title = "{ Microsoft COCO Image Captioning Challenge}",
  howpublished = {\url{http://mscoco.org/dataset/#overview}},
}

@article{DBLP:journals/corr/BojarskiTDFFGJM16,
  author    = {M. Bojarski and
               D. Del Testa and
               D. Dworakowski and
               B. Firner and
               B. Flepp and
               P. Goyal and
               L. D. Jackel and
               M. Monfort and
               U. Muller and
               J. Zhang and
               X. Zhang and
               J. Zhao and
               K. Zieba},
  title     = {End to End Learning for Self-Driving Cars},
  journal   = {CoRR},
  volume    = {abs/1604.07316},
  year      = {2016},
  url       = {http://arxiv.org/abs/1604.07316},
}

@inproceedings{conf/iccv/ChenSKX15,
  author = {Chen, C. and Seff, A. and Kornhauser, A. L. and Xiao, J.},
  booktitle = {ICCV},
  title = {DeepDriving: Learning Affordance for Direct Perception in Autonomous Driving.},
  year = 2015,
}

@inproceedings{DBLP:journals/corr/Shalev-ShwartzS16a,
  author    = {S. Shalev{-}Shwartz and
               S. Shammah and
               A. Shashua},
  title     = {Safe, Multi-Agent, Reinforcement Learning for Autonomous Driving},
  booktitle = {NIPS Workshop on Learning, Inference and Control of Multi-Agent Systems},
  year      = {2016},
  url       = {http://arxiv.org/abs/1610.03295},
}

@article{Cortes:1995:SN:218919.218929,
 author = {Cortes, C. and Vapnik, V.},
 title = {Support-Vector Networks},
 journal = {Mach. Learn.},
 volume = {20},
 number = {3},
 year = {1995},
 pages = {273--297},
} 


@misc{TDLICML2018,
author = {R. Vidal and J. Bruna and R. Giryes},
  title = "{Theory of Deep Learning, Workshop at ICML}",
  year = {2018},
  howpublished = {\url{https://sites.google.com/site/deeplearningtheory/}},
}

@misc{IAS2019,
author = {S. Arora and J. Bruna and R. Ge and S. Gunasekar and J. Lee and B. Yu},
year = {2019},
  title = "{Theory of Deep Learning: Where next?, Workshop at the Institute for Advanced Study in Princeton}",
  howpublished = {\url{https://www.math.ias.edu/wtdl/}},
}

@inproceedings{DBLP:journals/corr/abs-1906-04349,
  author    = {A. Pacchiano and
               J. Parker{-}Holder and
               Y. Tang and
               \textbf{A. Choromanska} and
               K. Choromanski and
               M. I. Jordan},
  title     = {Learning to Score Behaviors for Guided Policy Optimization},
  booktitle = {ICML},
  year      = {2020},
}

@misc{BeyondBackProp,
author = {\textbf{A. Choromanska} and M. Gori and Y. Huang and S. Lowe and M. Malinowski and V. Patraucean and G. Swirszcz},
year = {2020},
  title = "{Beyond BackPropagation: Novel Ideas for Training Neural Architectures, Workshop at NeurIPS (Submitted)}",
}

@misc{ECMLPKDD2015,
  title = "{International Workshop on Big Multi-Target Prediction, ECML/PKDD'15}",
  howpublished = {\url{http://www.kermit.ugent.be/big-multi-target-prediction/index.php/}},
}

@misc{SSB2015,
  title = "{Summer School on Non-convex Optimization in Machine Learning at the Indian Institute of Technology, Bombay, India'15}",
  howpublished = {\url{https://www.cse.iitb.ac.in/~ganesh/noml2015/}},
}

@inproceedings{DBLP:journals/corr/GoodfellowV14,
  author    = {I. J. Goodfellow and
               O. Vinyals},
  title     = {Qualitatively characterizing neural network optimization problems},
  booktitle = {ICLR},
  year      = {2015},
  url       = {http://arxiv.org/abs/1412.6544},
}

@inproceedings{DBLP:conf/aistats/ChoromanskaHMAL15,
  author    = {\textbf{A. Choromanska} and
               M. Henaff and
               M. Mathieu and
               G. Ben Arous and
               Y. LeCun},
  title     = {The Loss Surfaces of Multilayer Networks},
  booktitle = {AISTATS},
  year      = {2015},
  url       = {http://jmlr.org/proceedings/papers/v38/choromanska15.html},
}

@article{DBLP:journals/corr/KingmaB14,
  author    = {D. P. Kingma and
               J. Ba},
  title     = {Adam: {A} Method for Stochastic Optimization},
  journal   = {CoRR},
  volume    = {abs/1412.6980},
  year      = {2014},
  url       = {http://arxiv.org/abs/1412.6980},
}

@article{DBLP:journals/corr/GongLYB14,
  author    = {Y. Gong and
               L. Liu and
               M. Yang and
               L. D. Bourdev},
  title     = {Compressing Deep Convolutional Networks using Vector Quantization},
  journal   = {CoRR},
  volume    = {abs/1412.6115},
  year      = {2014},
  url       = {http://arxiv.org/abs/1412.6115},
}

@inproceedings{DBLP:journals/corr/HanMD15,
  author    = {S. Han and
               H. Mao and
               W. J. Dally},
  title     = {Deep Compression: Compressing Deep Neural Network with Pruning, Trained
               Quantization and Huffman Coding},
  booktitle = {ICLR},
  year      = {2016},
}

@inproceedings{DBLP:conf/kdd/ChenWTWC16,
  author    = {W. Chen and
               J. T. Wilson and
               S. Tyree and
               K. Q. Weinberger and
               Y. Chen},
  title     = {Compressing Convolutional Neural Networks in the Frequency Domain},
  booktitle = {KDD},
  year      = {2016},
  url       = {http://doi.acm.org/10.1145/2939672.2939839},
}

@article{DBLP:journals/corr/SimonyanZ14a,
  author    = {K. Simonyan and
               A. Zisserman},
  title     = {Very Deep Convolutional Networks for Large-Scale Image Recognition},
  journal   = {CoRR},
  volume    = {abs/1409.1556},
  year      = {2014},
  url       = {http://arxiv.org/abs/1409.1556},
}

@article{Rifkin2004,
 author = {Rifkin, R. and Klautau, A.},
 title = {In Defense of One-Vs-All Classification},
 journal = {J. Mach. Learn. Res.},
 month = dec,
 volume = {5},
 year = {2004},
 pages = {101--141},
} 

@incollection{NIPS2008_3583,
title = {A Scalable Hierarchical Distributed Language Model},
author = {Mnih, A. and G. E. Hinton},
booktitle = {NIPS},
year = {2009},
}

@misc{NIPS2016W,
  author = {M. Cissé and M. Varma and S. Bengio},
  title = "{Multi-class and Multi-label Learning in Extremely Large Label Spaces, Workshop at NIPS}",
  year = {2016},
  howpublished = {\url{https://manikvarma.github.io/events/XC16/index.html}},
}

@misc{ICML2015W,
  title = "{The ICML Extreme Classification Workshop 2015}",
  howpublished = {\url{https://sites.google.com/site/extremeclassification/}},
}

@misc{EMCDU14,
  title = "{The Digital Universe of Opportunities: Rich Data and the Increasing Value of the Internet of Things - study conducted by the International Data Corporation in 2014}",
  howpublished = {\url{https://uk.emc.com/leadership/digital-universe/2014iview/executive-summary.htm}},
}

@inproceedings{BengioWG10,
  title = {Label Embedding Trees for Large Multi-Class Tasks.},
  author = {Bengio, S. and Weston, J. and Grangier, D.},
  booktitle = {NIPS},
  year = 2010
}

@article{DBLP:journals/corr/JoulinGBM16,
  author    = {A. Joulin and
               E. Grave and
               P. Bojanowski and
               T. Mikolov},
  title     = {Bag of Tricks for Efficient Text Classification},
  journal   = {CoRR},
  volume    = {abs/1607.01759},
  year      = {2016}
}

@article{journals/informaticaSI/MadzarovGC09,
  author = {Madzarov, G. and Gjorgjevikj, D. and Chorbev, I.},
  title = {A Multi-class SVM Classifier Utilizing Binary Decision Tree.},
  journal = {Informatica},
  number = 2,
  pages = {225-233},
  volume = 33,
  year = 2009
}

@inproceedings{weston13, 
    Title = {Label Partitioning For Sublinear Ranking},
    Booktitle = {ICML}, 
    Author = {J. Weston and A. Makadia and H. Yee}, 
    Year = {2013}, 
   }

@inproceedings{DengSBL11,
  title = {Fast and Balanced: Efficient Label Tree Learning for Large Scale Object Recognition.},
  author = {Deng, J. and Satheesh, S. and Berg, A. C. and Fei-Fei, L.},
  booktitle = {NIPS},
  year = 2011
}

@inproceedings{conf/cvpr/ZhaoX13,
  author = {Zhao, B. and Xing, E. P.},
  booktitle = {CVPR},
  title = {Sparse Output Coding for Large-Scale Visual Recognition.},
  year = 2013
}

@article{JL2016,
  author    = {H. Daume and N. Karampatziakis and J. Langford and P. Mineiro},
  title     = {Logarithmic Time One-Against-Some},
  journal   = {CoRR},
  volume    = {abs/1606.04988},
  year      = {2016},
}

@inproceedings{DBLP:journals/corr/abs-0902-1284,
  author = {Hsu, D. and Kakade, S. and Langford, J. and Zhang, T.},
  title = {Multi-Label Prediction via Compressed Sensing.},
  booktitle = {NIPS},
  year = 2009
}

@inproceedings{DBLP:journals/corr/AgarwalKKSV13,
  author    = {A. Agarwal and
               S. M. Kakade and
               N. Karampatziakis and
               L. Song and
               G. Valiant},
  title     = {Least Squares Revisited: Scalable Approaches for Multi-class Prediction},
  booktitle   = {ICML},
  year      = {2014}
}

@inproceedings{icml2014c2_beijbom14,
    Author = {O. Beijbom and M. Saberian and D. Kriegman and N. Vasconcelos},
    Title = {Guess-Averse Loss Functions For Cost-Sensitive Multiclass Boosting},
  booktitle   = {ICML},
  year      = {2014}
   } 

@inproceedings{BeygelzimerLR09,
  author = {Beygelzimer, A. and Langford, J. and Ravikumar, P. D.},
  booktitle = {ALT},
  title = {Error-Correcting Tournaments.},
  year = 2009
}

@incollection{NIPS2015_5937,
title = {Logarithmic Time Online Multiclass prediction},
author = {\textbf{A. Choromanska} and Langford, J.},
booktitle = {NIPS},
year = {2015},
}

@inproceedings{BeygelzimerLLSS09,
  author = {Beygelzimer, A. and Langford, J. and Lifshits, Y. and Sorkin, G. B. and Strehl, A. L.},
  booktitle = {UAI},
  title = {Conditional Probability Tree Estimation Analysis and Algorithms.},
  year = 2009
}

@inproceedings{Agrawal:2013:MLM:2488388.2488391,
 author = {Agrawal, R. and Gupta, A. and Prabhu, Y. and Varma, M.},
 title = {Multi-label Learning with Millions of Labels: Recommending Advertiser Bid Phrases for Web Pages},
 booktitle = {WWW},
 year = {2013},
} 

@inproceedings{Prabhu:2014:FFA:2623330.2623651,
 author = {Prabhu, Y. and Varma, M.},
 title = {FastXML: A Fast, Accurate and Stable Tree-classifier for Extreme Multi-label Learning},
 booktitle = {KDD},
 year = {2014},
} 

@inproceedings{conf/icml/WestonMY13,
  author = {Weston, J. and Makadia, A. and Yee, H.},
  booktitle = {ICML},
  title = {Label Partitioning For Sublinear Ranking.},
  year = 2013
}

@article{Takimoto:2003:TDT:763058.763066,
 author = {Takimoto, E. and Maruoka, A.},
 title = {Top-down Decision Tree Learning As Information Based Boosting},
 journal = {Theor. Comput. Sci.},
 volume = {292},
 number = {2},
 month = jan,
 year = {2003},
 pages = {447--464},
 url = {http://dx.doi.org/10.1016/S0304-3975(02)00181-0},
} 

@inproceedings{conf/icml/BalasubramanianL12,
  author = {Balasubramanian, K. and Lebanon, G.},
  booktitle = {ICML},
  title = {The Landmark Selection Method for Multiple Output Prediction.},
  year = 2012
}

@incollection{NIPS2015_5969,
title = {Sparse Local Embeddings for Extreme Multi-label Classification},
author = {Bhatia, K. and Jain, H. and Kar, P. and Varma, M. and Jain, P.},
booktitle = {NIPS},
year = {2015},
}

@inproceedings{icml2013_bi13, 
    Title = {Efficient Multi-label Classification with Many Labels}, 
    Booktitle = {ICML}, 
    Author = {W. Bi and J. Kwok}, 
    Year = {2013}, 
   }

@incollection{NIPS2012_4561,
title = {Feature-aware Label Space Dimension Reduction for Multi-label Classification},
author = {Y{-}n. Chen and H{-}t. Lin},
booktitle = {NIPS},
year = {2012},
}

@incollection{NIPS2013_5083,
title = {Robust Bloom Filters for Large MultiLabel Classification Tasks},
author = {Cisse, M. M. and Usunier, N. and Arti\`{e}res, T. and Gallinari, P.},
booktitle = {NIPS},
year = {2013},
}

@INPROCEEDINGS{Ferng11multi-labelclassification,
    author = {C{-}s. Ferng and H{-}t. Lin},
    title = {Multi-label Classification with Error-correcting Codes},
    booktitle = {ACML},
    year = {2011},
}

@incollection{NIPS2009_3824,
title = {Multi-Label Prediction via Compressed Sensing},
author = {Hsu, D. J. and Kakade, S. M. and Langford, J. and Zhang, T.},
booktitle = {NIPS},
year = {2009},
}

@inproceedings{Ji:2008:ESS:1401890.1401939,
 author = {Ji, S. and Tang, L. and Yu, S. and Ye, J.},
 title = {Extracting Shared Subspace for Multi-label Classification},
 booktitle = {KDD},
 year = {2008},
} 

@article{DBLP:journals/corr/KarampatziakisM15,
  author    = {N. Karampatziakis and
               P. Mineiro},
  title     = {Multilabel Prediction via Calibration},
  journal   = {CoRR},
  volume    = {abs/1502.02710},
  year      = {2015},
  url       = {http://arxiv.org/abs/1502.02710},
}

@inproceedings{icml2014c2_linc14, 
    Title = {Multi-label Classification via Feature-aware Implicit Label Space Encoding}, 
    Url = {http://jmlr.org/proceedings/papers/v32/linc14.pdf}, 
    Author = {Z. Lin and G. Ding and M. Hu and J. Wang}, 
    Year = {2014}, 
    Booktitle = {ICML}, 
   }

@inproceedings{DBLP:journals/corr/MineiroK15,
  author    = {P. Mineiro and
               N. Karampatziakis},
  title     = {Fast Label Embeddings for Extremely Large Output Spaces},
 booktitle = {ECML},
  year      = {2015},
}

@inproceedings{NIPS2015_5770, 
   title = {Large-Scale Bayesian Multi-Label Learning via Topic-Based Label Embeddings}, 
   author = {Rai, P. and Hu, C. and Henao, R. and Carin, L.}, 
   booktitle = {NIPS},  
   year = {2015}, 
   }

@inproceedings{Tai:2012:MCP:2344802.2344812,
 author = {Tai, F. and Lin, H.-T.},
 title = {Multilabel Classification with Principal Label Space Transformation},
booktitle = {MLD},  
 year = {2010},
} 

@inproceedings{DBLP:conf/icml/Yu0KD14,
  author    = {H.{-}F. Yu and
               P. Jain and
               P. Kar and
               I. S. Dhillon},
  title     = {Large-scale Multi-label Learning with Missing Labels},
  booktitle = {ICML},
  year      = {2014},
}

@inproceedings{cca,
    author = {Zhang, Y. and Schneider, J.},
    booktitle = {AISTATS},
    title = {{Multi-label Output Codes using Canonical Correlation Analysis}},
    year = {2011}
}

@InProceedings{Jain16,
  author    = "Jain, H. and Prabhu, Y. and Varma, M.",
  title     = "Extreme Multi-label Loss Functions for Recommendation, Tagging, Ranking and Other Missing Label Applications",
  booktitle = "KDD",
  year      = "2016",
}

@incollection{DBLP:journals/corr/DauphinPGCGB14,
  author    = {Y. Dauphin and
               R. Pascanu and
               {\c{C}}. G{\"{u}}l{\c{c}}ehre and
               K. Cho and
               S. Ganguli and
               Y. Bengio},
  title     = {Identifying and attacking the saddle point problem in high-dimensional
               non-convex optimization},
  booktitle = {NIPS},
  year      = {2014},
}

@incollection{DBLP:journals/corr/SaxeMG13,
  author    = {A. M. Saxe and
               J. L. McClelland and
               S. Ganguli},
  title     = {Exact solutions to the nonlinear dynamics of learning in deep linear
               neural networks},
  booktitle   = {ICLR},
  year      = {2014},
}

@article{PhysRevA.32.1007,
  title = {Spin-glass models of neural networks},
  author = {Amit, D. J. and Gutfreund, H. and Sompolinsky, H.},
  journal = {Phys. Rev. A},
  volume = {32},
  pages = {1007--1018},
  year = {1985},
}

@article{0305-4470-30-23-009,
  author={K. Nakanishi and H. Takayama},
  title={Mean-field theory for a spin-glass model of neural networks: TAP free energy and the paramagnetic to spin-glass transition},
  journal={Journal of Physics A: Mathematical and General},
  volume={30},
  pages={8085},
  year={1997},
}

@book{Dotsenko1995,
  author = {V. Dotsenko},
  title = {An Introduction to the Theory of Spin Glasses and Neural Networks},
  publisher = {World Scientific Lecture Notes in Physics},
  year = {1995},
}

@article{Baldi:1989:NNP:70359.70362,
	Author = {Baldi, P. and Hornik, K.},
	Journal = {Neural Networks},
	Pages = {53--58},
	Title = {Neural Networks and Principal Component Analysis: Learning from Examples Without Local Minima},
	Volume = {2},
	Year = {1989}}

@inproceedings{hardt2015train,
	Author = {Hardt, M. and Recht, B. and Singer, Y.},
  title     = {Train faster, generalize better: Stability of stochastic gradient
               descent},
  booktitle = {ICML},
  year      = {2016},
  url       = {http://jmlr.org/proceedings/papers/v48/hardt16.html},
}

@article{anandkumar2016efficient,
	Author = {Anandkumar, A. and Ge, R.},
	Title = {Efficient approaches for escaping higher order saddle points in non-convex optimization},
	Journal = {arXiv:1602.05908},
	Year = {2016}}

@inproceedings{sutskever2013importance,
	Author = {Sutskever, I. and Martens, J. and Dahl, G. and Hinton, G.},
	Booktitle = {ICML},
	Title = {On the importance of initialization and momentum in deep learning},
	Year = {2013}}

@article{duchi2011adaptive,
	Author = {Duchi, J. and Hazan, E. and Singer, Y.},
	Journal = {JMLR},
	Numpages = {39},
	Pages = {2121--2159},
	Title = {Adaptive subgradient methods for online learning and stochastic optimization},
	Volume = {12},
	Year = {2011}}

@article{ioffe2015batch,
	Author = {Ioffe, S. and Szegedy, C.},
	Title = {{Batch normalization: Accelerating deep network training by reducing internal covariate shift}},
	journal = {CoRR},
	volume = {abs/1502.03167},
	Year = {2015}}

@article{cooijmans2016recurrent,
	Author = {Cooijmans, T. and Ballas, N. and Laurent, C. and Courville, A.},
	Journal = {arXiv:1603.09025},
	Title = {Recurrent Batch Normalization},
	Year = {2016}}

@article{salimans2016weight,
	Author = {Salimans, T. and Kingma, D.},
	Journal = {arXiv:1602.07868},
	Title = {Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks},
	Year = {2016}}

@article{baldassi2015subdominant,
	Author = {Baldassi, C. and Ingrosso, A. and Lucibello, C. and Saglietti, L. and Zecchina, R.},
	Journal = {Physical review letters},
	Number = {12},
	Pages = {128101},
	Title = {Subdominant dense clusters allow for simple learning and high computational performance in neural networks with discrete synapses},
	Volume = {115},
	Year = {2015}}

@article{baldassi2016unreasonable,
	Title = {Unreasonable effectiveness of learning neural networks: {From} accessible states and robust ensembles to basic algorithmic schemes},
	Volume = {113},
	Number = {48},
	Journal = {PNAS},
	Author = {Baldassi, C. and Borgs, C. and Chayes, J. T. and Ingrosso, A. and Lucibello, C. and Saglietti, L. and Zecchina, R.},	
	Year = {2016},
	Pages = {E7655--E7662}}

@article{baldassi2016multilevel,
        Title = {Learning may need only a few bits of synaptic precision},
        Volume = {93},
        Number = {5},
        Journal = {Physical Review E},
        Author = {Baldassi, C. and Gerace, F. and Lucibello, C. and Saglietti, L. and Zecchina, R.},
        Year = {2016},
        Pages = {052313},
}

@article{zhang2016understanding,
  title={Understanding deep learning requires rethinking generalization},
  author={Zhang, C. and Bengio, S. and Hardt, M. and Recht, B. and Vinyals, O.},
  journal   = {CoRR},
  volume={arXiv:1611.03530},
  year={2016},
}

@article{DBLP:journals/corr/abs-1711-04291,
  author    = {V. Codreanu and
               D. Podareanu and
               V. A. Saletore},
  title     = {Scale out for large minibatch {SGD:} Residual network training on
               ImageNet-1K with improved accuracy and reduced time to train},
  journal   = {CoRR},
  volume    = {abs/1711.04291},
  year      = {2017},
}

 @article{goyal2017imagenet1hr,
    Author    = {P. Goyal and P. Doll\'{a}r and R. Girshick and
                 P. Noordhuis and L. Wesolowski and A. Kyrola and
                 A. Tulloch and Y. Jia and K. He},
    Title     = {Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour},
    journal = {CoRR},
    volume   = {abs/1706.02677},
    Year      = {2017}},
    
@inproceedings{conf/icml/WellingT11,
  author = {Welling, M. and Teh, Y. W.},
  booktitle = {ICML},
  title = {Bayesian Learning via Stochastic Gradient Langevin Dynamics.},
  year = 2011,
}

@inproceedings{laffertyCrf,
  author = {Lafferty, J. D. and McCallum, A. and Pereira, F. C. N.},
  booktitle = {ICML},
  title = {Conditional Random Fields: Probabilistic Models for Segmenting and Labeling Sequence Data},
  year = 2001
}

@book{10.5555/3086980,
author = {Lange, K.},
title = {MM Optimization Algorithms},
year = {2016},
publisher = {SIAM-Society for Industrial and Applied Mathematics},
address = {Philadelphia, PA, USA}
}

@inproceedings{Globerson2007ExponentiatedGA,
  title={Exponentiated gradient algorithms for log-linear structured prediction},
  author={A. Globerson and T. K. Koo and X. Carreras and M. Collins},
  booktitle={ICML},
  year={2007}
}

@article{DBLP:journals/corr/abs-1905-11286,
  author    = {B. Ginsburg and
               P. Castonguay and
               O. Hrinchuk and
               O. Kuchaiev and
               R. Leary and
               V. Lavrukhin and
               J. Li and
               H. Nguyen and
               Y. Zhang and 
               J. M. Cohen},
  title     = {Training Deep Networks with Stochastic Gradient Normalized by Layerwise
Adaptive Second Moments},
  journal   = {CoRR},
  volume    = {abs/1905.11286},
  year      = {2019},
}

@article{DBLP:journals/corr/abs-1804-04235,
  author    = {N. Shazeer and
               M. Stern},
  title     = {Adafactor: Adaptive Learning Rates with Sublinear Memory Cost},
  journal   = {CoRR},
  volume    = {abs/1804.04235},
  year      = {2018},
}

@inproceedings{DBLP:journals/corr/abs-1902-09843,
  author    = {L. Luo and
               Y. Xiong and
               Y. Liu and
               X. Sun},
  title     = {Adaptive Gradient Methods with Dynamic Bound of Learning Rate},
  booktitle   = {ICLR},
  year      = {2019},
}

@article{Bartlett:2006:SCP:2263195.2265382,
 author = {Bartlett, P. L.},
 title = {The Sample Complexity of Pattern Classification with Neural Networks: The Size of the Weights is More Important Than the Size of the Network},
 journal = {IEEE Trans. Inf. Theor.},
 volume = {44},
 number = {2},
 month = sep,
 year = {2006},
 pages = {525--536},
 url = {http://dx.doi.org/10.1109/18.661502},
} 

@article{DBLP:journals/corr/NeyshaburTS14,
  author    = {B. Neyshabur and
               R. Tomioka and
               N. Srebro},
  title     = {In Search of the Real Inductive Bias: On the Role of Implicit Regularization
               in Deep Learning},
  journal   = {CoRR},
  volume    = {abs/1412.6614},
  year      = {2014},
  url       = {http://arxiv.org/abs/1412.6614},
}

@book{allgower2012numerical,
  title={Numerical continuation methods: an introduction},
  author={Allgower, E. L. and Georg, K.},
  volume={13},
  year={2012},
  publisher={Springer}
}

@inproceedings{mobahi2015link,
  title={{On the link between Gaussian homotopy continuation and convex envelopes}},
  author={Mobahi, H. and Fisher III, J.},
  booktitle={Workshop on Energy Minimization Methods in CVPR},
  pages={43--56},
  year={2015},
  organization={Springer}
}

@article{gulcehre2016mollifying,
  title={Mollifying Networks},
  author={Gulcehre, C. and Moczulski, M. and Visin, F. and Bengio, Y.},
  journal={arXiv:1608.04980},
  year={2016}
}

@article{doi:10.1117/12.2176598,
author = {Ji, Z. and Ovsiannikov, I, and Wang, Y. and Shi, L. and Zhang, Q.},
title = {Reducing weight precision of convolutional neural networks towards large-scale on-chip image recognition},
journal = {Proc. SPIE},
year = {2015},
}

@article{DBLP:journals/corr/ChenWTWC15,
  author    = {W. Chen and
               J. T. Wilson and
               S. Tyree and
               K. Q. Weinberger and
               Y. Chen},
  title     = {Compressing Neural Networks with the Hashing Trick},
  journal   = {CoRR},
  volume    = {abs/1504.04788},
  year      = {2015},
}

@article{DBLP:journals/corr/CollinsK14,
  author    = {M. D. Collins and
               P. Kohli},
  title     = {Memory Bounded Deep Convolutional Networks},
  journal   = {CoRR},
  volume    = {abs/1412.1442},
  year      = {2014},
}

@incollection{NIPS2013_5025,
title = {Predicting Parameters in Deep Learning},
author = {Denil, M. and Shakibi, B. and Dinh, L. and Ranzato, M. A. and de Freitas, N.},
booktitle = {NIPS},
year = {2013},
}

@incollection{NIPS2014_5544,
    Author = {E. L. Denton and W. Zaremba and J. Bruna and Y. Lecun and R. Fergus},
    Booktitle = {NIPS},
    Title = {Exploiting Linear Structure Within Convolutional Networks for Efficient Evaluation},
    Year = {2014},
   } 

@incollection{NIPS2013_5025,
title = {Predicting Parameters in Deep Learning},
author = {Denil, M. and Shakibi, B. and Dinh, L. and Ranzato, M. A. and de Freitas, N.},
booktitle = {NIPS},
year = {2013},
}

@inproceedings{Hinton:1993:KNN:168304.168306,
 author = {Hinton, G. E. and van Camp, D.},
 title = {Keeping the Neural Networks Simple by Minimizing the Description Length of the Weights},
 booktitle = {COLT},
 year = {1993},
} 

@incollection{nowlan:adaptive,
  author = {Nowlan, S. J. and Hinton, G. E.},
  booktitle = {NIPS},
  title = {Adaptive Soft Weight Tying Using Gaussian Mixtures},
  year = 1992
}

@incollection{Kim2015,
  author    = {M. Kim and P. Smaragdis},
  title     = {Bitwise neural networks},
  booktitle = {ICML Workshop on Resource-Efficient Machine Learning},
  year      = {2015},
}

@article{DBLP:journals/corr/LinCMB15,
  author    = {Z. Lin and
               M. Courbariaux and
               R. Memisevic and
               Y. Bengio},
  title     = {Neural Networks with Few Multiplications},
  journal   = {CoRR},
  volume    = {abs/1510.03009},
  year      = {2015},
}

@article{journals/tnn/MarchesiOPU93,
  author = {Marchesi, M. and Orlandi, G. and Piazza, F. and Uncini, A.},
  journal = {IEEE Transactions on Neural Networks},
  keywords = {dblp},
  number = 1,
  pages = {53-62},
  title = {Fast neural networks without multipliers.},
  volume = 4,
  year = 1993
}

@incollection{NIPS1993_833,
title = {Backpropagation without Multiplication},
author = {P. Y. Simard and H. P. Graf},
booktitle = {NIPS},
year = {1994},
}

@incollection{Burge:1999:SBN:296533.296552,
 author = {Burge, P. S. and van Daalen, M. R. and Rising, B. J. P. and Shawe-Taylor, J. S.},
 chapter = {Stochastic Bit-stream Neural Networks},
 title = {Pulsed Neural Networks},
 year = {1999},
 pages = {337--352},
 numpages = {16},
 publisher = {MIT Press},
} 

@article{DBLP:journals/corr/Courbariaux2015,
  author    = {M. Courbariaux and Y. Bengio and J.-P. David},
  title     = {BinaryConnect: Training Deep Neural Networks with binary weights during propagations},
  journal   = {CoRR},
  volume    = {abs/1511.00363},
  year      = {2015},
}

@unpublished{LSZ2017,
  author    = {L. Shi and S. Feng and Z. Zhu},
  title     = {HFH: Homologically Functional Hashing for Compressing Deep Neural Networks},
  note = {Preprint at \url{https://openreview.net/pdf?id=Hyanrrqlg}},
  year      = {2017},
}

@inproceedings{DBLP:journals/corr/KimPYCYS15,
  author    = {Y.{-}D. Kim and
               E. Park and
               S. Yoo and
               T. Choi and
               L. Yang and
               D. Shin},
  title     = {Compression of Deep Convolutional Neural Networks for Fast and Low
               Power Mobile Applications},
  booktitle = {ICLR},
  year      = {2016},
}

@inproceedings{Yang2015,
    author = "Yang, Z. and Moczulski, M. and Denil, M. and de Freitas, N. and Smola, A. and Song, L. and Wang, Z.",
    title = "Deep Fried Convnets",
    year = "2015",
    booktitle = "International Conference on Computer Vision (ICCV)",
}

@article{DBLP:journals/corr/HintonVD15,
  author    = {G. E. Hinton and
               O. Vinyals and
               J. Dean},
  title     = {Distilling the Knowledge in a Neural Network},
  journal   = {CoRR},
  volume    = {abs/1503.02531},
  year      = {2015},
  url       = {http://arxiv.org/abs/1503.02531},
}

@inproceedings{41466,
title = {Fastfood - Approximating Kernel Expansions in Loglinear Time},
author  = {Q. Le and T. Sarlos and A. Smola},
year  = 2013,
URL = {http://jmlr.org/proceedings/papers/v28/le13.html},
booktitle = {ICML}
}

@inproceedings{DBLP:conf/nips/HanPTD15,
  author    = {S. Han and
               J. Pool and
               J. Tran and
               W. J. Dally},
  title     = {Learning both Weights and Connections for Efficient Neural Network},
  booktitle = {NIPS},
  year      = {2015},
  url       = {http://papers.nips.cc/paper/5784-learning-both-weights-and-connections-for-efficient-neural-network},
}

@inproceedings{DBLP:journals/corr/GuoYC16,
  author    = {Y. Guo and
               A. Yao and
               Y. Chen},
  title     = {Dynamic Network Surgery for Efficient DNNs},
  booktitle   = {NIPS},
  year      = {2016},
}

@inproceedings{DBLP:journals/corr/C,
  author    = {W. Chen and
               J. T. Wilson and
               S. Tyree and
               K. Q. Weinberger and
               Y. Chen},
  title     = {Compressing Convolutional Neural Networks},
  booktitle   = {NIPS},
  year      = {2015},
  url       = {http://arxiv.org/abs/1506.04449},
}

@inproceedings{SWS17,
  author    = {K. Ullrich, E. Meeds, M. Welling},
  title     = {Soft Weight-Sharing for Neural Network Compression},
  booktitle   = {ICLR},
  year      = {2017},
}

@inproceedings{DBLP:conf/nips/WenWWCL16,
  author    = {W. Wen and
               C. Wu and
               Y. Wang and
               Y. Chen and
               H. Li},
  title     = {Learning Structured Sparsity in Deep Neural Networks},
  booktitle = {NIPS},
  year      = {2016},
  url       = {http://papers.nips.cc/paper/6504-learning-structured-sparsity-in-deep-neural-networks},
}

@article{DBLP:journals/corr/IandolaMAHDK16,
  author    = {F. N. Iandola and
               M. W. Moskewicz and
               K. Ashraf and
               S. Han and
               W. J. Dally and
               K. Keutzer},
  title     = {SqueezeNet: AlexNet-level accuracy with 50x fewer parameters and {\textless}1MB
               model size},
  journal   = {CoRR},
  volume    = {abs/1602.07360},
  year      = {2016},
  url       = {http://arxiv.org/abs/1602.07360},
}

@article{DBLP:journals/corr/CourbariauxB16,
  author    = {M. Courbariaux and
               Y. Bengio},
  title     = {BinaryNet: Training Deep Neural Networks with Weights and Activations
               Constrained to +1 or -1},
  journal   = {CoRR},
  volume    = {abs/1602.02830},
  year      = {2016},
}

@article{DBLP:journals/corr/VenkateshNM16,
  author    = {G. Venkatesh and
               E. Nurvitadhi and
               D. Marr},
  title     = {Accelerating Deep Convolutional Networks using low-precision and sparsity},
  journal   = {CoRR},
  volume    = {abs/1610.00324},
  year      = {2016},
  url       = {http://arxiv.org/abs/1610.00324},
}

@article{DBLP:journals/corr/GuptaAGN15,
  author    = {S. Gupta and
               A. Agrawal and
               K. Gopalakrishnan and
               P. Narayanan},
  title     = {Deep Learning with Limited Numerical Precision},
  journal   = {CoRR},
  volume    = {abs/1502.02551},
  year      = {2015},
  url       = {http://arxiv.org/abs/1502.02551},
}

@article{Molchanov2017VariationalDS,
  title={Variational Dropout Sparsifies Deep Neural Networks},
  author={D. Molchanov and A. Ashukha and D. Vetrov},
  journal={CoRR},
  year={2017},
  volume={abs/1701.05369}
}

@article{DBLP:journals/corr/LinTA15,
  author    = {D. Dexu Lin and
               S. S. Talathi and
               V. {Sreekanth Annapureddy}},
  title     = {Fixed Point Quantization of Deep Convolutional Networks},
  journal   = {CoRR},
  volume    = {abs/1511.06393},
  year      = {2015},
  url       = {http://arxiv.org/abs/1511.06393},
}

@article{DBLP:journals/corr/WuLWHC15,
  author    = {J. Wu and
               C. Leng and
               Y. Wang and
               Q. Hu and
               J. Cheng},
  title     = {Quantized Convolutional Neural Networks for Mobile Devices},
  journal   = {CoRR},
  volume    = {abs/1512.06473},
  year      = {2015},
  url       = {http://arxiv.org/abs/1512.06473},
}

@article{DBLP:journals/corr/MerollaAAEM16,
  author    = {P. Merolla and
               R. Appuswamy and
               J. V. Arthur and
               S. K. Esser and
               D. S. Modha},
  title     = {Deep neural networks are robust to weight binarization and other non-linear
               distortions},
  journal   = {CoRR},
  volume    = {abs/1606.01981},
  year      = {2016},
  url       = {http://arxiv.org/abs/1606.01981},
}

@article{DBLP:journals/corr/JaderbergVZ14,
  author    = {M. Jaderberg and
               A. Vedaldi and
               A. Zisserman},
  title     = {Speeding up Convolutional Neural Networks with Low Rank Expansions},
  journal   = {CoRR},
  volume    = {abs/1405.3866},
  year      = {2014},
  url       = {http://arxiv.org/abs/1405.3866},
}

@article{DBLP:journals/corr/IoannouRSCC15,
  author    = {Y. Ioannou and
               D. P. Robertson and
               J. Shotton and
               R. Cipolla and
               A. Criminisi},
  title     = {Training CNNs with Low-Rank Filters for Efficient Image Classification},
  journal   = {CoRR},
  volume    = {abs/1511.06744},
  year      = {2015},
  url       = {http://arxiv.org/abs/1511.06744},
}

@article{DBLP:journals/corr/TaiXWE15,
  author    = {C. Tai and
               T. Xiao and
               X. Wang and
               W. E},
  title     = {Convolutional neural networks with low-rank regularization},
  journal   = {CoRR},
  volume    = {abs/1511.06067},
  year      = {2015},
  url       = {http://arxiv.org/abs/1511.06067},
}

@inproceedings{conf/cvpr/LiuWFTP15,
  author = {Liu, B. and Wang, M. and Foroosh, H. and Tappen, M. F. and Pensky, M.},
  booktitle = {CVPR},
  title = {Sparse Convolutional Neural Networks.},
  url = {http://dblp.uni-trier.de/db/conf/cvpr/cvpr2015.html#LiuWFTP15},
  year = 2015
}

@inproceedings{WCBN17,
  author    = {S. Wang, H. Cai, J. Bilmes, W. Noble},
  title     = {Training Compressed Fully-Connected Networks with a Density-Diversity Penalty},
  booktitle   = {ICLR},
  year      = {2017},
}

@inproceedings{DBLP:conf/icassp/SainathKSAR13,
  author    = {T. N. Sainath and
               B. Kingsbury and
               V. Sindhwani and
               E. Arisoy and
               B. Ramabhadran},
  title     = {Low-rank matrix factorization for Deep Neural Network training with
               high-dimensional output targets},
  booktitle = {ICASSP},
  year      = {2013},
}

@inproceedings{Li13,
author = {J. Li},
title = {Restructuring of Deep Neural Network Acoustic Models with Singular Value Decomposition},
booktitle = {Interspeech},
year = {2013},
}

@inproceedings{sindhwani_15,
  author    = {V. Sindhwani and
               T. N. Sainath and
               S. Kumar},
  title     = {Structured Transforms for Small-Footprint Deep Learning},
  booktitle = {NIPS},
  year      = {2015},
  url       = {http://papers.nips.cc/paper/5869-structured-transforms-for-small-footprint-deep-learning},
}

@incollection{NIPS2015_5937,
title = {Logarithmic Time Online Multiclass prediction},
author = {\textbf{A. Choromanska} and Langford, J.},
booktitle = {NIPS},
year = {2015},
url = {http://papers.nips.cc/paper/5937-logarithmic-time-online-multiclass-prediction.pdf}
}

@inproceedings{YAD2017,
    author = {Y. Jernite and \textbf{A. Choromanska} and D. Sontag},
    title = {Simultaneous Learning of Trees and Representations for Extreme Classification and Density Estimation},
    booktitle = {ICML},
    year      = {2017},
}

@article{DBLP:journals/corr/ChoromanskaCB16,
AUTHOR = {\textbf{A. Choromanska} and K. Jain, I.},
TITLE = {Extreme Multiclass Classification Criteria},
JOURNAL = {MDPI Computation},
VOLUME = {7},
YEAR = {2019},
NUMBER = {1},
ARTICLE-NUMBER = {16},
URL = {https://www.mdpi.com/2079-3197/7/1/16},
ISSN = {2079-3197},
DOI = {10.3390/computation7010016}
}

@inproceedings{8dbd67a2cb874e1580570a37d7f463ad,
title = "Adversarial Learning-Based On-Line Anomaly Monitoring for Assured Autonomy",
author = "N. Patel and {Nandini Saridena}, A. and \textbf{A. Choromanska} and P. Krishnamurthy and F. Khorrami",
year = "2018",
booktitle = "IROS",
}

@incollection{AnnaBook,
  author      = "A. Y. Aravkin and \textbf{A. Choromanska} and T. Jebara and D. Kanevsky",
  title       = "Semistochastic quadratic bound methods",
  booktitle   = "Log-Linear Models, Extensions and Applications",
  publisher   = "MIT Press",
  year        = 2018,
  pages       = "95-112",
  chapter     = 4,
}

@article{55b6bbffa7d947be82b6ccceb8b785b9,
title = "A deep learning gated architecture for UGV navigation robust to sensor failures",
keywords = "Autonomous vehicles, Deep learning for autonomous navigation, Learning from demonstration, Robustness to sensor failures, Sensor fusion, Vision/LiDAR based navigation",
author = "N. Patel and \textbf{A. Choromanska} and P. Krishnamurthy and F. Khorrami",
year = "2019",
day = "1",
volume = "116",
pages = "80--97",
journal = "Robotics and Autonomous Systems",
issn = "0921-8890",
publisher = "Elsevier",
}

@article{Cowen2019,
author = {Cowen, B. and {Nandini Saridena}, A. and \textbf{A. Choromanska}},
year = {2019},
title = {LSALSA: accelerated source separation via learned sparse coding},
journal = {Machine Learning (also appeared at ECML-PKDD)},
volume = {108}, 
pages = {1307–1327},
doi = {10.1007/s10994-019-05812-3}
}

@article{Yunfei2018,
author = {Teng, Y. and \textbf{A. Choromanska}},
year = {2019},
pages = {},
title = {Invertible Autoencoder for Domain Adaptation},
number = {2},
volume = {7},
journal = {MDPI Computation},
doi = {10.3390/computation7020020}
}

@InProceedings{Bisla_2019_CVPR_Workshops,
author = {Bisla, D. and \textbf{A. Choromanska} and Berman, R. S. and Stein, J. A. and Polsky, D.},
title = {Towards Automated Melanoma Detection With Deep Learning: Data Purification and Augmentation},
booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR) Workshop: ISIC Skin Image Analysis},
year = {2019}
}

@article{DBLP:journals/corr/NeyshaburBMS17aa,
  author    = {B. Neyshabur and
               S. Bhojanapalli and
               D. McAllester and
               N. Srebro},
  title     = {A PAC-Bayesian Approach to Spectrally-Normalized Margin Bounds for
               Neural Networks},
  booktitle  = {ICLR},
  year      = {2018},
}

@incollection{NIPS2017_7176,
title = {Exploring Generalization in Deep Learning},
author = {Neyshabur, B. and Bhojanapalli, S. and Mcallester, D. and Srebro, N.},
booktitle = {NIPS},
year = {2017},
}

@inproceedings{Dziugaite2017,
author = {Dziugaite, G. and Roy, D.},
year = {2017},
booktitle = {UAI},
title = {Computing Nonvacuous Generalization Bounds for Deep (Stochastic) Neural Networks with Many More Parameters than Training Data}
}

@article{jiang2019fantastic,
  author = {Jiang, Y. and Neyshabur, B. and Mobahi, H. and Krishnan, D. and Bengio, S.},
  title = {Fantastic Generalization Measures and Where to Find Them},
journal   = {CoRR},
  volume    = {abs/1912.02178},  
  year = 2019
}

@InProceedings{Fang2019,
author = {S. Fang and \textbf{A. Choromanska}},
title = {Reconﬁgurable Network for Eﬃcient Inferencing in Autonomous Vehicles},
booktitle = {ICRA},
year = {2019}
}

@inproceedings{DBLP:conf/icml/ChoromanskaCKLR19,
  author    = {\textbf{A. Choromanska} and
               B. Cowen and
               S. Kumaravel and
               R. Luss and
               M. Rigotti and
               I. Rish and
               P. Diachille and
               V. Gurev and
               B. Kingsbury and
               R. Tejwani and
               D. Bouneffouf},
  title     = {Beyond Backprop: Online Alternating Minimization with Auxiliary Variables},
  booktitle = {ICML},
  year      = {2019},
}

@inproceedings{DBLP:conf/soda/BoutsidisGKL15,
  author    = {C. Boutsidis and
               D. Garber and
               Z. Shay Karnin and
               E. Liberty},
  title     = {Online Principal Components Analysis},
  booktitle = {SODA},
  year      = {2015},
}

@incollection{NIPS2017_6816,
title = {The Reversible Residual Network: Backpropagation Without Storing Activations},
author = {Gomez, A. N. and Ren, M. and Urtasun, R. and Grosse, R. B.},
booktitle = {NIPS},
year = {2017},
}

@incollection{NIPS2019_9175,
title = {Leader Stochastic Gradient Descent for Distributed Training of Deep Learning Models},
author = {Teng, Y. and Gao, W. and Chalus, F. and \textbf{A. Choromanska} and Goldfarb, D. and Weller, A.},
booktitle = {NeurIPS},
year = {2019},
}


@inproceedings{AALWork2013,
  author    = {\textbf{A. Choromanska} and A. Agarwal and J. Langford},
  title     = {Extreme Multi Class Classification},
  booktitle = {NIPS Workshop: eXtreme Classification},
  year      = {2013},
}

@inproceedings{AISTATS2012_ChoromanskaM12, 
    Title = {Online Clustering with Experts}, 
    Url = {http://jmlr.csail.mit.edu/proceedings/papers/v22/choromanska12/choromanska12.pdf}, 
    Author = {\textbf{A. Choromanska} and C. Monteleoni}, 
    Year = {2012}, 
    Booktitle = {AISTATS} 
   }

@inproceedings{conf/alt/ChoromanskaJKMM13,
  author = {\textbf{A. Choromanska} and Jebara, T. and Kim, H. and Mohan, M. and Monteleoni, C.},
  booktitle = {ALT},
  title = {Fast Spectral Clustering via the Nyström Method.},
  url = {http://dblp.uni-trier.de/db/conf/alt/alt2013.html#ChoromanskaJKMM13},
  year = 2013
}

@inproceedings{DBLP:conf/aistats/ChoromanskaHMAL15,
  author    = {\textbf{A. Choromanska} and
               M. Henaff and
               M. Mathieu and
               G. {Ben Arous} and
               Y. LeCun},
  title     = {The Loss Surfaces of Multilayer Networks},
  booktitle = {AISTATS},
  year      = {2015},
}

@inproceedings{DBLP:journals/corr/ChaudhariCSL17,
  author    = {P. Chaudhari and \textbf{A. Choromanska} and S. Soatto and Y. LeCun and C. Baldassi and C. Borgs and J. Chayes and L. Sagun and R. Zecchina},
  title     = {Entropy-SGD: Biasing Gradient Descent Into Wide Valleys},
  booktitle = {ICLR},
  year      = {2017},
}

@article{DBLP:journals/corr/ChaudhariCSL17journal,
  author    = {P. Chaudhari and \textbf{A. Choromanska} and S. Soatto and Y. LeCun and C. Baldassi and C. Borgs and J. Chayes and L. Sagun and R. Zecchina},
  title     = {Entropy-SGD: Biasing Gradient Descent Into Wide Valleys (journal version)},
  journal = {Journal of Statistical Mechanics: Theory and Experiment},
  volume = {2019},
  number = {12},
  pages = {124018},
  year      = {2019},
}

@article{doi:10.1162/neco.1997.9.1.1,
author = {Hochreiter, S. and Schmidhuber, J.},
title = {Flat Minima},
journal = {Neural Computation},
volume = {9},
number = {1},
pages = {1-42},
year = {1997},
}

@inproceedings{keskar2016largebatch,
  author = {Keskar, N. S. and Mudigere, D. and Nocedal, J. and Smelyanskiy, M. and Tang, P. T. P.},
  title = {On Large-Batch Training for Deep Learning: Generalization Gap and Sharp
  Minima},
  booktitle = {ICLR},
  year = 2017,
}

@inproceedings{McAllester:1999:PMA:307400.307435,
  author = {McAllester, D. A.},
  booktitle = {COLT},
  title = {PAC-Bayesian Model Averaging},
  year = 1999,
}

@article{DBLP:journals/corr/abs-1805-07898,
  author    = {W. Wen and
               Y. Wang and
               F. Yan and
               C. Xu and
               Y. Chen and
               H. Li},
  title     = {SmoothOut: Smoothing Out Sharp Minima for Generalization in Large-Batch
               Deep Learning},
  journal   = {CoRR},
  volume    = {abs/1805.07898},
  year      = {2018},
}

@inproceedings{DBLP:conf/colt/ChoromanskaLA15,
  author    = {\textbf{A. Choromanska} and
               Y. LeCun and
               G. {Ben Arous}},
  title     = {Open Problem: The landscape of the loss surfaces of multilayer networks},
  booktitle = {COLT},
  year      = {2015},
}

@inproceedings{DBLP:conf/nips/JebaraC12,
  author    = {T. Jebara and
               \textbf{A. Choromanska}},
  title     = {Majorization for CRFs and Latent Likelihoods},
  booktitle = {NIPS},
  year      = {2012},
}

@inproceedings{DBLP:journals/corr/AravkinCKJ13,
  author    = {A. Y. Aravkin and
               \textbf{A. Choromanska} and
               D. Kanevsky and
               T. Jebara},
  title     = {Semistochastic Quadratic Bound Methods for Convex and Nonconvex Learning
               Problems},
  booktitle   = {ICLR Workshop},
  year      = {2014},
  url       = {http://arxiv.org/abs/1309.1369},
}

@inproceedings{DBLP:conf/icml/ChoromanskaCBJK16,
  author    = {\textbf{A. Choromanska} and
               K. Choromanski and
               M. Bojarski and
               T. Jebara and
               S. Kumar and
               Y. LeCun},
  title     = {Binary embeddings with structured hashed projections},
  booktitle = {ICML},
  year      = {2016},
}

@inproceedings{DBLP:journals/corr/BojarskiCCFGMSS16,
  author    = {M. Bojarski and
               \textbf{A. Choromanska} and
               K. Choromanski and
               F. Fagan and
               C. Gouy{-}Pailler and
               A. Morvan and
               N. Sakr and
               T. Sarl{\'{o}}s and
               J. Atif},
  title     = {Structured adaptive and random spinners for fast machine learning
               computations},
  booktitle = {AISTATS},
  year      = {2017},
}

@inproceedings{DBLP:conf/alt/ChoromanskaCJM13,
  author    = {\textbf{A. Choromanska} and
               K. Choromanski and
               G. Jagannathan and
               C. Monteleoni},
  title     = {Differentially-Private Learning of Low Dimensional Manifolds},
  booktitle = {ALT},
  year      = {2013},
}

@article{DBLP:journals/tcs/ChoromanskaCJM16,
  author    = {\textbf{A. Choromanska} and
               K. Choromanski and
               G. Jagannathan and
               C. Monteleoni},
  title     = {Differentially-private learning of low dimensional manifolds},
  journal   = {Theor. Comput. Sci.},
  year      = {2016},
}

@inproceedings{CKJ12,
  author    = {\textbf{A. Choromanska} and D. Kanevsky and T. Jebara},
  title     = {Majorization for Deep Belief Networks},
  booktitle = {NIPS Workshop: Log-linear models},
  year      = {2012},
}

@article{FNC12:GraphTracking, 
   Author = {\textbf{A. Choromanska} and Chang, S.-F. and Yuste, R.},
   Title = {Automatic reconstruction of neural morphologies with multi-scale graph-based tracking},
   Journal = {Frontiers in Neural Circuits},
   Volume = {6:25},
   Year = {2012}
} 

@article{journals/jmlr/MukherjeeS13,
  author = {Mukherjee, I. and Schapire, R. E.},
  journal = {Journal of Machine Learning Research},
  pages = {437-497},
  title = {A theory of multiclass boosting.},
  volume = 14,
  year = 2013
}

@article{janzamin2015beating,
	Author = {Janzamin, M. and Sedghi, H. and Anandkumar, A.},
	Title = {{Beating the Perils of Non-Convexity: Guaranteed Training of Neural Networks using Tensor Methods}},
	Journal = {arXiv:1506.08473},
	Year = {2015}}

@inproceedings{mikolov2013distributed,
  title={Distributed representations of words and phrases and their compositionality},
  author={Mikolov, T. and Sutskever, I. and Chen, K. and Corrado, G. S. and Dean, J.},
  booktitle={NIPS},
  year={2013}
}

@article{ShaiSS2012,
 author = {Shalev-Shwartz, S.},
 title = {Online Learning and Online Convex Optimization},
 journal = {Found. Trends Mach. Learn.},
 volume = {4},
 number = {2},
 year = {2012},
 pages = {107--194}
} 

@ARTICLE{Bengio03aneural,
    author = {Y. Bengio and R. Ducharme and P. Vincent and C. Jauvin},
    title = {A Neural Probabilistic Language Model},
    journal = {Journal of Machine Learning Research},
    year = {2003},
    volume = {3},
    pages = {1137--1155}
}

@inproceedings{morin2005hierarchical,
    author = {Morin, F. and Bengio, Y.},
    booktitle = {AISTATS},
    title = {{Hierarchical probabilistic neural network language model}},
    year = {2005}
}

@incollection{NIPS2008_3583,
title = {A Scalable Hierarchical Distributed Language Model},
author = {Mnih, A. and G. E. Hinton},
booktitle = {NIPS},
year = {2009},
}

@inproceedings{Chen:1996:ESS:981863.981904,
 author = {Chen, S. F. and Goodman, J.},
 title = {An Empirical Study of Smoothing Techniques for Language Modeling},
 booktitle = {ACL},
 year = {1996}
} 

@article{Brown:1992:CNG:176313.176316,
 author = {Brown, P. F. and deSouza, P. V. and Mercer, R. L. and Pietra, V. J. Della and Lai, J. C.},
 title = {Class-based N-gram Models of Natural Language},
 journal = {Comput. Linguist.},
 volume = {18},
 number = {4},
 year = {1992},
 pages = {467--479},
 publisher = {MIT Press}
} 

@inproceedings{conf/interspeech/KneserN93,
  author = {Kneser, R. and Ney, H.},
  booktitle = {EUROSPEECH},
  title = {Improved clustering techniques for class-based statistical language modelling.},
  year = 1993
}

@INPROCEEDINGS{Niesler98comparisonof,
    author = {T.R. Niesler and E. W. D. Whittaker and P.C. Woodland},
    title = {Comparison Of Part-Of-Speech And Automatically Derived Category-Based Language Models For Speech Recognition},
    booktitle = {ICASSP},
    year = {1998},
}

@Inproceedings {export:163344,
author       = {G. Zweig and J. C. Platt and C. Meek and C. J.C. Burges
                and A. Yessenalina and Q. Liu},
title = {Computational approaches to sentence completion},
booktitle    = {ACL},
year         = {2012},
}

@inproceedings{conf/emnlp/GubbinsV13,
  author = {Gubbins, J. and Vlachos, A.},
  booktitle = {EMNLP},
  title = {Dependency Language Models for Sentence Completion.},
  year = 2013
}

@article{Bengio:2003:NPL:944919.944966,
 author = {Bengio, Y. and Ducharme, R. and V., Pascal and Janvin, C.},
 title = {A Neural Probabilistic Language Model},
 journal = {J. Mach. Learn. Res.},
 volume = {3},
 year = {2003},
 pages = {1137--1155},
 publisher = {JMLR.org}
} 

@INPROCEEDINGS{Mnih12afast,
    author = {A. Mnih and Y. W. Teh},
    title = {A fast and simple algorithm for training neural probabilistic language models},
    booktitle = {ICML},
    year = {2012}
}

@Inproceedings {export:175560,
author       = {T. Mikolov and A. Deoras and S. Kombrink and L. Burget and J. Honza
                Cernocky},
booktitle    = {INTERSPEECH},
title        = {Empirical Evaluation and Combination of Advanced Language Modeling Techniques},
year         = {2011}
}

@inproceedings{conf/icml/SocherLNM11,
  author = {Socher, R. and Lin, C. C.-Y. and Ng, A. Y. and Manning, C. D.},
  booktitle = {ICML},
  title = {Parsing Natural Scenes and Natural Language with Recursive Neural Networks.},
  year = 2011,
}

@inproceedings{Turian:2010:WRS:1858681.1858721,
 author = {Turian, J. and Ratinov, L. and Bengio, Y.},
 title = {Word Representations: A Simple and General Method for Semi-supervised Learning},
 booktitle = {ACL},
 year = {2010}
}

@inproceedings{Collobert:2008:UAN:1390156.1390177,
 author = {Collobert, R. and Weston, J.},
 title = {A Unified Architecture for Natural Language Processing: Deep Neural Networks with Multitask Learning},
 booktitle = {ICML},
 year = {2008}
} 

@inproceedings{maas2010probabilistic,
  author = {Maas, A. L. and Ng, A. Y.},
  booktitle = {Workshop on Deep Learning and Unsupervised Feature Learning, NIPS},
  title = {A probabilistic model for semantic word vectors},
  year = 2010
}

@inproceedings{conf/www/DjuricWRGB15,
  author = {Djuric, N. and Wu, H. and Radosavljevic, V. and Grbovic, M. and Bhamidipati, N.},
  booktitle = {WWW},
  title = {Hierarchical Neural Language Models for Joint Representation of Streaming Documents and their Content},
  year = 2015
}

@article{DBLP:journals/corr/MirowskiV15,
  author    = {P. Mirowski and
               A. Vlachos},
  title     = {Dependency Recurrent Neural Language Models for Sentence Completion},
  journal   = {CoRR},
  volume    = {abs/1507.01193},
  year      = {2015}
}

@inproceedings{conf/interspeech/MikolovKBCK10,
  author = {Mikolov, T. and Karafiát, M. and Burget, L. and Cernocký, J. and Khudanpur, S.},
  booktitle = {INTERSPEECH},
  title = {Recurrent neural network based language model.},
  year = 2010
}

@article{DBLP:journals/corr/TaiSM15,
  author    = {K. S. Tai and
               R. Socher and
               C. D. Manning},
  title     = {Improved Semantic Representations From Tree-Structured Long Short-Term
               Memory Networks},
  journal   = {CoRR},
  volume    = {abs/1503.00075},
  year      = {2015}
}

@article{DBLP:journals/corr/KumarISBEPOGS15,
  author    = {A. Kumar and
               O. Irsoy and
               J. Su and
               J. Bradbury and
               R. English and
               B. Pierce and
               P. Ondruska and
               I. Gulrajani and
               R. Socher},
  title     = {Ask Me Anything: Dynamic Memory Networks for Natural Language Processing},
  journal   = {CoRR},
  volume    = {abs/1506.07285},
  year      = {2015}
}

@inproceedings{Schwenk:2005:TNN:1220575.1220601,
 author = {Schwenk, H. and Gauvain, J.-L.},
 title = {Training Neural Network Language Models on Very Large Corpora},
 booktitle = {HLT},
 year = {2005}
} 

@INPROCEEDINGS{Bengio+Senecal-2003,
     author = {Bengio, Y. and S{\'{e}}n{\'{e}}cal, J.-S.},
      title = {Quick Training of Probabilistic Neural Nets by Importance Sampling},
  booktitle = {AISTATS},
       year = {2003}
}

@article{journals/tnn/BengioS08,
  author = {Bengio, Y. and Senecal, J.-S.},
  journal = {IEEE Trans. Neural Networks},
  pages = {713-722},
  title = {Adaptive Importance Sampling to Accelerate Training of a Neural Probabilistic Language Model.},
  volume = 19,
  year = 2008
}

@article{Breiman:2001:RF:570181.570182,
 author = {Breiman, L.},
 title = {Random Forests},
 journal = {Mach. Learn.},
 volume = {45},
 year = {2001},
 pages = {5--32},
 publisher = {Kluwer Academic Publishers}
} 

@book{ig,
  author = {Breiman, L. and Friedman, J. H. and Olshen, R. A. and Stone, C. J.},
  publisher = {CRC Press LLC, Boca Raton, Florida},
  title = {Classification and Regression Trees},
  year = 1984
}

@inproceedings{mnih2007three,
 author = {Mnih, A. and Hinton, G.},
 title = {Three New Graphical Models for Statistical Language Modelling},
 booktitle = {ICML},
 year = {2007}
} 

@incollection{JelMer80,
  author = {Jelinek, F. and Mercer, R. L.},
  booktitle = {Proceedings, Workshop on Pattern Recognition in Practice},
  pages = {381--397},
  publisher = {North Holland},
  title = {Interpolated estimation of {M}arkov source parameters from sparse
  data},
  year = 1980,
}

@inproceedings{Katz1987,
  author = {Katz, S. M.},
  booktitle = {IEEE Trans. on Acoustics, Speech and Singal proc.},
  pages = {400-401},
  title = {Estimation of probabilities from sparse data for the language model
	component of a speech recognizer},
  volume = {ASSP-35},
  year = 1987
}

@inproceedings{Pereira+Tishby+Lee:93a,
   author = {F. Pereira and N. Tishby and L. Lee},
   title = {Distributional Clustering Of {English} Words},
   year = {1993},
   booktitle = {ACL}
}

@inproceedings{conf/interspeech/KneserN93,
  author = {Kneser, R. and Ney, H.},
  booktitle = {EUROSPEECH},
  title = {Improved clustering techniques for class-based statistical language modelling.},
  year = 1993
}

@inproceedings{Baker:1998:DCW:290941.290970,
 author = {Baker, L. Douglas and McCallum, Andrew Kachites},
 title = {Distributional Clustering of Words for Text Classification},
 booktitle = {SIGIR},
 year = {1998},
} 

@inproceedings{conf/icassp/SchwenkG02,
  author = {Schwenk, H. and Gauvain, J.-L.},
  booktitle = {ICASSP},
  title = {Connectionist language modeling for large vocabulary continuous speech recognition},
  year = 2002
}

@inproceedings{SchwenkH04,
  author = {Schwenk, H.},
  booktitle = {IJCNN},
  title = {Efficient training of large neural networks for language modeling},
  year = 2004
}

@inproceedings{Xu03,
  author = {Xu, P. and Emami, A. and Jelinek, F.},
  booktitle = {EMNLP},
  title = {Training connectionist models for the structured language model},
  year = 2003
}

@misc{VowpalWabbit,
  author = {Langford, J. and Li, L. and Strehl, A.},
  howpublished = {\url{ http://hunch.net/~vw}},
  year = 2007
}
	
@article{AKM2016,
  author    = {\textbf{A. Choromanska} and K. Choromanski and M. Bojarski},
  title     = {On the boosting ability of top-down decision tree learning algorithm for multiclass classification},
  journal   = {CoRR},
  volume    = {abs/1605.05223},
  year      = {2016},
}

@article{AGNS2011,
  author    = {A. Azocar and J. Gimenez and K. Nikodem and J. L. Sanchez},
  title     = {On strongly midconvex functions},
  journal = {Opuscula Math.},
  number = 1,
  pages = {15-26},
  volume = 31,
  year = 2011
}

@inproceedings{Ram:2011:DET:2020408.2020507,
 author = {Ram, P. and Gray, A. G.},
 title = {Density Estimation Trees},
 booktitle = {KDD},
 year = {2011}
} 

@Book{           bishop:2006:PRML,
  author = 	 "C. M. Bishop",
  title = 	 "Pattern Recognition and Machine Learning",
  publisher = 	 "Springer",
  year = 	 "2006",
}

@inproceedings{37180,
title = {Wsabie: Scaling Up To Large Vocabulary Image Annotation},
author  = {J. Weston and S. Bengio and N. Usunier},
year  = 2011,
booktitle = {IJCAI}
}

@incollection{NIPS2015_5969,
title = {Sparse Local Embeddings for Extreme Multi-label Classification},
author = {Bhatia, K. and Jain, H. and Kar, P. and Varma, M. and Jain, P.},
booktitle = {NIPS},
year = {2015},
}

@inproceedings{collobert2011torch7,
  title={Torch7: A matlab-like environment for machine learning},
  author={Collobert, R. and Kavukcuoglu, K. and Farabet, C.},
  booktitle={BigLearn, NIPS Workshop},
  number={EPFL-CONF-192376},
  year={2011},
}

@article{gutmann2010noise,
 author = {Gutmann, M. U. and Hyv\"{a}rinen, A.},
 title = {Noise-contrastive Estimation of Unnormalized Statistical Models, with Applications to Natural Image Statistics},
 journal = {J. Mach. Learn. Res.},
 volume = {13},
 number = {1},
 year = {2012},
 pages = {307--361},
} 

@inproceedings{DBLP:conf/naacl/AndreasK15,
  author    = {J. Andreas and
               D. Klein},
  title     = {When and why are log-linear models self-normalizing?},
  booktitle = {{NAACL} {HLT}},
  year      = {2015},
}

@article{DBLP:journals/corr/GraveJCGJ16,
  author    = {E. Grave and
               A. Joulin and
               M. Ciss{\'{e}} and
               D. Grangier and
               H. J{\'{e}}gou},
  title     = {Efficient softmax approximation for GPUs},
  journal   = {CoRR},
  volume    = {abs/1609.04309},
  year      = {2016},
}

@inproceedings{export:255952,
    author = {Kontschieder, P. and Fiterau, M. and Criminisi, A. and Bul{\`o}, S. Rota},
    booktitle = {ICCV},
    title = {{Deep Neural Decision Forests}},
    year = {2015},
}

@inproceedings{DBLP:conf/nips/VincentBB15,
  author    = {P. Vincent and
               A. de Br{\'{e}}bisson and
               X. Bouthillier},
  title     = {Efficient Exact Gradient Update for training Deep Networks with Very
               Large Sparse Targets},
  booktitle = {NIPS},
  year      = {2015},
}

@inproceedings{DBLP:journals/corr/BrebissonV15,
  author    = {A. de Br{\'{e}}bisson and
               P. Vincent},
  title     = {An Exploration of Softmax Alternatives Belonging to the Spherical
               Loss Family},
  booktitle = {ICLR},
  year      = {2016},
}

@article{DBLP:journals/cacm/ThomeeSFENPBL16,
  author    = {Bart Thomee and
               David A. Shamma and
               Gerald Friedland and
               Benjamin Elizalde and
               Karl Ni and
               Douglas Poland and
               Damian Borth and
               Li{-}Jia Li},
  title     = {{YFCC100M:} the new data in multimedia research},
  journal   = {Commun. {ACM}},
  volume    = {59},
  number    = {2},
  pages     = {64--73},
  year      = {2016}
}

@article{DBLP:journals/corr/JoulinGBM16,
  author    = {Armand Joulin and
               Edouard Grave and
               Piotr Bojanowski and
               Tomas Mikolov},
  title     = {Bag of Tricks for Efficient Text Classification},
  journal   = {CoRR},
  volume    = {abs/1607.01759},
  year      = {2016}
}

@incollection{NIPS2015_5797, 
   title = {Path-SGD: Path-Normalized Optimization in Deep Neural Networks}, 
   author = {Neyshabur, B. and Salakhutdinov, R. R. and Srebro, N.}, 
   booktitle = {NIPS}, 
   year = {2015}, 
   url = {http://papers.nips.cc/paper/5797-path-sgd-path-normalized-optimization-in-deep-neural-networks.pdf} 
   }

@inproceedings{DBLP:journals/corr/KeskarMNST16,
  author    = {N. S. Keskar and
               D. Mudigere and
               J. Nocedal and
               M. Smelyanskiy and
               P. T. P. Tang},
  title     = {On Large-Batch Training for Deep Learning: Generalization Gap and
               Sharp Minima},
  booktitle = {ICLR},
  year      = {2017},
  url       = {http://arxiv.org/abs/1609.04836},
}

@article{SB2017,
  author    = {L. Dinh, R. Pascanu, S. Bengio, Y. Bengio},
  title     = {Sharp Minima Can Generalize For Deep Nets},
  journal   = {CoRR},
  volume    = {abs/1703.04933},
  year      = {2017},
}

@misc{mnistlecun,
    author = {LeCun, Y. and Cortes, C.},
    title = {{The MNIST database of handwritten digits}},
    howpublished = {\url{http://yann.lecun.com/exdb/mnist/}},
}

@incollection{NIPS1989_293,
title = {Handwritten Digit Recognition with a Back-Propagation Network},
author = {LeCun, Y. and B. E. Boser and J. S. Denker and D. Henderson and R. E. Howard and W. E. Hubbard and L. D. Jackel},
booktitle = {NIPS},
year = {1990},
}

@article{physics2007,
   author = "Berthier, L. and Biroli, G. and Bouchaud, J.-P. and Kob, W. and Miyazaki, K. and Reichman, D. R.",
   title = "Spontaneous and induced dynamic fluctuations in glass formers. I. General results and dependence on ensemble and dynamics",
   journal = "The Journal of Chemical Physics",
   year = "2007",
   volume = "126",
   number = "18", 
}

@book{citeulike:3359427,
    author = {van Laarhoven, P. J. and Aarts, E. H.},
    month = jun,
    publisher = {Springer},
    title = {{Simulated Annealing: Theory and Applications (Mathematics and Its Applications)}},
    url = {http://www.amazon.com/exec/obidos/redirect?tag=citeulike07-20\&path=ASIN/9027725136},
    year = {1987}
}

@incollection{NIPS2015_5953,
title = {Natural Neural Networks},
author = {Desjardins, G. and Simonyan, K. and Pascanu, R. and Kavukcuoglu, K.},
booktitle = {NIPS},
year = {2015},
url = {http://papers.nips.cc/paper/5953-natural-neural-networks.pdf}
}

@incollection{2012distbelief,
 title ={Large Scale Distributed Deep Networks},
 author={Dean, J. and Corrado, G. and Monga, R. and Chen, K. and Devin, M. and Le, Q. and Mao, M. and Ranzato, M. and Senior, A. and Tucker, P. and Yang, K. and Ng, A.},
 booktitle = {NIPS},
 year = {2012},
}

@article{DBLP:journals/corr/BalakrishnanWY14,
  author    = {S. Balakrishnan and
               M. J. Wainwright and
               B. Yu},
  title     = {Statistical guarantees for the {EM} algorithm: From population to
               sample-based analysis},
  journal   = {The Annals of Statistics},
   volume = "45",
   number = "1", 
 pages = {77--120},
  year      = {2015},
}

@article{csiszar_information_1984,
    author = {Csisz\'{a}r, I. and Tusn\'{a}dy, G.},
    journal = {Statistics and Decisions},
    title = {{Information Geometry and Alternating minimization procedures}},
    volume = {Supplement Issue 1},
    year = {1984}
}

@article{bousquet2002stability,
  title={Stability and generalization},
  author={Bousquet, O. and Elisseeff, A.},
  journal={JMLR},
  volume={2},
  number={Mar},
  pages={499--526},
  year={2002}
}

@article{Boyd2011,
 author = {Boyd, S. and Parikh, N. and Chu, E. and Peleato, B. and Eckstein, J.},
 title = {Distributed Optimization and Statistical Learning via the Alternating Direction Method of Multipliers},
 journal = {Found. Trends Mach. Learn.},
 volume = {3},
 number = {1},
 year = {2011},
 pages = {1--122},
 numpages = {122},
}

@article{Shalev-Shwartz:2010:LSU:1756006.1953019,
 author = {Shalev-Shwartz, S. and Shamir, O. and Srebro, N. and Sridharan, K.},
 title = {Learnability, Stability and Uniform Convergence},
 journal = {J. Mach. Learn. Res.},
 volume = {11},
 year = {2010},
 pages = {2635--2670},
 url = {http://dl.acm.org/citation.cfm?id=1756006.1953019},
} 

@InProceedings{pmlr-v40-Ge15,
  title = 	 {Escaping From Saddle Points --- Online Stochastic Gradient for Tensor Decomposition},
  author = 	 {R. Ge and F. Huang and C. Jin and Y. Yuan},
  booktitle = 	 {COLT},
  year = 	 {2015},
  url = 	 {http://proceedings.mlr.press/v40/Ge15.html},
}

@InProceedings{pmlr-v49-anandkumar16,
  title = 	 {Efficient approaches for escaping higher order saddle points  in non-convex optimization},
  author = 	 {A. Anandkumar and R. Ge},
  booktitle = 	 {COLT},
  year = 	 {2016},
  url = 	 {http://proceedings.mlr.press/v49/anandkumar16.html},
}

@article{JGNKJ2017,
  author    = {C. Jin, R. Ge, P. Netrapalli, S. M. Kakade, M. I. Jordan},
  title     = {How to Escape Saddle Points Efficiently},
  journal   = {CoRR},
  volume    = {abs/1703.00887},
  year      = {2017},
}

@InProceedings{DBLP:journals/corr/GoodfellowSS14,
  author    = {I. J. Goodfellow and
               J. Shlens and
               C. Szegedy},
  title     = {Explaining and Harnessing Adversarial Examples},
  booktitle   = {ICLR},
  year      = {2015},
}

@InProceedings{ICML2011Saxe_551,
  author =    {A. Saxe and P. W. Koh and Z. Chen and M. Bhand and B. Suresh and A. Ng},
  title =     {On Random Weights and Unsupervised Feature Learning },
  booktitle = {ICML},
  year =      {2011},
}

@inproceedings{conf/iccv/JarrettKRL09,
  author = {Jarrett, K. and Kavukcuoglu, K. and Ranzato, M. and LeCun, Y.},
  booktitle = {ICCV},
  title = {What is the best multi-stage architecture for object recognition?},
  year = 2009
}

@article{journals/ploscb/PintoDDC09,
  author = {Pinto, N. and Doukhan, D. and DiCarlo, J. J. and Cox, D. D.},
  journal = {PLoS Computational Biology},
  number = 11,
  title = {A High-Throughput Screening Approach to Discovering Good Forms of Biologically Inspired Visual Representation.},
  volume = 5,
  year = 2009
}

@inproceedings{ pinto:bionetics_2010,
    author={Pinto, N. and Cox, D. D.},
    title={{An Evaluation of the Invariance Properties of a Biologically-Inspired System for Unconstrained Face Recognition}},
    booktitle={BIONETICS},
    year={2010}
}

@article{Huang2006489,
title = "Extreme learning machine: Theory and applications ",
journal = "Neurocomputing ",
volume = "70",
number = "1–3",
pages = "489 - 501",
year = "2006",
author = "G.-B. Huang and Q.-Y. Zhu and C.-K. Siew",
}

@inproceedings{Moczulski2016,
  title={ACDC: A Structured Efficient Linear Layer},
  author={Moczulski, M. and Denil, M. and Appleyard, J. and de Freitas, N.},
  booktitle= {ICLR},
  year={2016},
}

@article{tyree,
  author    = {W. Chen and
                 J. T. Wilson and
                  S. Tyree and
                  K. Q. Weinberger and 
                 Y. Chen},
  title     = {Compressing Neural Networks with the Hashing Trick},
  journal   = {CoRR},
  volume    = {abs/1504.04788},
  year      = {2015},
}

@inproceedings{felix,
  author    = {F. X. Yu and
               S. Kumar and
               Y. Gong and
               S.{-}F. Chang},
  title     = {Circulant Binary Embedding},
  booktitle = {ICML},
  year      = {2014},
}

@inproceedings{indyk2015,
  author    = {A. Andoni and
               P. Indyk and
               T. Laarhoven and
               I. P. Razenshteyn and
               L. Schmidt},
  title     = {Practical and Optimal {LSH} for Angular Distance},
  booktitle = {NIPS},
  year      = {2015},
}

@misc{NSFBIGDATA,
  title = "{The NSF BIGDATA program: Critical Techniques, Technologies and Methodologies for Advancing Foundations and Applications of Big Data Sciences and Engineering (BIGDATA)}",
  year = {2018},
  howpublished = {\url{https://www.nsf.gov/funding/pgm_summ.jsp?pims_id=504767}},
}

@misc{NSTC,
author = "{Executive Office of the President, National Science and Technology Council}",
title = "{The Federal Big Data Research and Development Strategic Plan}",
year = {2016},
howpublished = {\url{https://www.nitrd.gov/PUBS/bigdatardstrategicplan.pdf}},
}

@misc{ARL,
title = "{Army Research Laboratory's Technical Implementation Plan 2015 - 2019}",
howpublished = {\url{https://www.arl.army.mil/www/pages/172/docs/ARL_Technical_Implementation_Plan.pdf}},
}

@misc{AChoromaweb,
title = "{\textbf{A. Choromanska} website}",
howpublished = {\url{http://cims.nyu.edu/~achoroma}},
}

@misc{Trinity,
author = {A. Anandkumar},
title = "{The AI Trinity: Data + Algorithms + Infrastructure, Research Talk at the NYU Tandon ECE Seminar Series on Modern AI}",
year = {2018},
howpublished = {\url{https://www.youtube.com/watch?v=Mzior-Jmp8A}},
}

@inproceedings{KarpathyCVPR14,
  title     = {Large-scale Video Classification with Convolutional Neural Networks},
  author    = {A. Karpathy and G. Toderici and S. Shetty and T. Leung and R. Sukthankar and L. Fei-Fei},
  year      = {2014},
  booktitle = {CVPR}
}

@inproceedings{DBLP:journals/corr/BojarskiCCFJMZ16,
  author    = {M. Bojarski and
               \textbf{A. Choromanska} and
               K. Choromanski and
               B. Firner and
               L. D. Jackel and
               U. Muller and
               P. Yeres and
               K. Zieba},
  title     = {VisualBackProp: efficient visualization of CNNs for autonomous driving},
  booktitle   = {ICRA},
  year      = {2018},
  }

@inproceedings{SMF17,
  author    = {N. Patel and \textbf{A. Choromanska} and P. Krishnamurthy and F. Khorrami},
  title     = "{Sensor Modality Fusion with CNNs for UGV Autonomous Driving in Indoor Environments}",
  booktitle = {IROS},
  year      = {2017},
  howpublished = {\url{http://cims.nyu.edu/~achoroma/NonFlash/Papers/SMF_CNN.pdf}},
}

@inproceedings{zhang2019lookahead,
 author = {Zhang, M. and Lucas, J. and Ba, J. and Hinton, G. E.},
 booktitle = {NeurIPS},
 title = {Lookahead Optimizer: k steps forward, 1 step back},
 year = {2019},
}

@inproceedings{schaul2013pesky,
      title={No More Pesky Learning Rates}, 
      author={T. Schaul and S. Zhang and Y. LeCun},
      booktitle = {ICML},
      year={2013},
}

@InProceedings{pmlr-v37-martens15, title = {Optimizing Neural Networks with Kronecker-factored Approximate Curvature}, author = {J. Martens and R. Grosse}, booktitle = {ICML}, year = {2015},}

@inproceedings{yan2018unified,
  title={A unified analysis of stochastic momentum methods for deep learning},
  author={Yan, Y and Yang, T and Li, Z and Lin, Q and Yang, Y},
  booktitle={IJCAI International Joint Conference on Artificial Intelligence},
  year={2018}
}

@inproceedings{le2012stochastic,
  title={A Stochastic Gradient Method with an Exponential Convergence Rate for Finite Training Sets},
  author={Le Roux, N. and Schmidt, M. and Bach, F.},
  booktitle={NeurIPS},
  year={2012}
}

@article{Polyak1964,
author = {Polyak, B.T.},
title = {Some methods of speeding up the convergence
of iteration methods},
journal = {Computational Mathematics and Mathematical Physics},
volume = {4},
number = {5},
pages = {1-17},
year = {1964},
}

@article{schmidt2017minimizing,
  title={Minimizing finite sums with the stochastic average gradient},
  author={Schmidt, M. and Le Roux, N. and Bach, F.},
  journal={Mathematical Programming},
  volume={162},
  number={1-2},
  pages={83--112},
  year={2017},
}

@article{wu2018wngrad,
  title={Wngrad: Learn the learning rate in gradient descent},
  author={Wu, X. and Ward, R. and Bottou, L.},
  journal={CoRR},
  volume = {abs/1803.02865},
  year={2018}
}

@article{ramezani2018stability,
  title={On the stability and convergence of stochastic gradient descent with momentum},
  author={Ramezani-Kebrya, A. and Khisti, A. and Liang, B.},
  journal={CoRR},
  volume = {abs/1809.04564},
  year={2018},
}
@article{wu2019global,
  title={Global convergence of adaptive gradient methods for an over-parameterized neural network},
  author={Wu, X. and Du, S. S. and Ward, R.},
  journal={CoRR},
  volume = {abs/1902.07111},
  year={2019}
}
@inproceedings{zhang2019fast,
  title={Fast Convergence of Natural Gradient Descent for Over-Parameterized Neural Networks},
  author={Zhang, G. and Martens, J. and Grosse, R. B.},
  booktitle={NeurIPS},
  year={2019},
}
@inproceedings{gower2019sgd,
  title={SGD: General analysis and improved rates},
  author={Gower, R. M. and Loizou, N. and Qian, X. and Sailanbayev, A. and Shulgin, E. and Richt{\'a}rik, P.},
  booktitle={ICML},
  year={2019},
}

@article{li2019exponential,
  title={An exponential learning rate schedule for deep learning},
  author={Li, Zhiyuan and Arora, Sanjeev},
  journal={arXiv preprint arXiv:1910.07454},
  year={2019}
}

@inproceedings{bottou2010large,
  title={Large-scale machine learning with stochastic gradient descent},
  author={Bottou, L{\'e}on},
  booktitle={Proceedings of COMPSTAT'2010: 19th International Conference on Computational StatisticsParis France, August 22-27, 2010 Keynote, Invited and Contributed Papers},
  pages={177--186},
  year={2010},
  organization={Springer}
}

@inproceedings{sutskever2013importance,
  title={On the importance of initialization and momentum in deep learning},
  author={Sutskever, Ilya and Martens, James and Dahl, George and Hinton, Geoffrey},
  booktitle={International conference on machine learning},
  pages={1139--1147},
  year={2013},
  organization={PMLR}
}

@article{kingma2014adam,
  title={Adam: A method for stochastic optimization},
  author={Kingma, Diederik P and Ba, Jimmy},
  journal={arXiv preprint arXiv:1412.6980},
  year={2014}
}

@inproceedings{foretsharpness,
  title={Sharpness-aware Minimization for Efficiently Improving Generalization},
  author={Foret, Pierre and Kleiner, Ariel and Mobahi, Hossein and Neyshabur, Behnam},
  booktitle={International Conference on Learning Representations}
}


@article{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}

@inproceedings{bojar2014findings,
  title={Findings of the 2014 workshop on statistical machine translation},
  author={Bojar, Ond{\v{r}}ej and Buck, Christian and Federmann, Christian and Haddow, Barry and Koehn, Philipp and Leveling, Johannes and Monz, Christof and Pecina, Pavel and Post, Matt and Saint-Amand, Herve and others},
  booktitle={Proceedings of the ninth workshop on statistical machine translation},
  pages={12--58},
  year={2014}
}

@inproceedings{smith2019super,
  title={Super-convergence: Very fast training of neural networks using large learning rates},
  author={Smith, Leslie N and Topin, Nicholay},
  booktitle={Artificial intelligence and machine learning for multi-domain operations applications},
  volume={11006},
  pages={369--386},
  year={2019},
  organization={SPIE}
}

@misc{ReduceLROnPlateau,
  title={PyTorch: ReduceLROnPlateau},
  howpublished={\url{https://pytorch.org/docs/stable/generated/torch.optim.lr_scheduler.ReduceLROnPlateau.html#torch.optim.lr_scheduler.ReduceLROnPlateau}}
}

@article{devlin2018bert,
  title={Bert: Pre-training of deep bidirectional transformers for language understanding},
  author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
  journal={arXiv preprint arXiv:1810.04805},
  year={2018}
}

@article{wang2018glue,
  title={GLUE: A multi-task benchmark and analysis platform for natural language understanding},
  author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1804.07461},
  year={2018}
}

@article{pflug1990non,
  title={Non-asymptotic confidence bounds for stochastic approximation algorithms with constant step size},
  author={Pflug, Georg Ch},
  journal={Monatshefte f{\"u}r Mathematik},
  volume={110},
  pages={297--314},
  year={1990},
  publisher={Springer}
}

@inproceedings{chee2018convergence,
  title={Convergence diagnostics for stochastic gradient descent with constant learning rate},
  author={Chee, Jerry and Toulis, Panos},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={1476--1485},
  year={2018},
  organization={PMLR}
}

@inproceedings{pesme2020convergence,
  title={On convergence-diagnostic based step sizes for stochastic gradient descent},
  author={Pesme, Scott and Dieuleveut, Aymeric and Flammarion, Nicolas},
  booktitle={International Conference on Machine Learning},
  pages={7641--7651},
  year={2020},
  organization={PMLR}
}

@inproceedings{agarwal2021acceleration,
  title={Acceleration via fractal learning rate schedules},
  author={Agarwal, Naman and Goel, Surbhi and Zhang, Cyril},
  booktitle={International Conference on Machine Learning},
  pages={87--99},
  year={2021},
  organization={PMLR}
}

@inproceedings{cohen2020gradient,
  title={Gradient descent on neural networks typically occurs at the edge of stability},
  author={Cohen, Jeremy and Kaur, Simran and Li, Yuanzhi and Kolter, J Zico and Talwalkar, Ameet},
  booktitle={International Conference on Learning Representations},
  year={2020}
}
