
@misc{yang_fine-grained_2020,
	title = {A {Fine}-{Grained} {Spectral} {Perspective} on {Neural} {Networks}},
	abstract = {Are neural networks biased toward simple functions? Does depth always help learn more complex features? Is training the last layer of a network as good as training all layers? How to set the range for learning rate tuning? These questions seem unrelated at face value, but in this work we give all of them a common treatment from the spectral perspective. We will study the spectra of the *Conjugate Kernel, CK,* (also called the *Neural Network-Gaussian Process Kernel*), and the *Neural Tangent Kernel, NTK*. Roughly, the CK and the NTK tell us respectively "what a network looks like at initialization" and "what a network looks like during and after training." Their spectra then encode valuable information about the initial distribution and the training and generalization properties of neural networks. By analyzing the eigenvalues, we lend novel insights into the questions put forth at the beginning, and we verify these insights by extensive experiments of neural networks. We derive fast algorithms for computing the spectra of CK and NTK when the data is uniformly distributed over the boolean cube, and show this spectra is the same in high dimensions when data is drawn from isotropic Gaussian or uniformly over the sphere. Code replicating our results is available at github.com/thegregyang/NNspectra.},
	urldate = {2023-01-02},
	publisher = {arXiv},
	author = {Yang, Greg and Salman, Hadi},
	month = apr,
	year = {2020},
	note = {arXiv:1907.10599 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Computer Science - Neural and Evolutionary Computing, Statistics - Machine Learning},
	annote = {Comment: 8 pages of main text, 19 figures, 51 pages including appendix},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/QEHEUHW7/Yang and Salman - 2020 - A Fine-Grained Spectral Perspective on Neural Netw.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/6WTIM28M/1907.html:text/html},
}

@techreport{aghazadeh_sparse_2020,
	type = {preprint},
	title = {Sparse {Epistatic} {Regularization} of {Deep} {Neural} {Networks} for {Inferring} {Fitness} {Functions}},
	abstract = {Despite recent advances in high-throughput combinatorial mutagenesis assays, the number of labeled sequences available to predict molecular functions has remained small for the vastness of the sequence space combined with the ruggedness of many ﬁtness functions. Expressive models in machine learning (ML), such as deep neural networks (DNNs), can model the nonlinearities in rugged ﬁtness functions, which manifest as high-order epistatic interactions among the mutational sites. However, in the absence of an inductive bias, DNNs overﬁt to the small number of labeled sequences available for training. Herein, we exploit the recent biological evidence that epistatic interactions in many ﬁtness functions are sparse; this knowledge can be used as an inductive bias to regularize DNNs. We have developed a method for sparse epistatic regularization of DNNs, called the epistatic net (EN), which constrains the number of non-zero coeﬃcients in the spectral representation of DNNs. For larger sequences, where ﬁnding the spectral transform becomes computationally intractable, we have developed a scalable extension of EN, which subsamples the combinatorial sequence space uniformly inducing a sparse-graph-code structure, and regularizes DNNs using the resulting greedy optimization method. Results on several biological landscapes, from bacterial to protein ﬁtness functions, show that EN consistently improves the prediction accuracy of DNNs and enables them to outperform competing models which assume other forms of inductive biases. EN estimates all the higher-order epistatic interactions of DNNs trained on massive sequence spaces—a computational problem that takes years to solve without leveraging the epistatic sparsity in the ﬁtness functions.},
	language = {en},
	urldate = {2023-01-02},
	institution = {Bioinformatics},
	author = {Aghazadeh, Amirali and Nisonoff, Hunter and Ocal, Orhan and Brookes, David H. and Huang, Yijie and Koyluoglu, O. Ozan and Listgarten, Jennifer and Ramchandran, Kannan},
	month = nov,
	year = {2020},
	doi = {10.1101/2020.11.24.396994},
	file = {Aghazadeh et al. - 2020 - Sparse Epistatic Regularization of Deep Neural Net.pdf:/Users/agorji/Zotero/storage/E5EXPT2M/Aghazadeh et al. - 2020 - Sparse Epistatic Regularization of Deep Neural Net.pdf:application/pdf},
}

@misc{karimi_riemannian_2022,
	title = {Riemannian stochastic approximation algorithms},
	doi = {10.48550/arXiv.2206.06795},
	abstract = {We examine a wide class of stochastic approximation algorithms for solving (stochastic) nonlinear problems on Riemannian manifolds. Such algorithms arise naturally in the study of Riemannian optimization, game theory and optimal transport, but their behavior is much less understood compared to the Euclidean case because of the lack of a global linear structure on the manifold. We overcome this difficulty by introducing a suitable Fermi coordinate frame which allows us to map the asymptotic behavior of the Riemannian Robbins-Monro (RRM) algorithms under study to that of an associated deterministic dynamical system. In so doing, we provide a general template of almost sure convergence results that mirrors and extends the existing theory for Euclidean Robbins-Monro schemes, despite the significant complications that arise due to the curvature and topology of the underlying manifold. We showcase the flexibility of the proposed framework by applying it to a range of retraction-based variants of the popular optimistic / extra-gradient methods for solving minimization problems and games, and we provide a unified treatment for their convergence.},
	urldate = {2023-01-03},
	publisher = {arXiv},
	author = {Karimi, Mohammad Reza and Hsieh, Ya-Ping and Mertikopoulos, Panayotis and Krause, Andreas},
	month = dec,
	year = {2022},
	note = {arXiv:2206.06795 [cs, math]},
	keywords = {Computer Science - Machine Learning, Mathematics - Dynamical Systems, Mathematics - Optimization and Control, Primary 62L20, 37N40, secondary 90C15, 90C47, 90C48},
	annote = {Comment: 33 pages, 2 figures; a one-page abstract of this paper was presented in COLT 2022},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/9TBX6NFK/Karimi et al. - 2022 - Riemannian stochastic approximation algorithms.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/BHQNYDQ4/2206.html:text/html},
}

@inproceedings{amrollahi_efficiently_2019,
	title = {Efficiently {Learning} {Fourier} {Sparse} {Set} {Functions}},
	volume = {32},
	urldate = {2023-01-03},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Amrollahi, Andisheh and Zandieh, Amir and Kapralov, Michael and Krause, Andreas},
	year = {2019},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/S6YC2DF2/Amrollahi et al. - 2019 - Efficiently Learning Fourier Sparse Set Functions.pdf:application/pdf},
}


@book{boyd_distributed_2011,
	address = {Hanover, MA},
	title = {Distributed optimization and statistical learning via the alternating direction method of multipliers},
	isbn = {978-1-60198-460-9},
	language = {en},
	publisher = {Now Publishers Inc},
	author = {Boyd, Stephen P.},
	year = {2011},
	keywords = {Algorithms, Convex programming, Machine learning},
	file = {Boyd - 2011 - Distributed optimization and statistical learning .pdf:/nas/agorji/Zotero/storage/V5FIHL6Y/Boyd - 2011 - Distributed optimization and statistical learning .pdf:application/pdf},
}

@article{brookes_sparsity_2022,
	title = {On the sparsity of fitness functions and implications for learning},
	volume = {119},
	issn = {1091-6490},
	doi = {10.1073/pnas.2109649118},
	abstract = {Fitness functions map biological sequences to a scalar property of interest. Accurate estimation of these functions yields biological insight and sets the foundation for model-based sequence design. However, the fitness datasets available to learn these functions are typically small relative to the large combinatorial space of sequences; characterizing how much data are needed for accurate estimation remains an open problem. There is a growing body of evidence demonstrating that empirical fitness functions display substantial sparsity when represented in terms of epistatic interactions. Moreover, the theory of Compressed Sensing provides scaling laws for the number of samples required to exactly recover a sparse function. Motivated by these results, we develop a framework to study the sparsity of fitness functions sampled from a generalization of the NK model, a widely used random field model of fitness functions. In particular, we present results that allow us to test the effect of the Generalized NK (GNK) model's interpretable parameters-sequence length, alphabet size, and assumed interactions between sequence positions-on the sparsity of fitness functions sampled from the model and, consequently, the number of measurements required to exactly recover these functions. We validate our framework by demonstrating that GNK models with parameters set according to structural considerations can be used to accurately approximate the number of samples required to recover two empirical protein fitness functions and an RNA fitness function. In addition, we show that these GNK models identify important higher-order epistatic interactions in the empirical fitness functions using only structural information.},
	language = {eng},
	number = {1},
	journal = {Proceedings of the National Academy of Sciences of the United States of America},
	author = {Brookes, David H. and Aghazadeh, Amirali and Listgarten, Jennifer},
	month = jan,
	year = {2022},
	pmid = {34937698},
	pmcid = {PMC8740588},
	keywords = {Algorithms, compressed sensing, epistasis, Epistasis, Genetic, fitness functions, Learning, Models, Theoretical, protein structure, Cites},
	pages = {e2109649118},
	file = {Full Text:/Users/agorji/Zotero/storage/PDC5S6TG/Brookes et al. - 2022 - On the sparsity of fitness functions and implicati.pdf:application/pdf},
}

@misc{li_spright_2015,
	title = {{SPRIGHT}: {A} {Fast} and {Robust} {Framework} for {Sparse} {Walsh}-{Hadamard} {Transform}},
	shorttitle = {{SPRIGHT}},
	abstract = {We consider the problem of computing the Walsh-Hadamard Transform (WHT) of some \$N\$-length input vector in the presence of noise, where the \$N\$-point Walsh spectrum is \$K\$-sparse with \$K = \{O\}(N{\textasciicircum}\{{\textbackslash}delta\})\$ scaling sub-linearly in the input dimension \$N\$ for some \$0{\textless}{\textbackslash}delta{\textless}1\$. Over the past decade, there has been a resurgence in research related to the computation of Discrete Fourier Transform (DFT) for some length-\$N\$ input signal that has a \$K\$-sparse Fourier spectrum. In particular, through a sparse-graph code design, our earlier work on the Fast Fourier Aliasing-based Sparse Transform (FFAST) algorithm computes the \$K\$-sparse DFT in time \$\{O\}(K{\textbackslash}log K)\$ by taking \$\{O\}(K)\$ noiseless samples. Inspired by the coding-theoretic design framework, Scheibler et al. proposed the Sparse Fast Hadamard Transform (SparseFHT) algorithm that elegantly computes the \$K\$-sparse WHT in the absence of noise using \$\{O\}(K{\textbackslash}log N)\$ samples in time \$\{O\}(K{\textbackslash}log{\textasciicircum}2 N)\$. However, the SparseFHT algorithm explicitly exploits the noiseless nature of the problem, and is not equipped to deal with scenarios where the observations are corrupted by noise. Therefore, a question of critical interest is whether this coding-theoretic framework can be made robust to noise. Further, if the answer is yes, what is the extra price that needs to be paid for being robust to noise? In this paper, we show, quite interestingly, that there is \{{\textbackslash}it no extra price\} that needs to be paid for being robust to noise other than a constant factor. In other words, we can maintain the same sample complexity \$\{O\}(K{\textbackslash}log N)\$ and the computational complexity \$\{O\}(K{\textbackslash}log{\textasciicircum}2 N)\$ as those of the noiseless case, using our SParse Robust Iterative Graph-based Hadamard Transform (SPRIGHT) algorithm.},
	urldate = {2023-01-04},
	publisher = {arXiv},
	author = {Li, Xiao and Bradley, Joseph K. and Pawar, Sameer and Ramchandran, Kannan},
	month = aug,
	year = {2015},
	note = {arXiv:1508.06336 [cs, math]},
	keywords = {Computer Science - Machine Learning, Computer Science - Information Theory},
	annote = {Comment: Part of our results was reported in ISIT 2014, titled "The SPRIGHT algorithm for robust sparse Hadamard Transforms."},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/UW4C8UVH/Li et al. - 2015 - SPRIGHT A Fast and Robust Framework for Sparse Wa.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/QT6FYV48/1508.html:text/html},
}

@inproceedings{stobbe_learning_2012,
	title = {Learning {Fourier} {Sparse} {Set} {Functions}},
	abstract = {Can we learn a sparse graph from observing the value of a few random cuts? This and more general problems can be reduced to the challenge of learning set functions known to have sparse Fourier support contained in some collection. We prove that if we choose O(k log{\textasciicircum}4 n) sets uniformly at random, then with high probability, observing any k-sparse function on those sets is sufficient to recover that function exactly. We further show that other properties, such as symmetry or submodularity imply structure in the Fourier spectrum, which can be exploited to further reduce sample complexity. One interesting special case  is that it suffices to observe O({\textbar}E{\textbar} log{\textasciicircum}4 {\textbar}V{\textbar}) values of a cut function to recover a graph. We demonstrate the effectiveness of our results on two real-world reconstruction problems:  graph sketching and obtaining fast approximate surrogates to expensive submodular objective functions.},
	language = {en},
	urldate = {2023-01-04},
	booktitle = {Proceedings of the {Fifteenth} {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Stobbe, Peter and Krause, Andreas},
	month = mar,
	year = {2012},
	note = {ISSN: 1938-7228},
	pages = {1125--1133},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/EAIMCDUC/Stobbe and Krause - 2012 - Learning Fourier Sparse Set Functions.pdf:application/pdf},
}

@article{scheibler2015fast,
  title={A fast Hadamard transform for signals with sublinear sparsity in the transform domain},
  author={Scheibler, Robin and Haghighatshoar, Saeid and Vetterli, Martin},
  journal={IEEE Transactions on Information Theory},
  volume={61},
  number={4},
  pages={2115--2132},
  year={2015},
  publisher={IEEE}
}

@inproceedings{gu_towards_2020,
	title = {Towards {Area}-{Efficient} {Optical} {Neural} {Networks}: {An} {FFT}-based {Architecture}},
	shorttitle = {Towards {Area}-{Efficient} {Optical} {Neural} {Networks}},
	doi = {10.1109/ASP-DAC47756.2020.9045156},
	abstract = {As a promising neuromorphic framework, the optical neural network (ONN) demonstrates ultra-high inference speed with low energy consumption. However, the previous ONN architectures have high area overhead which limits their practicality. In this paper, we propose an area-efficient ONN architecture based on structured neural networks, leveraging optical fast Fourier transform for efficient computation. A two-phase software training flow with structured pruning is proposed to further reduce the optical component utilization. Experimental results demonstrate that the proposed architecture can achieve 2.2 3.7× area cost improvement compared with the previous singular value decomposition-based architecture with comparable inference accuracy.},
	booktitle = {2020 25th {Asia} and {South} {Pacific} {Design} {Automation} {Conference} ({ASP}-{DAC})},
	author = {Gu, Jiaqi and Zhao, Zheng and Feng, Chenghao and Liu, Mingjie and Chen, Ray T. and Pan, David Z.},
	month = jan,
	year = {2020},
	note = {ISSN: 2153-697X},
	keywords = {Computer architecture, Matrix decomposition, Neural networks, Optical computing, Optical devices, Optical fiber networks, Training},
	pages = {476--481},
	file = {IEEE Xplore Abstract Record:/Users/agorji/Zotero/storage/KDFKAZWC/9045156.html:text/html;IEEE Xplore Full Text PDF:/Users/agorji/Zotero/storage/5JQ3DABJ/Gu et al. - 2020 - Towards Area-Efficient Optical Neural Networks An.pdf:application/pdf},
}

@article{li_smooth_2018,
	title = {Smooth group {L1}/2 regularization for input layer of feedforward neural networks},
	volume = {314},
	issn = {0925-2312},
	doi = {10.1016/j.neucom.2018.06.046},
	abstract = {A smooth group regularization method is proposed to identify and remove the redundant input nodes of feedforward neural networks, or equivalently the redundant dimensions of the input data of a given data set. This is achieved by introducing a smooth group L1/2 regularizer with respect to the input nodes into the error function to drive some weight vectors of the input nodes to zero. The main advantage of the method is that it can remove not only the redundant nodes, but also some redundant weights of the surviving nodes. As a comparison, the L1 regularization (Lasso) is mainly designed for removing the redundant weights, and it is not very good at removing the redundant nodes. And the group Lasso can remove the redundant nodes, but not any weight of the surviving nodes. Another advantage of the proposed method is that it uses a smooth function to replace the non-smooth absolute value function in the common L1/2 regularizer, and thus it reduces the oscillation caused by the non-smoothness and enables us to prove the convergence properties of the proposed training algorithm. Numerical simulations are performed to illustrate the efficiency of the algorithm.},
	language = {en},
	urldate = {2023-01-07},
	journal = {Neurocomputing},
	author = {Li, Feng and Zurada, Jacek M. and Wu, Wei},
	month = nov,
	year = {2018},
	keywords = {Convergence, Feature selection, Feedforward neural network, Input layer compression, Smooth group  regularization},
	pages = {109--119},
	file = {ScienceDirect Full Text PDF:/Users/agorji/Zotero/storage/USIBT9XS/Li et al. - 2018 - Smooth group L12 regularization for input layer o.pdf:application/pdf;ScienceDirect Snapshot:/Users/agorji/Zotero/storage/U8KJ8H7L/S0925231218307823.html:text/html},
}

@article{girosi_regularization_1995,
	title = {Regularization {Theory} and {Neural} {Networks} {Architectures}},
	volume = {7},
	issn = {0899-7667},
	doi = {10.1162/neco.1995.7.2.219},
	abstract = {We had previously shown that regularization principles lead to approximation schemes that are equivalent to networks with one layer of hidden units, called regularization networks. In particular, standard smoothness functionals lead to a subclass of regularization networks, the well known radial basis functions approximation schemes. This paper shows that regularization networks encompass a much broader range of approximation schemes, including many of the popular general additive models and some of the neural networks. In particular, we introduce new classes of smoothness functionals that lead to different classes of basis functions. Additive splines as well as some tensor product splines can be obtained from appropriate classes of smoothness functionals. Furthermore, the same generalization that extends radial basis functions (RBF) to hyper basis functions (HBF) also leads from additive models to ridge approximation models, containing as special cases Breiman's hinge functions, some forms of projection pursuit regression, and several types of neural networks. We propose to use the term generalized regularization networks for this broad class of approximation schemes that follow from an extension of regularization. In the probabilistic interpretation of regularization, the different classes of basis functions correspond to different classes of prior probabilities on the approximating function spaces, and therefore to different types of smoothness assumptions. In summary, different multilayer networks with one hidden layer, which we collectively call generalized regularization networks, correspond to different classes of priors and associated smoothness functionals in a classical regularization principle. Three broad classes are (1) radial basis functions that can be generalized to hyper basis functions, (2) some tensor product splines, and (3) additive splines that can be generalized to schemes of the type of ridge approximation, hinge functions, and several perceptron-like neural networks with one hidden layer.},
	number = {2},
	journal = {Neural Computation},
	author = {Girosi, Federico and Jones, Michael and Poggio, Tomaso},
	month = mar,
	year = {1995},
	note = {Conference Name: Neural Computation},
	pages = {219--269},
	file = {IEEE Xplore Abstract Record:/Users/agorji/Zotero/storage/7ACDTJUG/stamp.html:text/html;IEEE Xplore Full Text PDF:/Users/agorji/Zotero/storage/SLSH4NA3/Girosi et al. - 1995 - Regularization Theory and Neural Networks Architec.pdf:application/pdf},
}

@inproceedings{hssayni_novel_2020,
	title = {A {Novel} {Pooling} {Method} for {Regularization} of {Deep} {Neural} networks},
	doi = {10.1109/ISCV49265.2020.9204322},
	abstract = {Dropout has been introduced as a powerful regularization approach to preventing overfitting problem in deep neural networks, particularly in deep convolutional neural networks(DCNNs). A number of methods have been designed recently to improve and generalize the dropout technique. These methods include spectral dropout which achieves improved generalization and avoids overfitting by eliminating noisy and weak Fourier domain coefficients of the neural network activations. On the other hand, a pooling process plays a crucial role in deep convolutional neural networks, which serves to reduce the dimensionality of processed data for decreasing computational cost as well as for avoiding overfitting and enhancing the generalization capability of the network. For this reason, we focus on the pooling layer, and we propose a new pooling method called Spectral Dropout Pooling, by applying the Spectral dropout technique in the pooling region, in order to avoid overfitting problem, as well as to enhance the generalization ability of DCNNs. Experimental results on several image benchmarks show that Spectral Dropout Pooling outperforms the existing pooling methods in classification performance as well as is effective for improving the generalization ability of DCNNs. Moreover, we show that Spectral Dropout Pooling combined with other regularization methods, such as batch normalization, is competitive with other existing methods in classification performance.},
	booktitle = {2020 {International} {Conference} on {Intelligent} {Systems} and {Computer} {Vision} ({ISCV})},
	author = {Hssayni, El houssaine and Ettaouil, Mohamed},
	month = jun,
	year = {2020},
	keywords = {Training, Benchmark testing, Convolutional neural networks, Convolutional Neural Networks, Deep Neural Networks, Feature extraction, Generalization Ability, Noise measurement, Pooling methods, Regularization methods, Spectral Dropout., Transforms},
	pages = {1--6},
	file = {IEEE Xplore Abstract Record:/Users/agorji/Zotero/storage/RRUK2LSL/stamp.html:text/html;IEEE Xplore Full Text PDF:/Users/agorji/Zotero/storage/H58PL9ND/Hssayni and Ettaouil - 2020 - A Novel Pooling Method for Regularization of Deep .pdf:application/pdf},
}

@article{breiman2001random,
  title={Random forests},
  author={Breiman, Leo},
  journal={Machine learning},
  volume={45},
  pages={5--32},
  year={2001},
  publisher={Springer}
}

@article{khan_regularization_2019,
	title = {Regularization of deep neural networks with spectral dropout},
	volume = {110},
	issn = {0893-6080},
	doi = {10.1016/j.neunet.2018.09.009},
	abstract = {The big breakthrough on the ImageNet challenge in 2012 was partially due to the ‘Dropout’ technique used to avoid overfitting. Here, we introduce a new approach called ‘Spectral Dropout’ to improve the generalization ability of deep neural networks. We cast the proposed approach in the form of regular Convolutional Neural Network (CNN) weight layers using a decorrelation transform with fixed basis functions. Our spectral dropout method prevents overfitting by eliminating weak and ‘noisy’ Fourier domain coefficients of the neural network activations, leading to remarkably better results than the current regularization methods. Furthermore, the proposed is very efficient due to the fixed basis functions used for spectral transformation. In particular, compared to Dropout and Drop-Connect, our method significantly speeds up the network convergence rate during the training process (roughly ×2), with considerably higher neuron pruning rates (an increase of ∼30\%). We demonstrate that the spectral dropout can also be used in conjunction with other regularization approaches resulting in additional performance gains.},
	language = {en},
	urldate = {2023-01-05},
	journal = {Neural Networks},
	author = {Khan, Salman H. and Hayat, Munawar and Porikli, Fatih},
	month = feb,
	year = {2019},
	keywords = {Deep learning, Image classification, intro, Regularization, Spectral analysis},
	pages = {82--90},
	file = {ScienceDirect Full Text PDF:/Users/agorji/Zotero/storage/VBLHHLCD/Khan et al. - 2019 - Regularization of deep neural networks with spectr.pdf:application/pdf},
}

@inproceedings{li_functional_2021,
	title = {Functional {Regularization} for {Reinforcement} {Learning} via {Learned} {Fourier} {Features}},
	volume = {34},
	abstract = {We propose a simple architecture for deep reinforcement learning by embedding inputs into a learned Fourier basis and show that it improves the sample efficiency of both state-based and image-based RL. We perform infinite-width analysis of our architecture using the Neural Tangent Kernel and theoretically show that tuning the initial variance of the Fourier basis is equivalent to functional regularization of the learned deep network. That is, these learned Fourier features allow for adjusting the degree to which networks underfit or overfit different frequencies in the training data, and hence provide a controlled mechanism to improve the stability and performance of RL optimization. Empirically, this allows us to prioritize learning low-frequency functions and speed up learning by reducing networks' susceptibility to noise in the optimization process, such as during Bellman updates. Experiments on standard state-based and image-based RL benchmarks show clear benefits of our architecture over the baselines.},
	urldate = {2023-01-07},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Li, Alexander and Pathak, Deepak},
	year = {2021},
	pages = {19046--19055},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/C9SM8TYY/Li and Pathak - 2021 - Functional Regularization for Reinforcement Learni.pdf:application/pdf},
}

@inproceedings{lyle_understanding_2022,
	title = {Understanding and {Preventing} {Capacity} {Loss} in {Reinforcement} {Learning}},
	abstract = {The reinforcement learning (RL) problem is rife with sources of non-stationarity that can destabilize or inhibit learning progress. We identify a key mechanism by which this occurs in agents using neural networks as function approximators: {\textbackslash}textit\{capacity loss\}, whereby networks trained to predict a sequence of target values lose their ability to quickly fit new functions over time. We demonstrate that capacity loss occurs in a broad range of RL agents and environments, and is particularly damaging to learning progress in sparse-reward tasks. We then present a simple regularizer, Initial Feature Regularization (InFeR), that mitigates this phenomenon by regressing a subspace of features towards its value at initialization, improving performance over a state-of-the-art model-free algorithm in the Atari 2600 suite. Finally, we study how this regularization affects different notions of capacity and evaluate other mechanisms by which it may improve performance.},
	language = {en},
	urldate = {2023-01-06},
	author = {Lyle, Clare and Rowland, Mark and Dabney, Will},
	month = may,
	year = {2022},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/DFK2AI59/Lyle et al. - 2022 - Understanding and Preventing Capacity Loss in Rein.pdf:application/pdf},
}

@misc{piche_bridging_2022,
	title = {Bridging the {Gap} {Between} {Target} {Networks} and {Functional} {Regularization}},
	doi = {10.48550/arXiv.2210.12282},
	abstract = {Bootstrapping is behind much of the successes of Deep Reinforcement Learning. However, learning the value function via bootstrapping often leads to unstable training due to fast-changing target values. Target Networks are employed to stabilize training by using an additional set of lagging parameters to estimate the target values. Despite the popularity of Target Networks, their effect on the optimization is still misunderstood. In this work, we show that they act as an implicit regularizer. This regularizer has disadvantages such as being inflexible and non convex. To overcome these issues, we propose an explicit Functional Regularization that is a convex regularizer in function space and can easily be tuned. We analyze the convergence of our method theoretically and empirically demonstrate that replacing Target Networks with the more theoretically grounded Functional Regularization approach leads to better sample efficiency and performance improvements.},
	urldate = {2023-01-06},
	publisher = {arXiv},
	author = {Piche, Alexandre and Thomas, Valentin and Marino, Joseph and Pardinas, Rafael and Marconi, Gian Maria and Pal, Christopher and Khan, Mohammad Emtiyaz},
	month = oct,
	year = {2022},
	note = {arXiv:2210.12282 [cs]},
	keywords = {Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/3VN8EQF6/Piche et al. - 2022 - Bridging the Gap Between Target Networks and Funct.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/XYEI4GZK/2210.html:text/html},
}

@phdthesis{sun_function-space_2022,
	address = {Canada -- Ontario, CA},
	type = {Ph.{D}.},
	title = {Function-{Space} {Bayesian} {Learning}: from {Gaussian} {Processes} to {Bayesian} {Deep} {Learning}},
	copyright = {Database copyright ProQuest LLC; ProQuest does not claim copyright in the individual underlying works.},
	shorttitle = {Function-{Space} {Bayesian} {Learning}},
	abstract = {Bayesian methods provide a general and principled framework to quantify and update beliefs based on prior knowledge and observed evidence. This thesis presents my works about function-space Bayesian learning, whose priors and posteriors are specified and computed over the function-space. This thesis first focuses on Gaussian processes (GPs), a family of traditional function-space Bayesian models. To learn a suitable kernel for the GP prior, I propose a neural network structure that computes expressive kernel compositions. Because this structure is fully differentiable, we can learn the prior kernel from data with end-to-end gradient-based optimization. Moreover, I propose a new variational GP approximation to address computational bottlenecks in variational Gaussian processes. This variational GP achieves substantial improvement in computational complexity and approximates the true posterior with high fidelity. In the next part, I propose a general framework for function-space Bayesian learning that goes beyond GPs. I apply this framework in Bayesian deep learning and demonstrate its ability to train Bayesian neural networks with interpretable function-space priors. Furthermore, by exploiting a function-space perspective, I show that the hidden units in neural networks can be viewed as inter-domain inducing points so that I can draw an equivalence between finite neural networks and sparse Gaussian processes. In the last part, I employ Bayesian models for online memory selection and improve the robustness against imbalanced data streams in continual learning. Finally, I summarize the thesis and discuss potential future directions.},
	language = {English},
	urldate = {2023-01-06},
	school = {University of Toronto (Canada)},
	author = {Sun, Shengyang},
	year = {2022},
	note = {ISBN: 9798357550255},
	keywords = {Bayesian deep learning, Bayesian methods, Gaussian process, Intro, Machine learning, Uncertainty estimation},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/C7YVVLEB/Sun - 2022 - Function-Space Bayesian Learning from Gaussian Pr.pdf:application/pdf},
}


@misc{bai_geometric_2022,
	title = {A {Geometric} {Understanding} of {Natural} {Gradient}},
	doi = {10.48550/arXiv.2202.06232},
	abstract = {While natural gradients have been widely studied from both theoretical and empirical perspectives, we argue that some fundamental theoretical issues regarding the existence of gradients in infinite dimensional function spaces remain underexplored. We address these issues by providing a geometric perspective and mathematical framework for studying natural gradient that is more complete and rigorous than existing studies. Our results also establish new connections between natural gradients and RKHS theory, and specifically to the Neural Tangent Kernel (NTK). Based on our theoretical framework, we derive a new family of natural gradients induced by Sobolev metrics and develop computational techniques for efficient approximation in practice. Preliminary experimental results reveal the potential of this new natural gradient variant.},
	urldate = {2023-01-06},
	publisher = {arXiv},
	author = {Bai, Qinxun and Rosenberg, Steven and Xu, Wei},
	month = may,
	year = {2022},
	note = {arXiv:2202.06232 [cs]},
	keywords = {Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/7E85MKYN/Bai et al. - 2022 - A Geometric Understanding of Natural Gradient.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/4WNA447G/2202.html:text/html},
}

@article{schaeffer_learning_2017,
	title = {Learning partial differential equations via data discovery and sparse optimization},
	volume = {473},
	doi = {10.1098/rspa.2016.0446},
	abstract = {We investigate the problem of learning an evolution equation directly from some given data. This work develops a learning algorithm to identify the terms in the underlying partial differential equations and to approximate the coefficients of the terms only using data. The algorithm uses sparse optimization in order to perform feature selection and parameter estimation. The features are data driven in the sense that they are constructed using nonlinear algebraic equations on the spatial derivatives of the data. Several numerical experiments show the proposed method's robustness to data noise and size, its ability to capture the true features of the data, and its capability of performing additional analytics. Examples include shock equations, pattern formation, fluid flow and turbulence, and oscillatory convection.},
	number = {2197},
	urldate = {2023-01-06},
	journal = {Proceedings of the Royal Society A: Mathematical, Physical and Engineering Sciences},
	author = {Schaeffer, Hayden},
	month = jan,
	year = {2017},
	note = {Publisher: Royal Society},
	keywords = {Intro, feature selection, machine learning, parameter estimation, partial differential equations, sparse optimization},
	pages = {20160446},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/5RAICL34/Schaeffer - 2017 - Learning partial differential equations via data d.pdf:application/pdf},
}

@inproceedings{rudner_tractable_2022,
	title = {Tractable {Function}-{Space} {Variational} {Inference} in {Bayesian} {Neural} {Networks}},
	abstract = {Reliable predictive uncertainty estimation plays an important role in enabling the deployment of neural networks to safety-critical settings. A popular approach for estimating the predictive uncertainty of neural networks is to define a prior distribution over the network parameters, infer an approximate posterior distribution, and use it to make stochastic predictions. However, explicit inference over neural network parameters makes it difficult to incorporate meaningful prior information about the data-generating process into the model. In this paper, we pursue an alternative approach. Recognizing that the primary object of interest in most settings is the distribution over functions induced by the posterior distribution over neural network parameters, we frame Bayesian inference in neural networks explicitly as inferring a posterior distribution over functions and propose a scalable function-space variational inference method that allows incorporating prior information and results in reliable predictive uncertainty estimates. We show that the proposed method leads to state-of-the-art uncertainty estimation and predictive performance on a range of prediction tasks and demonstrate that it performs well on a challenging safety-critical medical diagnosis task in which reliable uncertainty estimation is essential.},
	language = {en},
	urldate = {2023-01-06},
	author = {Rudner, Tim G. J. and Chen, Zonghao and Teh, Yee Whye and Gal, Yarin},
	month = oct,
	year = {2022},
	keywords = {Intro},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/IDYYK4YG/Rudner et al. - 2022 - Tractable Function-Space Variational Inference in .pdf:application/pdf},
}

@misc{wang_sparcl_2022,
	title = {{SparCL}: {Sparse} {Continual} {Learning} on the {Edge}},
	shorttitle = {{SparCL}},
	doi = {10.48550/arXiv.2209.09476},
	abstract = {Existing work in continual learning (CL) focuses on mitigating catastrophic forgetting, i.e., model performance deterioration on past tasks when learning a new task. However, the training efficiency of a CL system is under-investigated, which limits the real-world application of CL systems under resource-limited scenarios. In this work, we propose a novel framework called Sparse Continual Learning(SparCL), which is the first study that leverages sparsity to enable cost-effective continual learning on edge devices. SparCL achieves both training acceleration and accuracy preservation through the synergy of three aspects: weight sparsity, data efficiency, and gradient sparsity. Specifically, we propose task-aware dynamic masking (TDM) to learn a sparse network throughout the entire CL process, dynamic data removal (DDR) to remove less informative training data, and dynamic gradient masking (DGM) to sparsify the gradient updates. Each of them not only improves efficiency, but also further mitigates catastrophic forgetting. SparCL consistently improves the training efficiency of existing state-of-the-art (SOTA) CL methods by at most 23X less training FLOPs, and, surprisingly, further improves the SOTA accuracy by at most 1.7\%. SparCL also outperforms competitive baselines obtained from adapting SOTA sparse training methods to the CL setting in both efficiency and accuracy. We also evaluate the effectiveness of SparCL on a real mobile phone, further indicating the practical potential of our method.},
	urldate = {2023-01-06},
	publisher = {arXiv},
	author = {Wang, Zifeng and Zhan, Zheng and Gong, Yifan and Yuan, Geng and Niu, Wei and Jian, Tong and Ren, Bin and Ioannidis, Stratis and Wang, Yanzhi and Dy, Jennifer},
	month = sep,
	year = {2022},
	note = {arXiv:2209.09476 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Artificial Intelligence, Computer Science - Computer Vision and Pattern Recognition},
	annote = {Comment: Published at NeurIPS 2022 as a conference paper},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/8BG6GL25/Wang et al. - 2022 - SparCL Sparse Continual Learning on the Edge.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/KRD3GJJ4/2209.html:text/html},
}

@inproceedings{kim_deep_2022,
	title = {Deep {Learning} and {Symbolic} {Regression} for {Discovering} {Parametric} {Equations}},
	abstract = {Symbolic regression is a machine learning technique that can learn the governing formulas from data and thus has the potential to transform scientific discovery. However, symbolic regression is still limited in the complexity of the systems that it can analyze. Deep learning on the other hand has transformed machine learning in its ability to analyze extremely complex and high-dimensional datasets. Here we develop a method that uses neural networks to extend symbolic regression to parametric systems where some coefficient may vary as a function of time but the underlying governing equation remains constant. We demonstrate our method on various analytic expressions and PDEs with varying coefficients and show that it extrapolate well outside of the training domain. The neural network-based architecture can also integrate with other deep learning architectures so that it can analyze high-dimensional data while being trained end-to-end in a single step. To this end we integrate our architecture with convolutional neural networks and train the system end-to-end to discover various physical quantities from 1D images of spring systems where the spring constant may vary.},
	language = {en},
	urldate = {2023-01-06},
	author = {Kim, Samuel and Zhang, Michael and Lu, Peter Y. and Soljacic, Marin},
	month = jul,
	year = {2022},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/5JI5FNTH/Kim et al. - 2022 - Deep Learning and Symbolic Regression for Discover.pdf:application/pdf},
}

@inproceedings{louizos_learning_2022,
	title = {Learning {Sparse} {Neural} {Networks} through {L}\_0 {Regularization}},
	abstract = {We propose a practical method for \$L\_0\$ norm regularization for neural networks: pruning the network during training by encouraging weights to become exactly zero. Such regularization is interesting since (1) it can greatly speed up training and inference, and (2) it can improve generalization. AIC and BIC, well-known model selection criteria, are special cases of \$L\_0\$ regularization. However, since the \$L\_0\$ norm of weights is non-differentiable, we cannot incorporate it directly as a regularization term in the objective function. We propose a solution through the inclusion of a collection of non-negative stochastic gates, which collectively determine which weights to set to zero. We show that, somewhat surprisingly, for certain distributions over the gates, the expected \$L\_0\$ regularized objective is differentiable with respect to the distribution parameters. We further propose the {\textbackslash}emph\{hard concrete\} distribution for the gates, which is obtained by ``stretching'' a binary concrete distribution and then transforming its samples with a hard-sigmoid. The parameters of the distribution over the gates can then be jointly optimized with the original network parameters. As a result our method allows for straightforward and efficient learning of model structures with stochastic gradient descent and allows for conditional computation in a principled way. We perform various experiments to demonstrate the effectiveness of the resulting approach and regularizer.},
	language = {en},
	urldate = {2023-01-06},
	author = {Louizos, Christos and Welling, Max and Kingma, Diederik P.},
	month = feb,
	year = {2022},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/CRXUZ9CM/Louizos et al. - 2022 - Learning Sparse Neural Networks through L_0 Regula.pdf:application/pdf},
}

@misc{guiroy_towards_2019,
	title = {Towards {Understanding} {Generalization} in {Gradient}-{Based} {Meta}-{Learning}},
	doi = {10.48550/arXiv.1907.07287},
	abstract = {In this work we study generalization of neural networks in gradient-based meta-learning by analyzing various properties of the objective landscapes. We experimentally demonstrate that as meta-training progresses, the meta-test solutions, obtained after adapting the meta-train solution of the model, to new tasks via few steps of gradient-based fine-tuning, become flatter, lower in loss, and further away from the meta-train solution. We also show that those meta-test solutions become flatter even as generalization starts to degrade, thus providing an experimental evidence against the correlation between generalization and flat minima in the paradigm of gradient-based meta-leaning. Furthermore, we provide empirical evidence that generalization to new tasks is correlated with the coherence between their adaptation trajectories in parameter space, measured by the average cosine similarity between task-specific trajectory directions, starting from a same meta-train solution. We also show that coherence of meta-test gradients, measured by the average inner product between the task-specific gradient vectors evaluated at meta-train solution, is also correlated with generalization. Based on these observations, we propose a novel regularizer for MAML and provide experimental evidence for its effectiveness.},
	urldate = {2023-01-06},
	publisher = {arXiv},
	author = {Guiroy, Simon and Verma, Vikas and Pal, Christopher},
	month = jul,
	year = {2019},
	note = {arXiv:1907.07287 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Computer Vision and Pattern Recognition},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/KEYLTNMK/Guiroy et al. - 2019 - Towards Understanding Generalization in Gradient-B.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/EJGM56XZ/1907.html:text/html},
}

@inproceedings{dong_how_2021,
	title = {How {Should} {Pre}-{Trained} {Language} {Models} {Be} {Fine}-{Tuned} {Towards} {Adversarial} {Robustness}?},
	volume = {34},
	abstract = {The fine-tuning of pre-trained language models has a great success in many NLP fields. Yet, it is strikingly vulnerable to adversarial examples, e.g., word substitution attacks using only synonyms can easily fool a BERT-based sentiment analysis model. In this paper, we demonstrate that adversarial training, the prevalent defense technique, does not directly fit a conventional fine-tuning scenario, because it suffers severely from catastrophic forgetting: failing to retain the generic and robust linguistic features that have already been captured by the pre-trained model. In this light, we propose Robust Informative Fine-Tuning (RIFT), a novel adversarial fine-tuning method from an information-theoretical perspective. In particular, RIFT encourages an objective model to retain the features learned from the pre-trained model throughout the entire fine-tuning process, whereas a conventional one only uses the pre-trained weights for initialization. Experimental results show that RIFT consistently outperforms the state-of-the-arts on two popular NLP tasks: sentiment analysis and natural language inference, under different attacks across various pre-trained language models.},
	urldate = {2023-01-06},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Dong, Xinshuai and Luu, Anh Tuan and Lin, Min and Yan, Shuicheng and Zhang, Hanwang},
	year = {2021},
	pages = {4356--4369},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/F88C44JP/Dong et al. - 2021 - How Should Pre-Trained Language Models Be Fine-Tun.pdf:application/pdf},
}

@inproceedings{pan_continual_2020,
	title = {Continual {Deep} {Learning} by {Functional} {Regularisation} of {Memorable} {Past}},
	volume = {33},
	abstract = {Continually learning new skills is important for intelligent systems, yet standard deep learning methods suffer from catastrophic forgetting of the past. Recent works address this with weight regularisation. Functional regularisation, although computationally expensive, is expected to perform better, but rarely does so in practice. In this paper, we fix this issue by using a new functional-regularisation approach that utilises a few memorable past examples crucial to avoid forgetting. By using a Gaussian Process formulation of deep networks, our approach enables training in weight-space while identifying both the memorable past and a functional prior. Our method achieves state-of-the-art performance on standard benchmarks and opens a new direction for life-long learning where regularisation and memory-based methods are naturally combined.},
	urldate = {2023-01-06},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Pan, Pingbo and Swaroop, Siddharth and Immer, Alexander and Eschenhagen, Runa and Turner, Richard and Khan, Mohammad Emtiyaz E},
	year = {2020},
	pages = {4453--4464},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/4DTWK4LD/Pan et al. - 2020 - Continual Deep Learning by Functional Regularisati.pdf:application/pdf},
}

@misc{titsias_functional_2020,
	title = {Functional {Regularisation} for {Continual} {Learning} with {Gaussian} {Processes}},
	doi = {10.48550/arXiv.1901.11356},
	abstract = {We introduce a framework for Continual Learning (CL) based on Bayesian inference over the function space rather than the parameters of a deep neural network. This method, referred to as functional regularisation for Continual Learning, avoids forgetting a previous task by constructing and memorising an approximate posterior belief over the underlying task-specific function. To achieve this we rely on a Gaussian process obtained by treating the weights of the last layer of a neural network as random and Gaussian distributed. Then, the training algorithm sequentially encounters tasks and constructs posterior beliefs over the task-specific functions by using inducing point sparse Gaussian process methods. At each step a new task is first learnt and then a summary is constructed consisting of (i) inducing inputs -- a fixed-size subset of the task inputs selected such that it optimally represents the task -- and (ii) a posterior distribution over the function values at these inputs. This summary then regularises learning of future tasks, through Kullback-Leibler regularisation terms. Our method thus unites approaches focused on (pseudo-)rehearsal with those derived from a sequential Bayesian inference perspective in a principled way, leading to strong results on accepted benchmarks.},
	urldate = {2023-01-06},
	publisher = {arXiv},
	author = {Titsias, Michalis K. and Schwarz, Jonathan and Matthews, Alexander G. de G. and Pascanu, Razvan and Teh, Yee Whye},
	month = feb,
	year = {2020},
	note = {arXiv:1901.11356 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	annote = {Comment: 17 pages, 7 figures},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/NW3M5GZT/Titsias et al. - 2020 - Functional Regularisation for Continual Learning w.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/4SSE97WU/1901.html:text/html},
}

@inproceedings{cheng_control_2019,
	title = {Control {Regularization} for {Reduced} {Variance} {Reinforcement} {Learning}},
	abstract = {Dealing with high variance is a significant challenge in model-free reinforcement learning (RL). Existing methods are unreliable, exhibiting high variance in performance from run to run using different initializations/seeds. Focusing on problems arising in continuous control, we propose a functional regularization approach to augmenting model-free RL. In particular, we regularize the behavior of the deep policy to be similar to a policy prior, i.e., we regularize in function space. We show that functional regularization yields a bias-variance trade-off, and propose an adaptive tuning strategy to optimize this trade-off. When the policy prior has control-theoretic stability guarantees, we further show that this regularization approximately preserves those stability guarantees throughout learning. We validate our approach empirically on a range of settings, and demonstrate significantly reduced variance, guaranteed dynamic stability, and more efficient learning than deep RL alone.},
	language = {en},
	urldate = {2023-01-06},
	booktitle = {Proceedings of the 36th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Cheng, Richard and Verma, Abhinav and Orosz, Gabor and Chaudhuri, Swarat and Yue, Yisong and Burdick, Joel},
	month = may,
	year = {2019},
	note = {ISSN: 2640-3498},
	pages = {1141--1150},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/Y3RCWQ8L/Cheng et al. - 2019 - Control Regularization for Reduced Variance Reinfo.pdf:application/pdf;Supplementary PDF:/Users/agorji/Zotero/storage/NWVB8H5E/Cheng et al. - 2019 - Control Regularization for Reduced Variance Reinfo.pdf:application/pdf},
}

@misc{von_oswald_continual_2022,
	title = {Continual learning with hypernetworks},
	doi = {10.48550/arXiv.1906.00695},
	abstract = {Artificial neural networks suffer from catastrophic forgetting when they are sequentially trained on multiple tasks. To overcome this problem, we present a novel approach based on task-conditioned hypernetworks, i.e., networks that generate the weights of a target model based on task identity. Continual learning (CL) is less difficult for this class of models thanks to a simple key feature: instead of recalling the input-output relations of all previously seen data, task-conditioned hypernetworks only require rehearsing task-specific weight realizations, which can be maintained in memory using a simple regularizer. Besides achieving state-of-the-art performance on standard CL benchmarks, additional experiments on long task sequences reveal that task-conditioned hypernetworks display a very large capacity to retain previous memories. Notably, such long memory lifetimes are achieved in a compressive regime, when the number of trainable hypernetwork weights is comparable or smaller than target network size. We provide insight into the structure of low-dimensional task embedding spaces (the input space of the hypernetwork) and show that task-conditioned hypernetworks demonstrate transfer learning. Finally, forward information transfer is further supported by empirical results on a challenging CL benchmark based on the CIFAR-10/100 image datasets.},
	urldate = {2023-01-06},
	publisher = {arXiv},
	author = {von Oswald, Johannes and Henning, Christian and Grewe, Benjamin F. and Sacramento, João},
	month = apr,
	year = {2022},
	note = {arXiv:1906.00695 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Artificial Intelligence, 68T99},
	annote = {Comment: Published at ICLR 2020},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/WXPVRKQV/von Oswald et al. - 2022 - Continual learning with hypernetworks.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/W8SGGM4F/1906.html:text/html},
}

@misc{li_fast_2019,
	title = {Fast {Haar} {Transforms} for {Graph} {Neural} {Networks}},
	doi = {10.48550/arXiv.1907.04786},
	abstract = {Graph Neural Networks (GNNs) have become a topic of intense research recently due to their powerful capability in high-dimensional classification and regression tasks for graph-structured data. However, as GNNs typically define the graph convolution by the orthonormal basis for the graph Laplacian, they suffer from high computational cost when the graph size is large. This paper introduces Haar basis which is a sparse and localized orthonormal system for a coarse-grained chain on graph. The graph convolution under Haar basis, called Haar convolution, can be defined accordingly for GNNs. The sparsity and locality of the Haar basis allow Fast Haar Transforms (FHTs) on graph, by which a fast evaluation of Haar convolution between graph data and filters can be achieved. We conduct experiments on GNNs equipped with Haar convolution, which demonstrates state-of-the-art results on graph-based regression and node classification tasks.},
	urldate = {2023-01-06},
	publisher = {arXiv},
	author = {Li, Ming and Ma, Zheng and Wang, Yu Guang and Zhuang, Xiaosheng},
	month = oct,
	year = {2019},
	note = {arXiv:1907.04786 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, 42C40, 05C85, 11Y16},
	annote = {Comment: 24 pages, 5 figures, 3 tables},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/KLEAAPTU/Li et al. - 2019 - Fast Haar Transforms for Graph Neural Networks.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/B4JRH7EL/1907.html:text/html},
}

@article{leenay_large_2019,
	title = {Large dataset enables prediction of repair after {CRISPR}–{Cas9} editing in primary {T} cells},
	volume = {37},
	copyright = {2019 The Author(s), under exclusive licence to Springer Nature America, Inc.},
	issn = {1546-1696},
	doi = {10.1038/s41587-019-0203-2},
	abstract = {Understanding of repair outcomes after Cas9-induced DNA cleavage is still limited, especially in primary human cells. We sequence repair outcomes at 1,656 on-target genomic sites in primary human T cells and use these data to train a machine learning model, which we have called CRISPR Repair Outcome (SPROUT). SPROUT accurately predicts the length, probability and sequence of nucleotide insertions and deletions, and will facilitate design of SpCas9 guide RNAs in therapeutically important primary human cells.},
	language = {en},
	number = {9},
	urldate = {2023-01-05},
	journal = {Nature Biotechnology},
	author = {Leenay, Ryan T. and Aghazadeh, Amirali and Hiatt, Joseph and Tse, David and Roth, Theodore L. and Apathy, Ryan and Shifrut, Eric and Hultquist, Judd F. and Krogan, Nevan and Wu, Zhenqin and Cirolia, Giana and Canaj, Hera and Leonetti, Manuel D. and Marson, Alexander and May, Andrew P. and Zou, James},
	month = sep,
	year = {2019},
	note = {Number: 9
Publisher: Nature Publishing Group},
	keywords = {Biological techniques, CRISPR-Cas9 genome editing, DNA damage and repair, Targeted gene repair},
	pages = {1034--1037},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/BURVKWXR/Leenay et al. - 2019 - Large dataset enables prediction of repair after C.pdf:application/pdf},
}

@article{brookes_sparsity_2022-1,
	title = {On the sparsity of fitness functions and implications for learning},
	volume = {119},
	doi = {10.1073/pnas.2109649118},
	abstract = {Fitness functions map biological sequences to a scalar property of interest. Accurate estimation of these functions yields biological insight and sets the foundation for model-based sequence design. However, the fitness datasets available to learn these functions are typically small relative to the large combinatorial space of sequences; characterizing how much data are needed for accurate estimation remains an open problem. There is a growing body of evidence demonstrating that empirical fitness functions display substantial sparsity when represented in terms of epistatic interactions. Moreover, the theory of Compressed Sensing provides scaling laws for the number of samples required to exactly recover a sparse function. Motivated by these results, we develop a framework to study the sparsity of fitness functions sampled from a generalization of the NK model, a widely used random field model of fitness functions. In particular, we present results that allow us to test the effect of the Generalized NK (GNK) model’s interpretable parameters—sequence length, alphabet size, and assumed interactions between sequence positions—on the sparsity of fitness functions sampled from the model and, consequently, the number of measurements required to exactly recover these functions. We validate our framework by demonstrating that GNK models with parameters set according to structural considerations can be used to accurately approximate the number of samples required to recover two empirical protein fitness functions and an RNA fitness function. In addition, we show that these GNK models identify important higher-order epistatic interactions in the empirical fitness functions using only structural information.},
	number = {1},
	urldate = {2023-01-05},
	journal = {Proceedings of the National Academy of Sciences},
	author = {Brookes, David H. and Aghazadeh, Amirali and Listgarten, Jennifer},
	month = jan,
	year = {2022},
	note = {Publisher: Proceedings of the National Academy of Sciences},
	pages = {e2109649118},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/4ZVVDBNE/Brookes et al. - 2022 - On the sparsity of fitness functions and implicati.pdf:application/pdf},
}

@inproceedings{tseng_fourier-transform-based_2020,
	title = {Fourier-transform-based attribution priors improve the interpretability and stability of deep learning models for genomics},
	volume = {33},
	abstract = {Deep learning models can accurately map genomic DNA sequences to associated functional molecular readouts such as protein-DNA binding data. Base-resolution importance (i.e. "attribution") scores inferred from these models can highlight predictive sequence motifs and syntax. Unfortunately, these models are prone to overfitting and are sensitive to random initializations, often resulting in noisy and irreproducible attributions that obfuscate underlying motifs. To address these shortcomings, we propose a novel attribution prior, where the Fourier transform of input-level attribution scores are computed at training-time, and high-frequency components of the Fourier spectrum are penalized. We evaluate different model architectures with and without our attribution prior, training on genome-wide binary labels or continuous molecular profiles. We show that our attribution prior significantly improves models' stability, interpretability, and performance on held-out data, especially when training data is severely limited. Our attribution prior also allows models to identify biologically meaningful sequence motifs more sensitively and precisely within individual regulatory elements. The prior is agnostic to the model architecture or predicted experimental assay, yet provides similar gains across all experiments. This work represents an important advancement in improving the reliability of deep learning models for deciphering the regulatory code of the genome.},
	urldate = {2023-01-05},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Tseng, Alex and Shrikumar, Avanti and Kundaje, Anshul},
	year = {2020},
	pages = {1913--1923},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/IBWNXRX9/Tseng et al. - 2020 - Fourier-transform-based attribution priors improve.pdf:application/pdf},
}

@inproceedings{ha_adaptive_2021,
	title = {Adaptive wavelet distillation from neural networks through interpretations},
	volume = {34},
	abstract = {Recent deep-learning models have achieved impressive prediction performance, but often sacrifice interpretability and computational efficiency. Interpretability is crucial in many disciplines, such as science and medicine, where models must be carefully vetted or where interpretation is the goal itself. Moreover, interpretable models are concise and often yield computational efficiency. Here, we propose adaptive wavelet distillation (AWD), a method which aims to distill information from a trained neural network into a wavelet transform. Specifically, AWD penalizes feature attributions of a neural network in the wavelet domain to learn an effective multi-resolution wavelet transform. The resulting model is highly predictive, concise, computationally efficient, and has properties (such as a multi-scale structure) which make it easy to interpret. In close collaboration with domain experts, we showcase how AWD addresses challenges in two real-world settings: cosmological parameter inference and molecular-partner prediction. In both cases, AWD yields a scientifically interpretable and concise model which gives predictive performance better than state-of-the-art neural networks. Moreover, AWD identifies predictive features that are scientifically meaningful in the context of respective domains. All code and models are released in a full-fledged package available on Github.},
	urldate = {2023-01-05},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Ha, Wooseok and Singh, Chandan and Lanusse, Francois and Upadhyayula, Srigokul and Yu, Bin},
	year = {2021},
	pages = {20669--20682},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/I7NWSYGN/Ha et al. - 2021 - Adaptive wavelet distillation from neural networks.pdf:application/pdf},
}

@misc{li_fast_2019-1,
	title = {Fast {Haar} {Transforms} for {Graph} {Neural} {Networks}},
	doi = {10.48550/arXiv.1907.04786},
	abstract = {Graph Neural Networks (GNNs) have become a topic of intense research recently due to their powerful capability in high-dimensional classification and regression tasks for graph-structured data. However, as GNNs typically define the graph convolution by the orthonormal basis for the graph Laplacian, they suffer from high computational cost when the graph size is large. This paper introduces Haar basis which is a sparse and localized orthonormal system for a coarse-grained chain on graph. The graph convolution under Haar basis, called Haar convolution, can be defined accordingly for GNNs. The sparsity and locality of the Haar basis allow Fast Haar Transforms (FHTs) on graph, by which a fast evaluation of Haar convolution between graph data and filters can be achieved. We conduct experiments on GNNs equipped with Haar convolution, which demonstrates state-of-the-art results on graph-based regression and node classification tasks.},
	urldate = {2023-01-05},
	publisher = {arXiv},
	author = {Li, Ming and Ma, Zheng and Wang, Yu Guang and Zhuang, Xiaosheng},
	month = oct,
	year = {2019},
	note = {arXiv:1907.04786 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, 42C40, 05C85, 11Y16},
	annote = {Comment: 24 pages, 5 figures, 3 tables},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/5VSJYL6I/Li et al. - 2019 - Fast Haar Transforms for Graph Neural Networks.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/T42MAPE9/1907.html:text/html},
}

@inproceedings{li_active_2015,
	title = {An {Active} {Learning} {Framework} using {Sparse}-{Graph} {Codes} for {Sparse} {Polynomials} and {Graph} {Sketching}},
	volume = {28},
	urldate = {2023-01-05},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Li, Xiao and Ramchandran, Kannan},
	year = {2015},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/KL5QZK79/Li and Ramchandran - 2015 - An Active Learning Framework using Sparse-Graph Co.pdf:application/pdf},
}

@article{ballal_sparse_2020,
	title = {Sparse {Epistatic} {Patterns} in the {Evolution} of {Terpene} {Synthases}},
	volume = {37},
	issn = {0737-4038},
	doi = {10.1093/molbev/msaa052},
	abstract = {We explore sequence determinants of enzyme activity and specificity in a major enzyme family of terpene synthases. Most enzymes in this family catalyze reactions that produce cyclic terpenes—complex hydrocarbons widely used by plants and insects in diverse biological processes such as defense, communication, and symbiosis. To analyze the molecular mechanisms of emergence of terpene cyclization, we have carried out in-depth examination of mutational space around (E)-β-farnesene synthase, an Artemisia annua enzyme which catalyzes production of a linear hydrocarbon chain. Each mutant enzyme in our synthetic libraries was characterized biochemically, and the resulting reaction rate data were used as input to the Michaelis–Menten model of enzyme kinetics, in which free energies were represented as sums of one-amino-acid contributions and two-amino-acid couplings. Our model predicts measured reaction rates with high accuracy and yields free energy landscapes characterized by relatively few coupling terms. As a result, the Michaelis–Menten free energy landscapes have simple, interpretable structure and exhibit little epistasis. We have also developed biophysical fitness models based on the assumption that highly fit enzymes have evolved to maximize the output of correct products, such as cyclic products or a specific product of interest, while minimizing the output of byproducts. This approach results in nonlinear fitness landscapes that are considerably more epistatic. Overall, our experimental and computational framework provides focused characterization of evolutionary emergence of novel enzymatic functions in the context of microevolutionary exploration of sequence space around naturally occurring enzymes.},
	number = {7},
	urldate = {2023-01-05},
	journal = {Molecular Biology and Evolution},
	author = {Ballal, Aditya and Laurendon, Caroline and Salmon, Melissa and Vardakou, Maria and Cheema, Jitender and Defernez, Marianne and O’Maille, Paul E and Morozov, Alexandre V},
	month = jul,
	year = {2020},
	pages = {1907--1924},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/GRLGKH72/Ballal et al. - 2020 - Sparse Epistatic Patterns in the Evolution of Terp.pdf:application/pdf},
}

@article{yang_higher-order_2019,
	title = {Higher-order epistasis shapes the fitness landscape of a xenobiotic-degrading enzyme},
	volume = {15},
	copyright = {2019 The Author(s), under exclusive licence to Springer Nature America, Inc.},
	issn = {1552-4469},
	doi = {10.1038/s41589-019-0386-3},
	abstract = {Characterizing the adaptive landscapes that encompass the emergence of novel enzyme functions can provide molecular insights into both enzymatic and evolutionary mechanisms. Here, we combine ancestral protein reconstruction with biochemical, structural and mutational analyses to characterize the functional evolution of methyl-parathion hydrolase (MPH), an organophosphate-degrading enzyme. We identify five mutations that are necessary and sufficient for the evolution of MPH from an ancestral dihydrocoumarin hydrolase. In-depth analyses of the adaptive landscapes encompassing this evolutionary transition revealed that the mutations form a complex interaction network, defined in part by higher-order epistasis, that constrained the adaptive pathways available. By also characterizing the adaptive landscapes in terms of their functional activities towards three additional organophosphate substrates, we reveal that subtle differences in the polarity of the substrate substituents drastically alter the network of epistatic interactions. Our work suggests that the mutations function collectively to enable substrate recognition via subtle structural repositioning.},
	language = {en},
	number = {11},
	urldate = {2023-01-05},
	journal = {Nature Chemical Biology},
	author = {Yang, Gloria and Anderson, Dave W. and Baier, Florian and Dohmen, Elias and Hong, Nansook and Carr, Paul D. and Kamerlin, Shina Caroline Lynn and Jackson, Colin J. and Bornberg-Bauer, Erich and Tokuriki, Nobuhiko},
	month = nov,
	year = {2019},
	note = {Number: 11
Publisher: Nature Publishing Group},
	keywords = {Enzyme mechanisms, Enzymes, Protein design, X-ray crystallography},
	pages = {1120--1128},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/9VB2QE8F/Yang et al. - 2019 - Higher-order epistasis shapes the fitness landscap.pdf:application/pdf},
}

@article{sailer_detecting_2017,
	title = {Detecting {High}-{Order} {Epistasis} in {Nonlinear} {Genotype}-{Phenotype} {Maps}},
	volume = {205},
	issn = {1943-2631},
	doi = {10.1534/genetics.116.195214},
	abstract = {High-order epistasis has been observed in many genotype-phenotype maps. These multi-way interactions between mutations may be useful for dissecting complex traits and could have profound implications for evolution. Alternatively, they could be a statistical artifact. High-order epistasis models assume the effects of mutations should add, when they could in fact multiply or combine in some other nonlinear way. A mismatch in the “scale” of the epistasis model and the scale of the underlying map would lead to spurious epistasis. In this article, we develop an approach to estimate the nonlinear scales of arbitrary genotype-phenotype maps. We can then linearize these maps and extract high-order epistasis. We investigated seven experimental genotype-phenotype maps for which high-order epistasis had been reported previously. We find that five of the seven maps exhibited nonlinear scales. Interestingly, even after accounting for nonlinearity, we found statistically significant high-order epistasis in all seven maps. The contributions of high-order epistasis to the total variation ranged from 2.2 to 31.0\%, with an average across maps of 12.7\%. Our results provide strong evidence for extensive high-order epistasis, even after nonlinear scale is taken into account. Further, we describe a simple method to estimate and account for nonlinearity in genotype-phenotype maps.},
	number = {3},
	urldate = {2023-01-05},
	journal = {Genetics},
	author = {Sailer, Zachary R and Harms, Michael J},
	month = mar,
	year = {2017},
	pages = {1079--1088},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/E65P3WGU/Sailer and Harms - 2017 - Detecting High-Order Epistasis in Nonlinear Genoty.pdf:application/pdf},
}

@article{chen_different_2002,
	title = {On {Different} {Facets} of {Regularization} {Theory}},
	volume = {14},
	issn = {0899-7667},
	doi = {10.1162/089976602760805296},
	abstract = {This review provides a comprehensive understanding of regularization theory from different perspectives, emphasizing smoothness and simplicity principles. Using the tools of operator theory and Fourier analysis, it is shown that the solution of the classical Tikhonov regularization problem can be derived from the regularized functional defined by a linear differential (integral) operator in the spatial (Fourier) domain. State-ofthe-art research relevant to the regularization theory is reviewed, covering Occam's razor, minimum length description, Bayesian theory, pruning algorithms, informational (entropy) theory, statistical learning theory, and equivalent regularization. The universal principle of regularization in terms of Kolmogorov complexity is discussed. Finally, some prospective studies on regularization theory and beyond are suggested.},
	number = {12},
	urldate = {2023-01-05},
	journal = {Neural Computation},
	author = {Chen, Zhe and Haykin, Simon},
	month = dec,
	year = {2002},
	pages = {2791--2846},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/2QBV9JQW/Chen and Haykin - 2002 - On Different Facets of Regularization Theory.pdf:application/pdf;Snapshot:/Users/agorji/Zotero/storage/FVZDUFVA/On-Different-Facets-of-Regularization-Theory.html:text/html},
}

@inproceedings{liu_spectral_2019,
	title = {Spectral {Regularization} for {Combating} {Mode} {Collapse} in {GANs}},
	urldate = {2023-01-05},
	author = {Liu, Kanglin and Tang, Wenming and Zhou, Fei and Qiu, Guoping},
	year = {2019},
	pages = {6382--6390},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/QI62GZSI/Liu et al. - 2019 - Spectral Regularization for Combating Mode Collaps.pdf:application/pdf},
}

@misc{yoshida_spectral_2017,
	title = {Spectral {Norm} {Regularization} for {Improving} the {Generalizability} of {Deep} {Learning}},
	doi = {10.48550/arXiv.1705.10941},
	abstract = {We investigate the generalizability of deep learning based on the sensitivity to input perturbation. We hypothesize that the high sensitivity to the perturbation of data degrades the performance on it. To reduce the sensitivity to perturbation, we propose a simple and effective regularization method, referred to as spectral norm regularization, which penalizes the high spectral norm of weight matrices in neural networks. We provide supportive evidence for the abovementioned hypothesis by experimentally confirming that the models trained using spectral norm regularization exhibit better generalizability than other baseline methods.},
	urldate = {2023-01-05},
	publisher = {arXiv},
	author = {Yoshida, Yuichi and Miyato, Takeru},
	month = may,
	year = {2017},
	note = {arXiv:1705.10941 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/S6AZ3JIS/Yoshida and Miyato - 2017 - Spectral Norm Regularization for Improving the Gen.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/2JQ2MDI6/1705.html:text/html},
}

@inproceedings{rahaman_spectral_2019,
	title = {On the {Spectral} {Bias} of {Neural} {Networks}},
	abstract = {Neural networks are known to be a class of highly expressive functions able to fit even random input-output mappings with 100\% accuracy. In this work we present properties of neural networks that complement this aspect of expressivity. By using tools from Fourier analysis, we highlight a learning bias of deep networks towards low frequency functions – i.e. functions that vary globally without local fluctuations – which manifests itself as a frequency-dependent learning speed. Intuitively, this property is in line with the observation that over-parameterized networks prioritize learning simple patterns that generalize across data samples. We also investigate the role of the shape of the data manifold by presenting empirical and theoretical evidence that, somewhat counter-intuitively, learning higher frequencies gets easier with increasing manifold complexity.},
	language = {en},
	urldate = {2023-01-05},
	booktitle = {Proceedings of the 36th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Rahaman, Nasim and Baratin, Aristide and Arpit, Devansh and Draxler, Felix and Lin, Min and Hamprecht, Fred and Bengio, Yoshua and Courville, Aaron},
	month = may,
	year = {2019},
	note = {ISSN: 2640-3498},
	pages = {5301--5310},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/EQ6QRI9J/Rahaman et al. - 2019 - On the Spectral Bias of Neural Networks.pdf:application/pdf;Supplementary PDF:/Users/agorji/Zotero/storage/VFVQM5D3/Rahaman et al. - 2019 - On the Spectral Bias of Neural Networks.pdf:application/pdf},
}

@misc{hou_spectral_2022,
	title = {Spectral {Regularization}: an {Inductive} {Bias} for {Sequence} {Modeling}},
	shorttitle = {Spectral {Regularization}},
	doi = {10.48550/arXiv.2211.02255},
	abstract = {Various forms of regularization in learning tasks strive for different notions of simplicity. This paper presents a spectral regularization technique, which attaches a unique inductive bias to sequence modeling based on an intuitive concept of simplicity defined in the Chomsky hierarchy. From fundamental connections between Hankel matrices and regular grammars, we propose to use the trace norm of the Hankel matrix, the tightest convex relaxation of its rank, as the spectral regularizer. To cope with the fact that the Hankel matrix is bi-infinite, we propose an unbiased stochastic estimator for its trace norm. Ultimately, we demonstrate experimental results on Tomita grammars, which exhibit the potential benefits of spectral regularization and validate the proposed stochastic estimator.},
	urldate = {2023-01-05},
	publisher = {arXiv},
	author = {Hou, Kaiwen and Rabusseau, Guillaume},
	month = nov,
	year = {2022},
	note = {arXiv:2211.02255 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Artificial Intelligence, Computer Science - Computation and Language},
	annote = {Comment: LearnAut paper in 2022 (https://learnaut22.github.io/programme.html\#abstract-20)},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/EQQK6RFY/Hou and Rabusseau - 2022 - Spectral Regularization an Inductive Bias for Seq.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/A3776XN4/2211.html:text/html},
}

@misc{gao_learning_2022,
	title = {Learning {Stable} {Graph} {Neural} {Networks} via {Spectral} {Regularization}},
	doi = {10.48550/arXiv.2211.06966},
	abstract = {Stability of graph neural networks (GNNs) characterizes how GNNs react to graph perturbations and provides guarantees for architecture performance in noisy scenarios. This paper develops a self-regularized graph neural network (SR-GNN) solution that improves the architecture stability by regularizing the filter frequency responses in the graph spectral domain. The SR-GNN considers not only the graph signal as input but also the eigenvectors of the underlying graph, where the signal is processed to generate task-relevant features and the eigenvectors to characterize the frequency responses at each layer. We train the SR-GNN by minimizing the cost function and regularizing the maximal frequency response close to one. The former improves the architecture performance, while the latter tightens the perturbation stability and alleviates the information loss through multi-layer propagation. We further show the SR-GNN preserves the permutation equivariance, which allows to explore the internal symmetries of graph signals and to exhibit transference on similar graph structures. Numerical results with source localization and movie recommendation corroborate our findings and show the SR-GNN yields a comparable performance with the vanilla GNN on the unperturbed graph but improves substantially the stability.},
	urldate = {2023-01-05},
	publisher = {arXiv},
	author = {Gao, Zhan and Isufi, Elvin},
	month = nov,
	year = {2022},
	note = {arXiv:2211.06966 [cs, eess]},
	keywords = {Computer Science - Machine Learning, Electrical Engineering and Systems Science - Signal Processing},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/HUQX6AYP/Gao and Isufi - 2022 - Learning Stable Graph Neural Networks via Spectral.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/E42H2NC2/2211.html:text/html},
}

@article{aghazadeh_epistatic_2021,
	title = {Epistatic {Net} allows the sparse spectral regularization of deep neural networks for inferring fitness functions},
	volume = {12},
	copyright = {2021 The Author(s)},
	issn = {2041-1723},
	doi = {10.1038/s41467-021-25371-3},
	abstract = {Despite recent advances in high-throughput combinatorial mutagenesis assays, the number of labeled sequences available to predict molecular functions has remained small for the vastness of the sequence space combined with the ruggedness of many fitness functions. While deep neural networks (DNNs) can capture high-order epistatic interactions among the mutational sites, they tend to overfit to the small number of labeled sequences available for training. Here, we developed Epistatic Net (EN), a method for spectral regularization of DNNs that exploits evidence that epistatic interactions in many fitness functions are sparse. We built a scalable extension of EN, usable for larger sequences, which enables spectral regularization using fast sparse recovery algorithms informed by coding theory. Results on several biological landscapes show that EN consistently improves the prediction accuracy of DNNs and enables them to outperform competing models which assume other priors. EN estimates the higher-order epistatic interactions of DNNs trained on massive sequence spaces-a computational problem that otherwise takes years to solve.},
	language = {en},
	number = {1},
	urldate = {2023-01-05},
	journal = {Nature Communications},
	author = {Aghazadeh, Amirali and Nisonoff, Hunter and Ocal, Orhan and Brookes, David H. and Huang, Yijie and Koyluoglu, O. Ozan and Listgarten, Jennifer and Ramchandran, Kannan},
	month = sep,
	year = {2021},
	note = {Number: 1
Publisher: Nature Publishing Group},
	keywords = {Machine learning, Applied mathematics, Computer science, Protein function predictions},
	pages = {5225},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/DUDDRXNN/Aghazadeh et al. - 2021 - Epistatic Net allows the sparse spectral regulariz.pdf:application/pdf},
}

@inproceedings{benjamin_measuring_2019,
title={Measuring and regularizing networks in function space},
author={Ari Benjamin and David Rolnick and Konrad Kording},
booktitle={International Conference on Learning Representations},
year={2019}
}

@misc{pereyra_regularizing_2017,
	title = {Regularizing {Neural} {Networks} by {Penalizing} {Confident} {Output} {Distributions}},
	abstract = {We systematically explore regularizing neural networks by penalizing low entropy output distributions. We show that penalizing low entropy output distributions, which has been shown to improve exploration in reinforcement learning, acts as a strong regularizer in supervised learning. Furthermore, we connect a maximum entropy based confidence penalty to label smoothing through the direction of the KL divergence. We exhaustively evaluate the proposed confidence penalty and label smoothing on 6 common benchmarks: image classification (MNIST and Cifar-10), language modeling (Penn Treebank), machine translation (WMT'14 English-to-German), and speech recognition (TIMIT and WSJ). We find that both label smoothing and the confidence penalty improve state-of-the-art models across benchmarks without modifying existing hyperparameters, suggesting the wide applicability of these regularizers.},
	urldate = {2023-01-05},
	publisher = {arXiv},
	author = {Pereyra, Gabriel and Tucker, George and Chorowski, Jan and Kaiser, Łukasz and Hinton, Geoffrey},
	month = jan,
	year = {2017},
	note = {arXiv:1701.06548 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Neural and Evolutionary Computing},
	annote = {Comment: Submitted to ICLR 2017},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/FEM5KYRE/Pereyra et al. - 2017 - Regularizing Neural Networks by Penalizing Confide.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/HL5X22IR/1701.html:text/html},
}

@inproceedings{
sun_functional_2019,
title={{Functional} {Variational} {Bayesian} {Neural} {Networks}},
author={Shengyang Sun and Guodong Zhang and Jiaxin Shi and Roger Grosse},
booktitle={International Conference on Learning Representations},
year={2019},
}

@inproceedings{
wang_function_2019,
title={Function Space Particle Optimization for Bayesian Neural Networks},
author={Ziyu Wang and Tongzheng Ren and Jun Zhu and Bo Zhang},
booktitle={International Conference on Learning Representations},
year={2019},
}


@misc{zhou_higher-order_2022,
	title = {Higher-order epistasis and phenotypic prediction},
	copyright = {© 2022, Posted by Cold Spring Harbor Laboratory. The copyright holder for this pre-print is the author. All rights reserved. The material may not be redistributed, re-used or adapted without the author's permission.},
	doi = {10.1101/2020.10.14.339804},
	abstract = {Contemporary high-throughput mutagenesis experiments are providing an increasingly detailed view of the complex patterns of genetic interaction that occur between multiple mutations within a single protein or regulatory element. By simultaneously measuring the effects of thousands of combinations of mutations, these experiments have revealed that the genotype-phenotype relationship typically reflects genetic interactions not only between pairs of sites, but also higher-order interactions among larger numbers of sites. However, modeling and understanding these higher-order interactions remains challenging. Here, we present a method for reconstructing sequence-to-function mappings from partially observed data that can accommodate all orders of genetic interaction. The main idea is to make predictions for unobserved genotypes that match the type and extent of epistasis found in the observed data. This information on the type and extent of epistasis can be extracted by considering how phenotypic correlations change as a function of mutational distance, which is equivalent to estimating the fraction of phenotypic variance due to each order of genetic interaction (additive, pairwise, three-way, etc.). Using these estimated variance components, we then define an empirical Bayes prior that in expectation matches the observed pattern of epistasis, and reconstruct the genotype-phenotype mapping by conducting Gaussian process regression under this prior. To demonstrate the power of this approach, we present an application to the antibody-binding domain GB1 and also provide a detailed exploration of a dataset consisting of high-throughput measurements for the splicing efficiency of human pre-mRNA 5′ splice sites, for which we also validate our model predictions via additional low-throughput experiments.},
	language = {en},
	urldate = {2023-01-05},
	publisher = {bioRxiv},
	author = {Zhou, Juannan and Wong, Mandy S. and Chen, Wei-Chia and Krainer, Adrian R. and Kinney, Justin B. and McCandlish, David M.},
	month = mar,
	year = {2022},
	note = {Pages: 2020.10.14.339804
Section: New Results},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/GSTRSA6C/Zhou et al. - 2022 - Higher-order epistasis and phenotypic prediction.pdf:application/pdf},
}

@misc{aghazadeh_spectral_2022,
	title = {Spectral {Regularization} {Allows} {Data}-frugal {Learning} over {Combinatorial} {Spaces}},
	abstract = {Data-driven machine learning models are being increasingly employed in several important inference problems in biology, chemistry, and physics which require learning over combinatorial spaces. Recent empirical evidence (see, e.g., [1], [2], [3]) suggests that regularizing the spectral representation of such models improves their generalization power when labeled data is scarce. However, despite these empirical studies, the theoretical underpinning of when and how spectral regularization enables improved generalization is poorly understood. In this paper, we focus on learning pseudo-Boolean functions and demonstrate that regularizing the empirical mean squared error by the L\_1 norm of the spectral transform of the learned function reshapes the loss landscape and allows for data-frugal learning, under a restricted secant condition on the learner's empirical error measured against the ground truth function. Under a weaker quadratic growth condition, we show that stationary points which also approximately interpolate the training data points achieve statistically optimal generalization performance. Complementing our theory, we empirically demonstrate that running gradient descent on the regularized loss results in a better generalization performance compared to baseline algorithms in several data-scarce real-world problems.},
	urldate = {2023-01-05},
	publisher = {arXiv},
	author = {Aghazadeh, Amirali and Rajaraman, Nived and Tu, Tony and Ramchandran, Kannan},
	month = oct,
	year = {2022},
	note = {arXiv:2210.02604 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/2H2CKJ7R/Aghazadeh et al. - 2022 - Spectral Regularization Allows Data-frugal Learnin.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/9FE8UTJH/2210.html:text/html},
}

@article{aghazadeh_epistatic_2021-1,
	title = {Epistatic {Net} allows the sparse spectral regularization of deep neural networks for inferring fitness functions},
	volume = {12},
	copyright = {2021 The Author(s)},
	issn = {2041-1723},
	doi = {10.1038/s41467-021-25371-3},
	abstract = {Despite recent advances in high-throughput combinatorial mutagenesis assays, the number of labeled sequences available to predict molecular functions has remained small for the vastness of the sequence space combined with the ruggedness of many fitness functions. While deep neural networks (DNNs) can capture high-order epistatic interactions among the mutational sites, they tend to overfit to the small number of labeled sequences available for training. Here, we developed Epistatic Net (EN), a method for spectral regularization of DNNs that exploits evidence that epistatic interactions in many fitness functions are sparse. We built a scalable extension of EN, usable for larger sequences, which enables spectral regularization using fast sparse recovery algorithms informed by coding theory. Results on several biological landscapes show that EN consistently improves the prediction accuracy of DNNs and enables them to outperform competing models which assume other priors. EN estimates the higher-order epistatic interactions of DNNs trained on massive sequence spaces-a computational problem that otherwise takes years to solve.},
	language = {en},
	number = {1},
	urldate = {2023-01-05},
	journal = {Nature Communications},
	author = {Aghazadeh, Amirali and Nisonoff, Hunter and Ocal, Orhan and Brookes, David H. and Huang, Yijie and Koyluoglu, O. Ozan and Listgarten, Jennifer and Ramchandran, Kannan},
	month = sep,
	year = {2021},
	note = {Number: 1
Publisher: Nature Publishing Group},
	keywords = {Machine learning, Applied mathematics, Computer science, Protein function predictions},
	pages = {5225},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/QB2NCP7V/Aghazadeh et al. - 2021 - Epistatic Net allows the sparse spectral regulariz.pdf:application/pdf},
}

@article{brookes_sparsity_2022-2,
	title = {On the sparsity of fitness functions and implications for learning},
	volume = {119},
	doi = {10.1073/pnas.2109649118},
	abstract = {Fitness functions map biological sequences to a scalar property of interest. Accurate estimation of these functions yields biological insight and sets the foundation for model-based sequence design. However, the fitness datasets available to learn these functions are typically small relative to the large combinatorial space of sequences; characterizing how much data are needed for accurate estimation remains an open problem. There is a growing body of evidence demonstrating that empirical fitness functions display substantial sparsity when represented in terms of epistatic interactions. Moreover, the theory of Compressed Sensing provides scaling laws for the number of samples required to exactly recover a sparse function. Motivated by these results, we develop a framework to study the sparsity of fitness functions sampled from a generalization of the NK model, a widely used random field model of fitness functions. In particular, we present results that allow us to test the effect of the Generalized NK (GNK) model’s interpretable parameters—sequence length, alphabet size, and assumed interactions between sequence positions—on the sparsity of fitness functions sampled from the model and, consequently, the number of measurements required to exactly recover these functions. We validate our framework by demonstrating that GNK models with parameters set according to structural considerations can be used to accurately approximate the number of samples required to recover two empirical protein fitness functions and an RNA fitness function. In addition, we show that these GNK models identify important higher-order epistatic interactions in the empirical fitness functions using only structural information.},
	number = {1},
	urldate = {2023-01-10},
	journal = {Proceedings of the National Academy of Sciences},
	author = {Brookes, David H. and Aghazadeh, Amirali and Listgarten, Jennifer},
	month = jan,
	year = {2022},
	note = {Publisher: Proceedings of the National Academy of Sciences},
	pages = {e2109649118},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/MVBKQCZ7/Brookes et al. - 2022 - On the sparsity of fitness functions and implicati.pdf:application/pdf},
}

@article{yang_higher-order_2019-1,
	title = {Higher-order epistasis shapes the fitness landscape of a xenobiotic-degrading enzyme},
	volume = {15},
	copyright = {2019 The Author(s), under exclusive licence to Springer Nature America, Inc.},
	issn = {1552-4469},
	doi = {10.1038/s41589-019-0386-3},
	abstract = {Characterizing the adaptive landscapes that encompass the emergence of novel enzyme functions can provide molecular insights into both enzymatic and evolutionary mechanisms. Here, we combine ancestral protein reconstruction with biochemical, structural and mutational analyses to characterize the functional evolution of methyl-parathion hydrolase (MPH), an organophosphate-degrading enzyme. We identify five mutations that are necessary and sufficient for the evolution of MPH from an ancestral dihydrocoumarin hydrolase. In-depth analyses of the adaptive landscapes encompassing this evolutionary transition revealed that the mutations form a complex interaction network, defined in part by higher-order epistasis, that constrained the adaptive pathways available. By also characterizing the adaptive landscapes in terms of their functional activities towards three additional organophosphate substrates, we reveal that subtle differences in the polarity of the substrate substituents drastically alter the network of epistatic interactions. Our work suggests that the mutations function collectively to enable substrate recognition via subtle structural repositioning.},
	language = {en},
	number = {11},
	urldate = {2023-01-12},
	journal = {Nature Chemical Biology},
	author = {Yang, Gloria and Anderson, Dave W. and Baier, Florian and Dohmen, Elias and Hong, Nansook and Carr, Paul D. and Kamerlin, Shina Caroline Lynn and Jackson, Colin J. and Bornberg-Bauer, Erich and Tokuriki, Nobuhiko},
	month = nov,
	year = {2019},
	note = {Number: 11
Publisher: Nature Publishing Group},
	keywords = {Enzyme mechanisms, Enzymes, Protein design, X-ray crystallography},
	pages = {1120--1128},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/5PWTER3Z/Yang et al. - 2019 - Higher-order epistasis shapes the fitness landscap.pdf:application/pdf},
}

@misc{noauthor_higher-order_nodate,
	title = {Higher-order epistasis shapes the fitness landscape of a xenobioticdegrading enzyme - {Google} {Search}},
	urldate = {2023-01-12},
	file = {Higher-order epistasis shapes the fitness landscape of a xenobioticdegrading enzyme - Google Search:/Users/agorji/Zotero/storage/B879KXJE/search.html:text/html},
}

@article{sailer_detecting_2017-1,
	title = {Detecting {High}-{Order} {Epistasis} in {Nonlinear} {Genotype}-{Phenotype} {Maps}},
	volume = {205},
	issn = {1943-2631},
	doi = {10.1534/genetics.116.195214},
	abstract = {High-order epistasis has been observed in many genotype-phenotype maps. These multi-way interactions between mutations may be useful for dissecting complex traits and could have profound implications for evolution. Alternatively, they could be a statistical artifact. High-order epistasis models assume the effects of mutations should add, when they could in fact multiply or combine in some other nonlinear way. A mismatch in the “scale” of the epistasis model and the scale of the underlying map would lead to spurious epistasis. In this article, we develop an approach to estimate the nonlinear scales of arbitrary genotype-phenotype maps. We can then linearize these maps and extract high-order epistasis. We investigated seven experimental genotype-phenotype maps for which high-order epistasis had been reported previously. We find that five of the seven maps exhibited nonlinear scales. Interestingly, even after accounting for nonlinearity, we found statistically significant high-order epistasis in all seven maps. The contributions of high-order epistasis to the total variation ranged from 2.2 to 31.0\%, with an average across maps of 12.7\%. Our results provide strong evidence for extensive high-order epistasis, even after nonlinear scale is taken into account. Further, we describe a simple method to estimate and account for nonlinearity in genotype-phenotype maps.},
	number = {3},
	urldate = {2023-01-12},
	journal = {Genetics},
	author = {Sailer, Zachary R and Harms, Michael J},
	month = mar,
	year = {2017},
	pages = {1079--1088},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/HE5MZIMD/Sailer and Harms - 2017 - Detecting High-Order Epistasis in Nonlinear Genoty.pdf:application/pdf},
}

@article{ballal_sparse_2020-1,
	title = {Sparse {Epistatic} {Patterns} in the {Evolution} of {Terpene} {Synthases}},
	volume = {37},
	issn = {0737-4038},
	doi = {10.1093/molbev/msaa052},
	abstract = {We explore sequence determinants of enzyme activity and specificity in a major enzyme family of terpene synthases. Most enzymes in this family catalyze reactions that produce cyclic terpenes—complex hydrocarbons widely used by plants and insects in diverse biological processes such as defense, communication, and symbiosis. To analyze the molecular mechanisms of emergence of terpene cyclization, we have carried out in-depth examination of mutational space around (E)-β-farnesene synthase, an Artemisia annua enzyme which catalyzes production of a linear hydrocarbon chain. Each mutant enzyme in our synthetic libraries was characterized biochemically, and the resulting reaction rate data were used as input to the Michaelis–Menten model of enzyme kinetics, in which free energies were represented as sums of one-amino-acid contributions and two-amino-acid couplings. Our model predicts measured reaction rates with high accuracy and yields free energy landscapes characterized by relatively few coupling terms. As a result, the Michaelis–Menten free energy landscapes have simple, interpretable structure and exhibit little epistasis. We have also developed biophysical fitness models based on the assumption that highly fit enzymes have evolved to maximize the output of correct products, such as cyclic products or a specific product of interest, while minimizing the output of byproducts. This approach results in nonlinear fitness landscapes that are considerably more epistatic. Overall, our experimental and computational framework provides focused characterization of evolutionary emergence of novel enzymatic functions in the context of microevolutionary exploration of sequence space around naturally occurring enzymes.},
	number = {7},
	urldate = {2023-01-12},
	journal = {Molecular Biology and Evolution},
	author = {Ballal, Aditya and Laurendon, Caroline and Salmon, Melissa and Vardakou, Maria and Cheema, Jitender and Defernez, Marianne and O’Maille, Paul E and Morozov, Alexandre V},
	month = jul,
	year = {2020},
	pages = {1907--1924},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/CIJH56UG/Ballal et al. - 2020 - Sparse Epistatic Patterns in the Evolution of Terp.pdf:application/pdf},
}

@misc{poggio_theory_2017,
	title = {Theory of {Deep} {Learning} {III}: explaining the non-overfitting puzzle},
	shorttitle = {Theory of {Deep} {Learning} {III}},
	author = {Poggio, Tomaso and Kawaguchi, Keji and Liao, Qianli and Miranda, Brando and Rosasco, Lorenzo and Boix, Xavier and Hidary, Jack and Mhaskar, Hrushikesh},
	year = {2017},
}

@misc{poggio_theory_2018,
	title = {Theory of {Deep} {Learning} {III}: explaining the non-overfitting puzzle},
	shorttitle = {Theory of {Deep} {Learning} {III}},
	doi = {10.48550/arXiv.1801.00173},
	abstract = {A main puzzle of deep networks revolves around the absence of overfitting despite large overparametrization and despite the large capacity demonstrated by zero training error on randomly labeled data. In this note, we show that the dynamics associated to gradient descent minimization of nonlinear networks is topologically equivalent, near the asymptotically stable minima of the empirical error, to linear gradient system in a quadratic potential with a degenerate (for square loss) or almost degenerate (for logistic or crossentropy loss) Hessian. The proposition depends on the qualitative theory of dynamical systems and is supported by numerical results. Our main propositions extend to deep nonlinear networks two properties of gradient descent for linear networks, that have been recently established (1) to be key to their generalization properties: 1. Gradient descent enforces a form of implicit regularization controlled by the number of iterations, and asymptotically converges to the minimum norm solution for appropriate initial conditions of gradient descent. This implies that there is usually an optimum early stopping that avoids overfitting of the loss. This property, valid for the square loss and many other loss functions, is relevant especially for regression. 2. For classification, the asymptotic convergence to the minimum norm solution implies convergence to the maximum margin solution which guarantees good classification error for "low noise" datasets. This property holds for loss functions such as the logistic and cross-entropy loss independently of the initial conditions. The robustness to overparametrization has suggestive implications for the robustness of the architecture of deep convolutional networks with respect to the curse of dimensionality.},
	urldate = {2023-01-12},
	publisher = {arXiv},
	author = {Poggio, Tomaso and Kawaguchi, Kenji and Liao, Qianli and Miranda, Brando and Rosasco, Lorenzo and Boix, Xavier and Hidary, Jack and Mhaskar, Hrushikesh},
	month = jan,
	year = {2018},
	note = {arXiv:1801.00173 [cs]},
	keywords = {Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/H5U2UM3B/Poggio et al. - 2018 - Theory of Deep Learning III explaining the non-ov.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/VDQPYZR9/1801.html:text/html},
}

@misc{neyshabur_exploring_2017,
	title = {Exploring {Generalization} in {Deep} {Learning}},
	doi = {10.48550/arXiv.1706.08947},
	abstract = {With a goal of understanding what drives generalization in deep networks, we consider several recently suggested explanations, including norm-based control, sharpness and robustness. We study how these measures can ensure generalization, highlighting the importance of scale normalization, and making a connection between sharpness and PAC-Bayes theory. We then investigate how well the measures explain different observed phenomena.},
	urldate = {2023-01-12},
	publisher = {arXiv},
	author = {Neyshabur, Behnam and Bhojanapalli, Srinadh and McAllester, David and Srebro, Nathan},
	month = jul,
	year = {2017},
	note = {arXiv:1706.08947 [cs]},
	keywords = {Computer Science - Machine Learning},
	annote = {Comment: 19 pages, 8 figures},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/VTBCGED9/Neyshabur et al. - 2017 - Exploring Generalization in Deep Learning.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/Q8ESB4W3/1706.html:text/html},
}
@inproceedings{neyshabur2015search,
  title={In Search of the Real Inductive Bias: On the Role of Implicit Regularization in Deep Learning.},
  author={Neyshabur, Behnam and Tomioka, Ryota and Srebro, Nathan},
  booktitle={ICLR (Workshop)},
  year={2015}
}
@article{poelwijk_learning_2019,
	title = {Learning the pattern of epistasis linking genotype and phenotype in a protein},
	volume = {10},
	copyright = {2019 The Author(s)},
	issn = {2041-1723},
	doi = {10.1038/s41467-019-12130-8},
	abstract = {Understanding the pattern of epistasis—the non-independence of mutations—is critical for relating genotype and phenotype. However, the combinatorial complexity of potential epistatic interactions has severely limited the analysis of this problem. Using new mutational approaches, we report a comprehensive experimental study of all 213 mutants that link two phenotypically distinct variants of the Entacmaea quadricolor fluorescent protein—an opportunity to examine epistasis up to the 13th order. The data show the existence of many high-order epistatic interactions between mutations, but also reveal extraordinary sparsity, enabling novel experimental and computational strategies for learning the relevant epistasis. We demonstrate that such information, in turn, can be used to accurately predict phenotypes in practical situations where the number of measurements is limited. Finally, we show how the observed epistasis shapes the solution space of single-mutation trajectories between the parental fluorescent proteins, informative about the protein’s evolutionary potential. This work provides conceptual and experimental strategies to profoundly characterize epistasis in a protein, relevant to both natural and laboratory evolution.},
	language = {en},
	number = {1},
	urldate = {2023-01-12},
	journal = {Nature Communications},
	author = {Poelwijk, Frank J. and Socolich, Michael and Ranganathan, Rama},
	month = sep,
	year = {2019},
	note = {Number: 1
Publisher: Nature Publishing Group},
	keywords = {Computational biology and bioinformatics, Epistasis, Molecular evolution, Synthetic biology},
	pages = {4213},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/2JPWDYWF/Poelwijk et al. - 2019 - Learning the pattern of epistasis linking genotype.pdf:application/pdf},
}

@article{aghazadeh_crisprl_2020,
	title = {{CRISPRL} and: {Interpretable} large-scale inference of {DNA} repair landscape based on a spectral approach},
	volume = {36},
	issn = {1367-4811},
	shorttitle = {{CRISPRL} and},
	doi = {10.1093/bioinformatics/btaa505},
	abstract = {SUMMARY: We propose a new spectral framework for reliable training, scalable inference and interpretable explanation of the DNA repair outcome following a Cas9 cutting. Our framework, dubbed CRISPRL and, relies on an unexploited observation about the nature of the repair process: the landscape of the DNA repair is highly sparse in the (Walsh-Hadamard) spectral domain. This observation enables our framework to address key shortcomings that limit the interpretability and scaling of current deep-learning-based DNA repair models. In particular, CRISPRL and reduces the time to compute the full DNA repair landscape from a striking 5230 years to 1 week and the sampling complexity from 1012 to 3 million guide RNAs with only a small loss in accuracy (R2R2 ∼ 0.9). Our proposed framework is based on a divide-and-conquer strategy that uses a fast peeling algorithm to learn the DNA repair models. CRISPRL and captures lower-degree features around the cut site, which enrich for short insertions and deletions as well as higher-degree microhomology patterns that enrich for longer deletions.
AVAILABILITY AND IMPLEMENTATION: The CRISPRL and software is publicly available at https://github.com/UCBASiCS/CRISPRLand.},
	language = {eng},
	number = {Suppl\_1},
	journal = {Bioinformatics (Oxford, England)},
	author = {Aghazadeh, Amirali and Ocal, Orhan and Ramchandran, Kannan},
	month = jul,
	year = {2020},
	pmid = {32657417},
	pmcid = {PMC7355252},
	keywords = {Algorithms, DNA Repair, Software},
	pages = {i560--i568},
	file = {Full Text:/Users/agorji/Zotero/storage/B78GTNMH/Aghazadeh et al. - 2020 - CRISPRL and Interpretable large-scale inference o.pdf:application/pdf},
}

@inproceedings{soudry_implicit_2022,
	title = {The {Implicit} {Bias} of {Gradient} {Descent} on {Separable} {Data}},
	abstract = {We show that gradient descent on an unregularized logistic regression problem, for almost all separable datasets, converges to the same direction as the max-margin solution. The result generalizes also to other monotone decreasing loss functions with an infimum at infinity, and we also discuss a multi-class generalizations to the cross entropy loss. Furthermore, we show this convergence is very slow, and only logarithmic in the convergence of the loss itself. This can help explain the benefit of continuing to optimize the logistic or cross-entropy loss even after the training error is zero and the training loss is extremely small, and, as we show, even if the validation loss increases. Our methodology can also aid in understanding implicit regularization in more complex models and with other optimization methods.},
	language = {en},
	urldate = {2023-01-14},
	author = {Soudry, Daniel and Hoffer, Elad and Nacson, Mor Shpigel and Srebro, Nathan},
	month = feb,
	year = {2022},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/5BVZ5D54/Soudry et al. - 2022 - The Implicit Bias of Gradient Descent on Separable.pdf:application/pdf},
}

@inproceedings{neyshabur_exploring_2017-1,
	title = {Exploring {Generalization} in {Deep} {Learning}},
	volume = {30},
	abstract = {With a goal of understanding what drives generalization in deep networks, we consider several recently suggested explanations, including norm-based control, sharpness and robustness. We study how these measures can ensure generalization, highlighting the importance of scale normalization, and making a connection between sharpness and PAC-Bayes theory.  We then investigate how well the measures explain different observed phenomena.},
	urldate = {2023-01-14},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Neyshabur, Behnam and Bhojanapalli, Srinadh and Mcallester, David and Srebro, Nati},
	year = {2017},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/HIHQGM4X/Neyshabur et al. - 2017 - Exploring Generalization in Deep Learning.pdf:application/pdf},
}

@inproceedings{soudry_implicit_2022-1,
	title = {The {Implicit} {Bias} of {Gradient} {Descent} on {Separable} {Data}},
	abstract = {We show that gradient descent on an unregularized logistic regression problem, for almost all separable datasets, converges to the same direction as the max-margin solution. The result generalizes also to other monotone decreasing loss functions with an infimum at infinity, and we also discuss a multi-class generalizations to the cross entropy loss. Furthermore, we show this convergence is very slow, and only logarithmic in the convergence of the loss itself. This can help explain the benefit of continuing to optimize the logistic or cross-entropy loss even after the training error is zero and the training loss is extremely small, and, as we show, even if the validation loss increases. Our methodology can also aid in understanding implicit regularization in more complex models and with other optimization methods.},
	language = {en},
	urldate = {2023-01-14},
	author = {Soudry, Daniel and Hoffer, Elad and Nacson, Mor Shpigel and Srebro, Nathan},
	month = feb,
	year = {2022},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/R2TZKPJQ/Soudry et al. - 2022 - The Implicit Bias of Gradient Descent on Separable.pdf:application/pdf},
}

@inproceedings{soudry_implicit_2018,
	title = {The {Implicit} {Bias} of {Gradient} {Descent} on {Separable} {Data}},
	booktitle = {International {Conference} on {Learning} {Representations}},
	author = {Soudry, Daniel and Hoffer, Elad and Srebro, Nathan},
	year = {2018},
	file = {Full Text:/Users/agorji/Zotero/storage/JWLXER2T/Soudry et al. - 2018 - The Implicit Bias of Gradient Descent on Separable.pdf:application/pdf},
}

@inproceedings{shah_pitfalls_2020,
	title = {The {Pitfalls} of {Simplicity} {Bias} in {Neural} {Networks}},
	volume = {33},
	abstract = {Several works have proposed Simplicity Bias (SB)---the tendency of standard training procedures such as Stochastic Gradient Descent (SGD) to find simple models---to justify why neural networks generalize well [Arpit et al. 2017, Nakkiran et al. 2019, Valle-Perez et al. 2019]. However, the precise notion of simplicity remains vague. Furthermore, previous settings [Soudry et al. 2018, Gunasekar et al. 2018] that use SB to theoretically justify why neural networks generalize well do not simultaneously capture the non-robustness of neural networks---a widely observed phenomenon in practice [Goodfellow et al. 2014, Jo and Bengio 2017]. We attempt to reconcile SB and the superior standard generalization of neural networks with the non-robustness observed in practice by introducing piecewise-linear and image-based datasets, which (a) incorporate a precise notion of simplicity, (b) comprise multiple predictive features with varying levels of simplicity, and (c) capture the non-robustness of neural networks trained on real data. Using theory and empirics on these datasets, we make four observations:
(i) SB of SGD and variants can be extreme: neural networks can exclusively rely on the simplest feature and remain invariant to all predictive complex features.
(ii) The extreme aspect of SB could explain why seemingly benign distribution shifts and small adversarial perturbations significantly degrade model performance.
(iii) Contrary to conventional wisdom, SB can also hurt generalization on the same data distribution, as SB persists even when the simplest feature has less predictive power than the more complex features.
(iv) Common approaches to improve generalization and robustness---ensembles and adversarial training---can fail in mitigating SB and its pitfalls.
Given the role of SB in training neural networks, we hope that the proposed datasets and methods serve as an effective testbed to evaluate novel algorithmic approaches aimed at avoiding the pitfalls of SB.},
	urldate = {2023-01-14},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Shah, Harshay and Tamuly, Kaustav and Raghunathan, Aditi and Jain, Prateek and Netrapalli, Praneeth},
	year = {2020},
	pages = {9573--9585},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/XQR7XMNB/Shah et al. - 2020 - The Pitfalls of Simplicity Bias in Neural Networks.pdf:application/pdf},
}


@article{wu_adaptation_2016,
	title = {Adaptation in protein fitness landscapes is facilitated by indirect paths},
	volume = {5},
	issn = {2050-084X},
	doi = {10.7554/eLife.16965},
	abstract = {The structure of fitness landscapes is critical for understanding adaptive protein evolution. Previous empirical studies on fitness landscapes were confined to either the neighborhood around the wild type sequence, involving mostly single and double mutants, or a combinatorially complete subgraph involving only two amino acids at each site. In reality, the dimensionality of protein sequence space is higher (20L) and there may be higher-order interactions among more than two sites. Here we experimentally characterized the fitness landscape of four sites in protein GB1, containing 204 = 160,000 variants. We found that while reciprocal sign epistasis blocked many direct paths of adaptation, such evolutionary traps could be circumvented by indirect paths through genotype space involving gain and subsequent loss of mutations. These indirect paths alleviate the constraint on adaptive protein evolution, suggesting that the heretofore neglected dimensions of sequence space may change our views on how proteins evolve.},
	urldate = {2023-01-29},
	journal = {eLife},
	author = {Wu, Nicholas C and Dai, Lei and Olson, C Anders and Lloyd-Smith, James O and Sun, Ren},
	editor = {Neher, Richard A},
	month = jul,
	year = {2016},
	note = {Publisher: eLife Sciences Publications, Ltd},
	keywords = {adaptive evolution, deep sequencing, epistasis, fitness landscape, saturation mutagenesis},
	pages = {e16965},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/SQ6IVWEI/Wu et al. - 2016 - Adaptation in protein fitness landscapes is facili.pdf:application/pdf},
}


@article{sarkisyan_local_2016,
	title = {Local fitness landscape of the green fluorescent protein},
	volume = {533},
	copyright = {2015 Macmillan Publishers Limited. All rights reserved},
	issn = {1476-4687},
	doi = {10.1038/nature17995},
	abstract = {Comprehensive genotype–phenotype mapping of the green fluorescent protein shows that the local fitness peak is narrow, shaped by a high prevalence of epistatic interactions, providing for the loss of fluorescence when the joint effect of mutations exceeds a threshold.},
	language = {en},
	number = {7603},
	urldate = {2023-01-29},
	journal = {Nature},
	author = {Sarkisyan, Karen S. and Bolotin, Dmitry A. and Meer, Margarita V. and Usmanova, Dinara R. and Mishin, Alexander S. and Sharonov, George V. and Ivankov, Dmitry N. and Bozhanova, Nina G. and Baranov, Mikhail S. and Soylemez, Onuralp and Bogatyreva, Natalya S. and Vlasov, Peter K. and Egorov, Evgeny S. and Logacheva, Maria D. and Kondrashov, Alexey S. and Chudakov, Dmitry M. and Putintseva, Ekaterina V. and Mamedov, Ilgar Z. and Tawfik, Dan S. and Lukyanov, Konstantin A. and Kondrashov, Fyodor A.},
	month = may,
	year = {2016},
	note = {Number: 7603
Publisher: Nature Publishing Group},
	keywords = {Computational biology and bioinformatics, Evolution},
	pages = {397--401},
	file = {Accepted Version:/Users/agorji/Zotero/storage/2KZDMD96/Sarkisyan et al. - 2016 - Local fitness landscape of the green fluorescent p.pdf:application/pdf},
}


@inproceedings{nugteren_cltune_2015,
	title = {{CLTune}: {A} {Generic} {Auto}-{Tuner} for {OpenCL} {Kernels}},
	shorttitle = {{CLTune}},
	doi = {10.1109/MCSoC.2015.10},
	abstract = {This work presents CLTune, an auto-tuner for OpenCL kernels. It evaluates and tunes kernel performance of a generic, user-defined search space of possible parameter-value combinations. Example parameters include the OpenCL workgroup size, vector data-types, tile sizes, and loop unrolling factors. CLTune can be used in the following scenarios: 1) when there are too many tunable parameters to explore manually, 2) when performance portability across OpenCL devices is desired, or 3) when the optimal parameters change based on input argument values (e.g. matrix dimensions). The auto-tuner is generic, easy to use, open-source, and supports multiple search strategies including simulated annealing and particle swarm optimisation. CLTune is evaluated on two GPU case-studies inspired by the recent successes in deep learning: 2D convolution and matrix-multiplication (GEMM). For 2D convolution, we demonstrate the need for auto-tuning by optimizing for different filter sizes, achieving performance on-par or better than the state-of-the-art. For matrix-multiplication, we use CLTune to explore a parameter space of more than two-hundred thousand configurations, we show the need for device-specific tuning, and outperform the clBLAS library on NVIDIA, AMD and Intel GPUs.},
	booktitle = {2015 {IEEE} 9th {International} {Symposium} on {Embedded} {Multicore}/{Many}-core {Systems}-on-{Chip}},
	author = {Nugteren, Cedric and Codreanu, Valeriu},
	month = sep,
	year = {2015},
	keywords = {Auto-Tuning, Convolution, GEMM, GPU, Kernel, OpenCL, Performance evaluation, Search problems, Simulated annealing, Tuners},
	pages = {195--202},
	file = {IEEE Xplore Abstract Record:/Users/agorji/Zotero/storage/8DCKWFP6/7328205.html:text/html;Submitted Version:/Users/agorji/Zotero/storage/Y47GXLY9/Nugteren and Codreanu - 2015 - CLTune A Generic Auto-Tuner for OpenCL Kernels.pdf:application/pdf},
}


@inproceedings{rajeswaran_towards_2017,
	title = {Towards {Generalization} and {Simplicity} in {Continuous} {Control}},
	volume = {30},
	abstract = {The remarkable successes of deep learning in speech recognition and computer vision have motivated efforts to adapt similar techniques to other problem domains, including reinforcement learning (RL). Consequently, RL methods have produced rich motor behaviors on simulated robot tasks, with their success largely attributed to the use of multi-layer neural networks. This work is among the first to carefully study what might be responsible for these recent advancements. Our main result calls this emerging narrative into question by showing that much simpler architectures -- based on linear and RBF parameterizations  -- achieve comparable performance to state of the art results.  We not only study different policy representations with regard to performance measures at hand, but also towards robustness to external perturbations. We again find that the learned neural network policies --- under the standard training scenarios ---  are no more robust than linear (or RBF) policies; in fact, all three are remarkably brittle. Finally, we then directly modify the training scenarios in order to favor more robust policies, and we again do not find a compelling case to favor multi-layer architectures.  Overall, this study suggests that multi-layer architectures should not be the default choice, unless a side-by-side comparison to simpler architectures shows otherwise. More generally, we hope that these results lead to more interest in carefully studying the architectural choices, and associated trade-offs, for training generalizable and robust policies.},
	urldate = {2023-01-14},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Rajeswaran, Aravind and Lowrey, Kendall and Todorov, Emanuel V. and Kakade, Sham M},
	year = {2017},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/WTSGEC5Z/Rajeswaran et al. - 2017 - Towards Generalization and Simplicity in Continuou.pdf:application/pdf},
}

@inproceedings{jacot_neural_2018,
	title = {Neural {Tangent} {Kernel}: {Convergence} and {Generalization} in {Neural} {Networks}},
	volume = {31},
	shorttitle = {Neural {Tangent} {Kernel}},
	abstract = {At initialization, artificial neural networks (ANNs) are equivalent to Gaussian processes in the infinite-width limit, thus connecting them to kernel methods. We prove that the evolution of an ANN during training can also be described by a kernel: during gradient descent on the parameters of an ANN, the network function (which maps input vectors to output vectors) follows the so-called kernel gradient associated with a new object, which we call the Neural Tangent Kernel (NTK). This kernel is central to describe the generalization features of ANNs. While the NTK is random at initialization and varies during training, in the infinite-width limit it converges to an explicit limiting kernel and stays constant during training. This makes it possible to study the training of ANNs in function space instead of parameter space. Convergence of the training can then be related to the positive-definiteness of the limiting NTK.},
	urldate = {2023-01-14},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Jacot, Arthur and Gabriel, Franck and Hongler, Clement},
	year = {2018},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/JLCG9VP8/Jacot et al. - 2018 - Neural Tangent Kernel Convergence and Generalizat.pdf:application/pdf},
}

@article{li2015active,
  title={An active learning framework using sparse-graph codes for sparse polynomials and graph sketching},
  author={Li, Xiao and Ramchandran, Kannan},
  journal={Advances in Neural Information Processing Systems},
  volume={28},
  year={2015}
}

@misc{fort_stiffness_2020,
	title = {Stiffness: {A} {New} {Perspective} on {Generalization} in {Neural} {Networks}},
	shorttitle = {Stiffness},
	doi = {10.48550/arXiv.1901.09491},
	abstract = {In this paper we develop a new perspective on generalization of neural networks by proposing and investigating the concept of a neural network stiffness. We measure how stiff a network is by looking at how a small gradient step in the network's parameters on one example affects the loss on another example. Higher stiffness suggests that a network is learning features that generalize. In particular, we study how stiffness depends on 1) class membership, 2) distance between data points in the input space, 3) training iteration, and 4) learning rate. We present experiments on MNIST, FASHION MNIST, and CIFAR-10/100 using fully-connected and convolutional neural networks, as well as on a transformer-based NLP model. We demonstrate the connection between stiffness and generalization, and observe its dependence on learning rate. When training on CIFAR-100, the stiffness matrix exhibits a coarse-grained behavior indicative of the model's awareness of super-class membership. In addition, we measure how stiffness between two data points depends on their mutual input-space distance, and establish the concept of a dynamical critical length -- a distance below which a parameter update based on a data point influences its neighbors.},
	urldate = {2023-01-14},
	publisher = {arXiv},
	author = {Fort, Stanislav and Nowak, Paweł Krzysztof and Jastrzebski, Stanislaw and Narayanan, Srini},
	month = mar,
	year = {2020},
	note = {arXiv:1901.09491 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Computer Science - Neural and Evolutionary Computing, Statistics - Machine Learning},
	annote = {Comment: Submitted for review},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/HFLHX2HQ/Fort et al. - 2020 - Stiffness A New Perspective on Generalization in .pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/T6NLKAY8/1901.html:text/html},
}

@misc{noauthor_180512076_nodate,
	title = {[1805.12076] {Towards} {Understanding} the {Role} of {Over}-{Parametrization} in {Generalization} of {Neural} {Networks}},
	urldate = {2023-01-14},
	file = {arXiv.org e-Print archive:/Users/agorji/Zotero/storage/6WIS6SNQ/1805.html:text/html},
}

@misc{yang_fine-grained_2020-1,
	title = {A {Fine}-{Grained} {Spectral} {Perspective} on {Neural} {Networks}},
	abstract = {Are neural networks biased toward simple functions? Does depth always help learn more complex features? Is training the last layer of a network as good as training all layers? How to set the range for learning rate tuning? These questions seem unrelated at face value, but in this work we give all of them a common treatment from the spectral perspective. We will study the spectra of the *Conjugate Kernel, CK,* (also called the *Neural Network-Gaussian Process Kernel*), and the *Neural Tangent Kernel, NTK*. Roughly, the CK and the NTK tell us respectively "what a network looks like at initialization" and "what a network looks like during and after training." Their spectra then encode valuable information about the initial distribution and the training and generalization properties of neural networks. By analyzing the eigenvalues, we lend novel insights into the questions put forth at the beginning, and we verify these insights by extensive experiments of neural networks. We derive fast algorithms for computing the spectra of CK and NTK when the data is uniformly distributed over the boolean cube, and show this spectra is the same in high dimensions when data is drawn from isotropic Gaussian or uniformly over the sphere. Code replicating our results is available at github.com/thegregyang/NNspectra.},
	urldate = {2023-01-14},
	publisher = {arXiv},
	author = {Yang, Greg and Salman, Hadi},
	month = apr,
	year = {2020},
	note = {arXiv:1907.10599 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Computer Science - Neural and Evolutionary Computing, Statistics - Machine Learning},
	annote = {Comment: 8 pages of main text, 19 figures, 51 pages including appendix},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/ZURLAMLH/Yang and Salman - 2020 - A Fine-Grained Spectral Perspective on Neural Netw.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/KQ8M68XN/1907.html:text/html},
}

@inproceedings{ronen_convergence_2019,
	title = {The {Convergence} {Rate} of {Neural} {Networks} for {Learned} {Functions} of {Different} {Frequencies}},
	volume = {32},
	abstract = {We study the relationship between the frequency of a function and the speed at which a neural network learns it.  We build on recent results that show that the dynamics of overparameterized neural networks trained with gradient descent can be well approximated by a linear system.  When normalized training data is uniformly distributed on a hypersphere, the eigenfunctions of this linear system are spherical harmonic functions.  We derive the corresponding eigenvalues for each frequency after introducing a bias term in the model.  This bias term had been omitted from the linear network model without significantly affecting previous theoretical results.  However, we show theoretically and experimentally that a shallow neural network without bias cannot represent or learn simple, low frequency functions with odd frequencies.  Our results lead to specific predictions of the time it will take a network to learn functions of varying frequency.  These predictions match the empirical behavior of both shallow and deep networks.},
	urldate = {2023-01-14},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Ronen, Basri and Jacobs, David and Kasten, Yoni and Kritchman, Shira},
	year = {2019},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/PZ5Y26W6/Ronen et al. - 2019 - The Convergence Rate of Neural Networks for Learne.pdf:application/pdf},
}

@article{trendafilov_simple_2014,
	title = {From simple structure to sparse components: a review},
	volume = {29},
	issn = {1613-9658},
	shorttitle = {From simple structure to sparse components},
	doi = {10.1007/s00180-013-0434-5},
	abstract = {The article begins with a review of the main approaches for interpretation the results from principal component analysis (PCA) during the last 50–60 years. The simple structure approach is compared to the modern approach of sparse PCA where interpretable solutions are directly obtained. It is shown that their goals are identical but they differ by the way they are realized. Next, the most popular and influential methods for sparse PCA are briefly reviewed. In the remaining part of the paper, a new approach to define sparse PCA is introduced. Several alternative definitions are considered and illustrated on a well-known data set. Finally, it is demonstrated, how one of these possible versions of sparse PCA can be used as a sparse alternative to the classical rotation methods.},
	language = {en},
	number = {3},
	urldate = {2023-01-15},
	journal = {Computational Statistics},
	author = {Trendafilov, Nickolay T.},
	month = jun,
	year = {2014},
	keywords = {Constrained optimization on matrix manifolds, Factor analysis, LASSO, Orthogonal and oblique rotations, Projected gradients, Simple structure loadings, Sparse component loadings, Sparseness inducing constraints},
	pages = {431--454},
}

@misc{noauthor_simple_nodate,
	title = {From simple structure to sparse components: a review {\textbar} {SpringerLink}},
	urldate = {2023-01-15},
	file = {From simple structure to sparse components\: a review | SpringerLink:/Users/agorji/Zotero/storage/ZP62DM6Q/s00180-013-0434-5.html:text/html},
}

@article{trendafilov_simple_2014-1,
	title = {From simple structure to sparse components: a review},
	volume = {29},
	issn = {0943-4062, 1613-9658},
	shorttitle = {From simple structure to sparse components},
	doi = {10.1007/s00180-013-0434-5},
	abstract = {The article begins with a review of the main approaches for interpretation the results from principal component analysis (PCA) during the last 50–60 years. The simple structure approach is compared to the modern approach of sparse PCA where interpretable solutions are directly obtained. It is shown that their goals are identical but they differ by the way they are realized. Next, the most popular and inﬂuential methods for sparse PCA are brieﬂy reviewed. In the remaining part of the paper, a new approach to deﬁne sparse PCA is introduced. Several alternative deﬁnitions are considered and illustrated on a well-known data set. Finally, it is demonstrated, how one of these possible versions of sparse PCA can be used as a sparse alternative to the classical rotation methods.},
	language = {en},
	number = {3-4},
	urldate = {2023-01-15},
	journal = {Computational Statistics},
	author = {Trendafilov, Nickolay T.},
	month = jun,
	year = {2014},
	pages = {431--454},
	file = {Trendafilov - 2014 - From simple structure to sparse components a revi.pdf:/Users/agorji/Zotero/storage/RH6SQG7L/Trendafilov - 2014 - From simple structure to sparse components a revi.pdf:application/pdf},
}

@article{xu_frequency_2020,
	title = {Frequency {Principle}: {Fourier} {Analysis} {Sheds} {Light} on {Deep} {Neural} {Networks}},
	volume = {28},
	issn = {1815-2406, 1991-7120},
	shorttitle = {Frequency {Principle}},
	doi = {10.4208/cicp.OA-2020-0085},
	abstract = {We study the training process of Deep Neural Networks (DNNs) from the Fourier analysis perspective. We demonstrate a very universal Frequency Principle (F-Principle) --- DNNs often fit target functions from low to high frequencies --- on high-dimensional benchmark datasets such as MNIST/CIFAR10 and deep neural networks such as VGG16. This F-Principle of DNNs is opposite to the behavior of most conventional iterative numerical schemes (e.g., Jacobi method), which exhibit faster convergence for higher frequencies for various scientific computing problems. With a simple theory, we illustrate that this F-Principle results from the regularity of the commonly used activation functions. The F-Principle implies an implicit bias that DNNs tend to fit training data by a low-frequency function. This understanding provides an explanation of good generalization of DNNs on most real datasets and bad generalization of DNNs on parity function or randomized dataset.},
	number = {5},
	urldate = {2023-01-15},
	journal = {Communications in Computational Physics},
	author = {Xu, Zhi-Qin John and Zhang, Yaoyu and Luo, Tao and Xiao, Yanyang and Ma, Zheng},
	month = jun,
	year = {2020},
	note = {arXiv:1901.06523 [cs, stat]},
	keywords = {68Q32, 68T01, Computer Science - Machine Learning, I.2.6, Statistics - Machine Learning},
	pages = {1746--1767},
	annote = {Comment: 8 pages, 5 figures, under review},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/MFLCYNZF/Xu et al. - 2020 - Frequency Principle Fourier Analysis Sheds Light .pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/24AAFPKG/1901.html:text/html},
}

@misc{xu_understanding_2018,
	title = {Understanding training and generalization in deep learning by {Fourier} analysis},
	abstract = {Background: It is still an open research area to theoretically understand why Deep Neural Networks (DNNs)---equipped with many more parameters than training data and trained by (stochastic) gradient-based methods---often achieve remarkably low generalization error. Contribution: We study DNN training by Fourier analysis. Our theoretical framework explains: i) DNN with (stochastic) gradient-based methods often endows low-frequency components of the target function with a higher priority during the training; ii) Small initialization leads to good generalization ability of DNN while preserving the DNN's ability to fit any function. These results are further confirmed by experiments of DNNs fitting the following datasets, that is, natural images, one-dimensional functions and MNIST dataset.},
	urldate = {2023-01-15},
	publisher = {arXiv},
	author = {Xu, Zhiqin John},
	month = nov,
	year = {2018},
	note = {arXiv:1808.04295 [cs, math, stat]},
	keywords = {68Q32, 68T01, Computer Science - Artificial Intelligence, Computer Science - Machine Learning, I.2.6, Mathematics - Optimization and Control, Mathematics - Statistics Theory, Statistics - Machine Learning},
	annote = {Comment: 10 pages, 4 figures},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/7KSAZIRT/Xu - 2018 - Understanding training and generalization in deep .pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/9TURR28D/1808.html:text/html},
}

@inproceedings{xu_training_2019,
	address = {Berlin, Heidelberg},
	title = {Training {Behavior} of {Deep} {Neural} {Network} in {Frequency} {Domain}},
	isbn = {978-3-030-36707-7},
	doi = {10.1007/978-3-030-36708-4_22},
	abstract = {Why deep neural networks (DNNs) capable of overfitting often generalize well in practice is a mystery [24]. To find a potential mechanism, we focus on the study of implicit biases underlying the training process of DNNs. In this work, for both real and synthetic datasets, we empirically find that a DNN with common settings first quickly captures the dominant low-frequency components, and then relatively slowly captures the high-frequency ones. We call this phenomenon Frequency Principle (F-Principle). The F-Principle can be observed over DNNs of various structures, activation functions, and training algorithms in our experiments. We also illustrate how the F-Principle helps understand the effect of early-stopping as well as the generalization of DNNs. This F-Principle potentially provides insight into a general principle underlying DNN optimization and generalization.},
	urldate = {2023-01-15},
	booktitle = {Neural {Information} {Processing}: 26th {International} {Conference}, {ICONIP} 2019, {Sydney}, {NSW}, {Australia}, {December} 12–15, 2019, {Proceedings}, {Part} {I}},
	publisher = {Springer-Verlag},
	author = {Xu, Zhi-Qin John and Zhang, Yaoyu and Xiao, Yanyang},
	month = dec,
	year = {2019},
	keywords = {Deep learning, Deep Neural Network, Fourier analysis, Generalization},
	pages = {264--274},
	file = {Submitted Version:/Users/agorji/Zotero/storage/HRGHDQXX/Xu et al. - 2019 - Training Behavior of Deep Neural Network in Freque.pdf:application/pdf},
}

@inproceedings{shah_pitfalls_2020-1,
	title = {The {Pitfalls} of {Simplicity} {Bias} in {Neural} {Networks}},
	volume = {33},
	abstract = {Several works have proposed Simplicity Bias (SB)---the tendency of standard training procedures such as Stochastic Gradient Descent (SGD) to find simple models---to justify why neural networks generalize well [Arpit et al. 2017, Nakkiran et al. 2019, Valle-Perez et al. 2019]. However, the precise notion of simplicity remains vague. Furthermore, previous settings [Soudry et al. 2018, Gunasekar et al. 2018] that use SB to theoretically justify why neural networks generalize well do not simultaneously capture the non-robustness of neural networks---a widely observed phenomenon in practice [Goodfellow et al. 2014, Jo and Bengio 2017]. We attempt to reconcile SB and the superior standard generalization of neural networks with the non-robustness observed in practice by introducing piecewise-linear and image-based datasets, which (a) incorporate a precise notion of simplicity, (b) comprise multiple predictive features with varying levels of simplicity, and (c) capture the non-robustness of neural networks trained on real data. Using theory and empirics on these datasets, we make four observations:
(i) SB of SGD and variants can be extreme: neural networks can exclusively rely on the simplest feature and remain invariant to all predictive complex features.
(ii) The extreme aspect of SB could explain why seemingly benign distribution shifts and small adversarial perturbations significantly degrade model performance.
(iii) Contrary to conventional wisdom, SB can also hurt generalization on the same data distribution, as SB persists even when the simplest feature has less predictive power than the more complex features.
(iv) Common approaches to improve generalization and robustness---ensembles and adversarial training---can fail in mitigating SB and its pitfalls.
Given the role of SB in training neural networks, we hope that the proposed datasets and methods serve as an effective testbed to evaluate novel algorithmic approaches aimed at avoiding the pitfalls of SB.},
	urldate = {2023-01-15},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Shah, Harshay and Tamuly, Kaustav and Raghunathan, Aditi and Jain, Prateek and Netrapalli, Praneeth},
	year = {2020},
	pages = {9573--9585},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/P6AA5345/Shah et al. - 2020 - The Pitfalls of Simplicity Bias in Neural Networks.pdf:application/pdf},
}

@misc{sun_sparse-softmax_2021,
	title = {Sparse-softmax: {A} {Simpler} and {Faster} {Alternative} {Softmax} {Transformation}},
	shorttitle = {Sparse-softmax},
	doi = {10.48550/arXiv.2112.12433},
	abstract = {The softmax function is widely used in artificial neural networks for the multiclass classification problems, where the softmax transformation enforces the output to be positive and sum to one, and the corresponding loss function allows to use maximum likelihood principle to optimize the model. However, softmax leaves a large margin for loss function to conduct optimizing operation when it comes to high-dimensional classification, which results in low-performance to some extent. In this paper, we provide an empirical study on a simple and concise softmax variant, namely sparse-softmax, to alleviate the problem that occurred in traditional softmax in terms of high-dimensional classification problems. We evaluate our approach in several interdisciplinary tasks, the experimental results show that sparse-softmax is simpler, faster, and produces better results than the baseline models.},
	urldate = {2023-01-15},
	publisher = {arXiv},
	author = {Sun, Shaoshi and Zhang, Zhenyuan and Huang, BoCheng and Lei, Pengbin and Su, Jianlin and Pan, Shengfeng and Cao, Jiarun},
	month = dec,
	year = {2021},
	note = {arXiv:2112.12433 [cs]},
	keywords = {Computer Science - Computation and Language, Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/KAGQVEBU/Sun et al. - 2021 - Sparse-softmax A Simpler and Faster Alternative S.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/79A94A2M/2112.html:text/html},
}

@article{durvasula_there_2020,
	title = {There is a simplicity bias when generalising from ambiguous data},
	volume = {37},
	issn = {0952-6757, 1469-8188},
	doi = {10.1017/S0952675720000093},
	abstract = {How exactly do learners generalise in the face of ambiguous data? While there has been a substantial amount of research studying the biases that learners employ, there has been very little work on what sorts of biases are employed in the face of data that is ambiguous between phonological generalisations with different degrees of complexity. In this article, we present the results from three artificial language learning experiments that suggest that, at least for phonotactic sequence patterns, learners are able to keep track of multiple generalisations related to the same segmental co-occurrences; however, the generalisations they learn are only the simplest ones consistent with the data.},
	language = {en},
	number = {2},
	urldate = {2023-01-15},
	journal = {Phonology},
	author = {Durvasula, Karthik and Liter, Adam},
	month = may,
	year = {2020},
	note = {Publisher: Cambridge University Press},
	pages = {177--213},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/BK4Y25UR/Durvasula and Liter - 2020 - There is a simplicity bias when generalising from .pdf:application/pdf},
}

@inproceedings{arpit_closer_2017,
	title = {A {Closer} {Look} at {Memorization} in {Deep} {Networks}},
	abstract = {We examine the role of memorization in deep learning, drawing connections to capacity, generalization, and adversarial robustness. While deep networks are capable of memorizing noise data, our results suggest that they tend to prioritize learning simple patterns first. In our experiments, we expose qualitative differences in gradient-based optimization of deep neural networks (DNNs) on noise vs.{\textasciitilde}real data. We also demonstrate that for appropriately tuned explicit regularization (e.g.,{\textasciitilde}dropout) we can degrade DNN training performance on noise datasets without compromising generalization on real data. Our analysis suggests that the notions of effective capacity which are dataset independent are unlikely to explain the generalization performance of deep networks when trained with gradient based methods because training data itself plays an important role in determining the degree of memorization.},
	language = {en},
	urldate = {2023-01-15},
	booktitle = {Proceedings of the 34th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Arpit, Devansh and Jastrzębski, Stanisław and Ballas, Nicolas and Krueger, David and Bengio, Emmanuel and Kanwal, Maxinder S. and Maharaj, Tegan and Fischer, Asja and Courville, Aaron and Bengio, Yoshua and Lacoste-Julien, Simon},
	month = jul,
	year = {2017},
	note = {ISSN: 2640-3498},
	pages = {233--242},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/SEUE5TP5/Arpit et al. - 2017 - A Closer Look at Memorization in Deep Networks.pdf:application/pdf},
}

@inproceedings{kalimeris_sgd_2019,
	title = {{SGD} on {Neural} {Networks} {Learns} {Functions} of {Increasing} {Complexity}},
	volume = {32},
	abstract = {We perform an experimental study of the dynamics of Stochastic Gradient Descent (SGD) in learning deep neural networks for several real and synthetic classification tasks.
We show that in the initial epochs, almost all of the performance improvement of the classifier obtained by SGD can be explained by a linear classifier.
More generally, we give evidence for the hypothesis that, as iterations progress,  SGD learns functions of increasing complexity. This hypothesis can be helpful in explaining why SGD-learned classifiers tend to generalize well even in the over-parameterized regime.
We also show that the linear classifier learned in the initial stages is ``retained'' throughout the execution even if training is continued to the point of zero training error, and complement this with a theoretical result in a simplified model.
Key to our work is a new measure of
how well one classifier explains the performance of another, based on conditional mutual information.},
	urldate = {2023-01-15},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Kalimeris, Dimitris and Kaplun, Gal and Nakkiran, Preetum and Edelman, Benjamin and Yang, Tristan and Barak, Boaz and Zhang, Haofeng},
	year = {2019},
	file = {Full Text PDF:/Users/agorji/Zotero/storage/9BHEQI5U/Kalimeris et al. - 2019 - SGD on Neural Networks Learns Functions of Increas.pdf:application/pdf},
}

@misc{huh_low-rank_2022,
	title = {The {Low}-{Rank} {Simplicity} {Bias} in {Deep} {Networks}},
	abstract = {Modern deep neural networks are highly over-parameterized compared to the data on which they are trained, yet they often generalize remarkably well. A flurry of recent work has asked: why do deep networks not overfit to their training data? In this work, we make a series of empirical observations that investigate and extend the hypothesis that deeper networks are inductively biased to find solutions with lower effective rank embeddings. We conjecture that this bias exists because the volume of functions that maps to low effective rank embedding increases with depth. We show empirically that our claim holds true on finite width linear and non-linear models on practical learning paradigms and show that on natural data, these are often the solutions that generalize well. We then show that the simplicity bias exists at both initialization and after training and is resilient to hyper-parameters and learning methods. We further demonstrate how linear over-parameterization of deep non-linear models can be used to induce low-rank bias, improving generalization performance on CIFAR and ImageNet without changing the modeling capacity.},
	urldate = {2023-01-15},
	publisher = {arXiv},
	author = {Huh, Minyoung and Mobahi, Hossein and Zhang, Richard and Cheung, Brian and Agrawal, Pulkit and Isola, Phillip},
	month = apr,
	year = {2022},
	note = {arXiv:2103.10427 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/agorji/Zotero/storage/KRBQZ27H/Huh et al. - 2022 - The Low-Rank Simplicity Bias in Deep Networks.pdf:application/pdf;arXiv.org Snapshot:/Users/agorji/Zotero/storage/V6633HVY/2103.html:text/html},
}













@article{cybenko1989approximation,
  title={Approximation by superpositions of a sigmoidal function},
  author={Cybenko, George},
  journal={Mathematics of control, signals and systems},
  volume={2},
  number={4},
  pages={303--314},
  year={1989},
  publisher={Springer}
}
@article{hornik1989multilayer,
  title={Multilayer feedforward networks are universal approximators},
  author={Hornik, Kurt and Stinchcombe, Maxwell and White, Halbert},
  journal={Neural networks},
  volume={2},
  number={5},
  pages={359--366},
  year={1989},
  publisher={Elsevier}
}
@article{domingos2020every,
  title={Every model learned by gradient descent is approximately a kernel machine},
  author={Domingos, Pedro},
  journal={arXiv preprint arXiv:2012.00152},
  year={2020}
}



@inproceedings{nakkiran2019sgd,
  title={SGD on neural networks learns functions of increasing complexity},
  author={Nakkiran, Preetum and Kaplun, Gal and Kalimeris, Dimitris and Yang, Tristan and Edelman, Benjamin L and Zhang, Fred and Barak, Boaz},
  booktitle={Proceedings of the 33rd International Conference on Neural Information Processing Systems},
  pages={3496--3506},
  year={2019}
}

@inproceedings{
valle2018deep,
title={Deep learning generalizes because the parameter-function map is biased towards simple functions},
author={Guillermo Valle-Perez and Chico Q. Camargo and Ard A. Louis},
booktitle={International Conference on Learning Representations},
year={2019},
}
@article{mangalam2019deep,
  title={Do deep neural networks learn shallow learnable examples first?},
  author={Mangalam, Karttikeya and Prabhu, Vinay Uday},
  year={2019}
}
@inproceedings{lee2018deep,
  title={Deep Neural Networks as Gaussian Processes},
  author={Lee, Jaehoon and Bahri, Yasaman and Novak, Roman and Schoenholz, Samuel S and Pennington, Jeffrey and Sohl-Dickstein, Jascha},
  booktitle={International Conference on Learning Representations},
  year={2018}
}

@article{daniely2017sgd,
  title={SGD learns the conjugate kernel class of the network},
  author={Daniely, Amit},
  journal={Advances in Neural Information Processing Systems},
  volume={30},
  year={2017}
}

@article{lee2019wide,
  title={Wide neural networks of any depth evolve as linear models under gradient descent},
  author={Lee, Jaehoon and Xiao, Lechao and Schoenholz, Samuel and Bahri, Yasaman and Novak, Roman and Sohl-Dickstein, Jascha and Pennington, Jeffrey},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}

@article{daniely2016toward,
  title={Toward deeper understanding of neural networks: The power of initialization and a dual view on expressivity},
  author={Daniely, Amit and Frostig, Roy and Singer, Yoram},
  journal={Advances in neural information processing systems},
  volume={29},
  year={2016}
}

@inproceedings{basri2020frequency,
  title={Frequency bias in neural networks for input of non-uniform density},
  author={Basri, Ronen and Galun, Meirav and Geifman, Amnon and Jacobs, David and Kasten, Yoni and Kritchman, Shira},
  booktitle={International Conference on Machine Learning},
  pages={685--694},
  year={2020},
  organization={PMLR}
}


@inproceedings{rasmussen2004gaussian,
  title={Gaussian processes in machine learning},
  author={Rasmussen, Carl Edward},
  booktitle={Summer school on machine learning},
  pages={63--71},
  year={2004},
  organization={Springer}
}

@article{krogh1991simple,
  title={A simple weight decay can improve generalization},
  author={Krogh, Anders and Hertz, John},
  journal={Advances in neural information processing systems},
  volume={4},
  year={1991}
}
@article{srivastava2014dropout,
  title={Dropout: a simple way to prevent neural networks from overfitting},
  author={Srivastava, Nitish and Hinton, Geoffrey and Krizhevsky, Alex and Sutskever, Ilya and Salakhutdinov, Ruslan},
  journal={The journal of machine learning research},
  volume={15},
  number={1},
  pages={1929--1958},
  year={2014},
  publisher={JMLR. org}
}
@inproceedings{ioffe2015batch,
  title={Batch normalization: Accelerating deep network training by reducing internal covariate shift},
  author={Ioffe, Sergey and Szegedy, Christian},
  booktitle={International conference on machine learning},
  pages={448--456},
  year={2015},
  organization={PMLR}
}

@book{o2014analysis,
  title={Analysis of boolean functions},
  author={O'Donnell, Ryan},
  year={2014},
  publisher={Cambridge University Press}
}

@article{chizat2019lazy,
  title={On lazy training in differentiable programming},
  author={Chizat, Lenaic and Oyallon, Edouard and Bach, Francis},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}

@inproceedings{
du2018gradient,
title={Gradient Descent Provably Optimizes Over-parameterized Neural Networks},
author={Simon S. Du and Xiyu Zhai and Barnabas Poczos and Aarti Singh},
booktitle={International Conference on Learning Representations},
year={2019},
}
@inproceedings{allen2019convergence,
  title={A convergence theory for deep learning via over-parameterization},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
  booktitle={International Conference on Machine Learning},
  pages={242--252},
  year={2019},
  organization={PMLR}
}
@article{allen2019convergencernn,
  title={On the convergence rate of training recurrent neural networks},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}
@article{tancik2020fourier,
  title={Fourier features let networks learn high frequency functions in low dimensional domains},
  author={Tancik, Matthew and Srinivasan, Pratul and Mildenhall, Ben and Fridovich-Keil, Sara and Raghavan, Nithin and Singhal, Utkarsh and Ramamoorthi, Ravi and Barron, Jonathan and Ng, Ren},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={7537--7547},
  year={2020}
}

@inproceedings{kushilevitz1991learning,
  title={Learning decision trees using the Fourier spectrum},
  author={Kushilevitz, Eyal and Mansour, Yishay},
  booktitle={Proceedings of the twenty-third annual ACM symposium on Theory of computing},
  pages={455--464},
  year={1991}
}

@article{alon1999linear,
  title={Linear hash functions},
  author={Alon, Noga and Dietzfelbinger, Martin and Miltersen, Peter Bro and Petrank, Erez and Tardos, G{\'a}bor},
  journal={Journal of the ACM (JACM)},
  volume={46},
  number={5},
  pages={667--683},
  year={1999},
  publisher={ACM New York, NY, USA}
}

@inproceedings{tabnet,
  title={Tabnet: Attentive interpretable tabular learning},
  author={Arik, Sercan {\"O} and Pfister, Tomas},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  pages={6679--6687},
  year={2021}
}

@article{carter1979universal,
  title={Universal classes of hash functions},
  author={Carter, J Lawrence and Wegman, Mark N},
  journal={Journal of computer and system sciences},
  volume={18},
  number={2},
  pages={143--154},
  year={1979},
  publisher={Elsevier}
}