# encoding:utf-8
@inproceedings{touati2018convergent,
  title={Convergent TREE BACKUP and RETRACE with function approximation},
  author={Touati, Ahmed and Bacon, Pierre-Luc and Precup, Doina and Vincent, Pascal},
  booktitle={Proceedings of the 35th International Conference on Machine Learning},
  pages={4955--4964},
  year={2018}
}

@inproceedings{hallak2016generalized,
  title={Generalized emphatic temporal difference learning: bias-variance analysis},
  author={Hallak, Assaf and Tamar, Aviv and Munos, Remi and Mannor, Shie},
  booktitle={Proceedings of the 30th AAAI Conference on Artificial Intelligence},
  pages={1631--1637},
  year={2016}
}
@inproceedings{harutyunyan2016q,
  title={Q($\lambda$) with Off-Policy Corrections},
  author={Harutyunyan, Anna and Bellemare, Marc G and Stepleton, Tom and Munos, R{\'e}mi},
  booktitle={International Conference on Algorithmic Learning Theory},
  pages={305--320},
  year={2016},
  organization={Springer}
}

@article{graves2022importance,
  title={Importance Sampling Placement in Off-Policy Temporal-Difference Methods},
  author={Graves, Eric and Ghiassian, Sina},
  journal={arXiv preprint arXiv:2203.10172},
  year={2022}
}
@InProceedings{csimcsek2016most,
  title = 	 {Why Most Decisions Are Easy in Tetris---And Perhaps in Other Sequential Decision Problems, As Well},
  author = 	 {Simsek, Ozgur and Algorta, Simon and Kothiyal, Amit},
  booktitle = 	 {Proceedings of The 33rd International Conference on Machine Learning},
  pages = 	 {1757--1765},
  year = 	 {2016},
  editor = 	 {Balcan, Maria Florina and Weinberger, Kilian Q.},
  volume = 	 {48},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {New York, New York, USA},
  month = 	 {20--22 Jun},
  publisher =    {PMLR}
  }
@article{glanois2021survey,
  title={A Survey on Interpretable Reinforcement Learning},
  author={Glanois, Claire and Weng, Paul and Zimmer, Matthieu and Li, Dong and Yang, Tianpei and Hao, Jianye and Liu, Wulong},
  journal={arXiv preprint arXiv:2112.13112},
  year={2021}
}
@inproceedings{garg2020symbolic,
  title={Symbolic network: generalized neural policies for relational MDPs},
  author={Garg, Sankalp and Bajpai, Aniket and others},
  booktitle={International Conference on Machine Learning},
  pages={3397--3407},
  year={2020},
  organization={PMLR}
}
@inproceedings{yang2021dhqn,
  title={{DHQN:} a Stable Approach to Remove Target Network from Deep Q-learning Network},
  author={Yang, Guang and Li, Yang and Huang, Tian and Li, Qingyun and Chen, Xingguo},
  booktitle={2021 IEEE 33rd International Conference on Tools with Artificial Intelligence (ICTAI)},
  pages={1474--1479},
  year={2021},
  organization={IEEE}
}
@article{chen2023fixed,
  title={A survey of reinforcement learning algorithms from a fixed point perspective},
  author={Xingguo Chen and  Dingyuanhao Sun  and Guang Yang and Shangdong Yang and Yang Gao},
  journal={Chinese Journal of Computers},
  volume={46},
  number={6},
  pages={1246--1271},
  year={2023}
}
@article{chen2016reinforcement,
  title={Reinforcement learning and its application to the game of Go},
  author={Chen, Xingguo and Yu, Yang},
  journal={Acta Automatica Sinica},
  volume={42},
  number={5},
  pages={685--695},
  year={2016}
}
@article{sanner2009practical,
  title={Practical solution techniques for first-order MDPs},
  author={Sanner, Scott and Boutilier, Craig},
  journal={Artificial Intelligence},
  volume={173},
  number={5-6},
  pages={748--788},
  year={2009},
  publisher={Elsevier}
}
@inproceedings{grand2021scalable,
  title={Scalable First-Order Methods for Robust MDPs},
  author={Grand-Cl{\'e}ment, Julien and Kroer, Christian},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={35},
  number={13},
  pages={12086--12094},
  year={2021}
}
@article{driessens2006graph,
  title={Graph kernels and Gaussian processes for relational reinforcement learning},
  author={Driessens, Kurt and Ramon, Jan and G{\"a}rtner, Thomas},
  journal={Machine Learning},
  volume={64},
  number={1},
  pages={91--119},
  year={2006},
  publisher={Springer}
}
@inproceedings{driessens2003relational,
  title={Relational instance based regression for relational reinforcement learning},
  author={Driessens, Kurt and Ramon, Jan},
  booktitle={Proceedings of the 20th International Conference on Machine Learning (ICML-03)},
  pages={123--130},
  year={2003}
}
@article{driessens2004integrating,
  title={Integrating guidance into relational reinforcement learning},
  author={Driessens, Kurt and D{\v{z}}eroski, Sa{\v{s}}o},
  journal={Machine Learning},
  volume={57},
  number={3},
  pages={271--304},
  year={2004},
  publisher={Springer}
}
@article{dvzeroski2001relational,
  title={Relational reinforcement learning},
  author={D{\v{z}}eroski, Sa{\v{s}}o and De Raedt, Luc and Driessens, Kurt},
  journal={Machine learning},
  volume={43},
  number={1},
  pages={7--52},
  year={2001},
  publisher={Springer}
}
@article{liu2021learning,
  title={Learning Tree Interpretation from Object Representation for Deep Reinforcement Learning},
  author={Liu, Guiliang and Sun, Xiangyu and Schulte, Oliver and Poupart, Pascal},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}
@article{barreto2016practical,
  title={Practical kernel-based reinforcement learning},
  author={Barreto, Andr{\'e} MS and Precup, Doina and Pineau, Joelle},
  journal={The Journal of Machine Learning Research},
  volume={17},
  number={1},
  pages={2372--2441},
  year={2016},
  publisher={JMLR. org}
}
@inproceedings{kroemer2011non,
  title={A non-parametric approach to dynamic programming},
  author={Kroemer, Oliver B and Peters, Jan R},
  booktitle={Advances in neural information processing systems},
  pages={1719--1727},
  year={2011}
}
@inproceedings{amit2020discount,
  title={Discount factor as a regularizer in reinforcement learning},
  author={Amit, Ron and Meir, Ron and Ciosek, Kamil},
  booktitle={International conference on machine learning},
  pages={269--278},
  year={2020},
  organization={PMLR}
}
@inproceedings{nguyen2013online,
  title={Online feature selection for model-based reinforcement learning},
  author={Nguyen, Trung and Li, Zhuoru and Silander, Tomi and Leong, Tze Yun},
  booktitle={International Conference on Machine Learning},
  pages={498--506},
  year={2013},
  organization={PMLR}
}
@article{song2021online,
  title={Online Sparse Temporal Difference Learning Based on Nested Optimization and Regularized Dual Averaging},
  author={Song, Tianheng and Li, Dazi and Xu, Xin},
  journal={IEEE Transactions on Systems, Man, and Cybernetics: Systems},
  year={2021},
  publisher={IEEE}
}
@article{koppel2020policy,
  title={Policy evaluation in continuous MDPs with efficient kernelized gradient temporal difference},
  author={Koppel, Alec and Warnell, Garrett and Stump, Ethan and Stone, Peter and Ribeiro, Alejandro},
  journal={IEEE Transactions on Automatic Control},
  volume={66},
  number={4},
  pages={1856--1863},
  year={2020},
  publisher={IEEE}
}
@inproceedings{hao2021sparse,
  title={Sparse feature selection makes batch reinforcement learning more sample efficient},
  author={Hao, Botao and Duan, Yaqi and Lattimore, Tor and Szepesv{\'a}ri, Csaba and Wang, Mengdi},
  booktitle={International Conference on Machine Learning},
  pages={4063--4073},
  year={2021},
  organization={PMLR}
}
@inproceedings{feng2020accountable,
  title={Accountable off-policy evaluation with kernel bellman statistics},
  author={Feng, Yihao and Ren, Tongzheng and Tang, Ziyang and Liu, Qiang},
  booktitle={International Conference on Machine Learning},
  pages={3102--3111},
  year={2020},
  organization={PMLR}
}
@inproceedings{jin2021pessimism,
  title={Is pessimism provably efficient for offline rl?},
  author={Jin, Ying and Yang, Zhuoran and Wang, Zhaoran},
  booktitle={International Conference on Machine Learning},
  pages={5084--5096},
  year={2021},
  organization={PMLR}
}
@inproceedings{li2020orthogonal,
  title={Orthogonal Matching Pursuit for Least Squares Temporal Difference with Gradient Correction},
  author={Li, Dazi and Ma, Chao and Zhang, Jianqing and Ma, Xin and Jin, Qibing},
  booktitle={2020 Chinese Automation Congress (CAC)},
  pages={4108--4112},
  year={2020},
  organization={IEEE}
}
@article{song2018sparse,
  title={Sparse proximal reinforcement learning via nested optimization},
  author={Song, Tianheng and Li, Dazi and Jin, Qibing and Hirasawa, Kotaro},
  journal={IEEE Transactions on Systems, Man, and Cybernetics: Systems},
  volume={50},
  number={11},
  pages={4020--4032},
  year={2018},
  publisher={IEEE}
}
@inproceedings{li2016regularization,
  title={Regularization and feature selection in least squares temporal difference with gradient correction},
  author={Li, Dazi and Li, Luntong and Song, Tianheng and Jin, Qibing},
  booktitle={2016 12th World Congress on Intelligent Control and Automation (WCICA)},
  pages={2289--2293},
  year={2016},
  organization={IEEE}
}
@article{farahmand2016regularized,
  title={Regularized policy iteration with nonparametric function spaces},
  author={Farahmand, Amir-massoud and Ghavamzadeh, Mohammad and Szepesv{\'a}ri, Csaba and Mannor, Shie},
  journal={The Journal of Machine Learning Research},
  volume={17},
  number={1},
  pages={4809--4874},
  year={2016},
  publisher={JMLR. org}
}
@article{van2018deep,
  title={Deep reinforcement learning and the deadly triad},
  author={Van Hasselt, Hado and Doron, Yotam and Strub, Florian and Hessel, Matteo and 
Sonnerat, Nicolas and Modayil, Joseph},
  journal={arXiv preprint arXiv:1812.02648},
  year={2018}
}

@article{liu2019off,
  title={Off-policy policy gradient with state distribution correction},
  author={Liu, Yao and Swaminathan, Adith and Agarwal, Alekh and Brunskill, Emma},
  journal={arXiv preprint arXiv:1904.08473},
  year={2019}
}

@article{liu2015generalized,
  title={Generalized policy iteration adaptive dynamic programming for discrete-time nonlinear 
systems},
  author={Liu, Derong and Wei, Qinglai and Yan, Pengfei},
  journal={IEEE Transactions on Systems, Man, and Cybernetics: Systems},
  volume={45},
  number={12},
  pages={1577--1591},
  year={2015},
  publisher={IEEE}
}

@inproceedings{gelada2019off,
  title={Off-policy deep reinforcement learning by bootstrapping the covariate shift},
  author={Gelada, Carles and Bellemare, Marc G},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={33},
  number={01},
  pages={3647--3655},
  year={2019}
}

@article{gupta2019finite,
  title={Finite-time performance bounds and adaptive learning rate selection for two time-scale 
reinforcement learning},
  author={Gupta, Harsh and Srikant, R and Ying, Lei},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  pages={4704--4713},
  year={2019}
}

@inproceedings{fairbank2012divergence,
  title={The divergence of reinforcement learning algorithms with value-iteration and function 
approximation},
  author={Fairbank, Michael and Alonso, Eduardo},
  booktitle={The 2012 International Joint Conference on Neural Networks (IJCNN)},
  pages={1--8},
  year={2012},
  organization={IEEE}
}

@article{feng2019kernel,
  title={A Kernel Loss for Solving the Bellman Equation},
  author={Feng, Yihao and Li, Lihong and Liu, Qiang},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  pages={15456--15467},
  year={2019}
}

@article{kumar2020discor,
  title={DisCor: Corrective Feedback in Reinforcement Learning via Distribution Correction},
  author={Kumar, Aviral and Gupta, Abhishek and Levine, Sergey},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages = {18560--18572},
  year={2020}
}

@inproceedings{lee2014convergent,
  title={Convergent reinforcement learning control with neural networks and continuous action 
search},
  author={Lee, Minwoo and Anderson, Charles W},
  booktitle={2014 IEEE Symposium on Adaptive Dynamic Programming and Reinforcement Learning 
(ADPRL)},
  pages={1--8},
  year={2014},
  organization={IEEE}
}

@inproceedings{brandfonbrener2020geometric,
  title={Geometric Insights into the Convergence of Non-linear TD Learning},
  author={Brandfonbrener, David and Bruna, Joan},
  booktitle={International Conference on Learning Representations},
  year={2020}
}

@inproceedings{yang2016task,
  title={Task-specific pre-learning to improve the convergence of reinforcement learning based 
on a deep neural network},
  author={Yang, Yuan and Li, Xiaoan and Zhang, Lu},
  booktitle={2016 12th World Congress on Intelligent Control and Automation (WCICA)},
  pages={2209--2214},
  year={2016},
  organization={IEEE}
}

@inproceedings{ghosh2020representations,
  title={Representations for stable off-policy reinforcement learning},
  author={Ghosh, Dibya and Bellemare, Marc G},
  booktitle={International Conference on Machine Learning},
  pages={3556--3565},
  year={2020},
  organization={PMLR}
}

@inproceedings{doan2021nonlinear,
  title={Nonlinear Two-Time-Scale Stochastic Approximation: Convergence and Finite-Time 
Performance},
  author={Doan, Thinh T},
  booktitle={Learning for Dynamics and Control},
  pages={47--47},
  year={2021},
  organization={PMLR}
}

@article{qiu2020single,
  title={Single-timescale stochastic nonconvex-concave optimization for smooth nonlinear TD 
learning},
  author={Qiu, Shuang and Yang, Zhuoran and Wei, Xiaohan and Ye, Jieping and Wang, Zhaoran},
  journal={arXiv preprint arXiv:2008.10103},
  year={2020}
}


@article{wai2020provably,
  title={Provably Efficient Neural GTD for Off-Policy Learning},
  author={Wai, Hoi-To and Yang, Zhuoran and Wang, Zhaoran and Hong, Mingyi},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={10431--10442},
  year={2020}
}

@inproceedings{zhu2020borrowing,
  title={Borrowing from the future: An attempt to address double sampling},
  author={Zhu, Yuhua and Ying, Lexing},
  booktitle={Mathematical and scientific machine learning},
  pages={246--268},
  year={2020},
  organization={PMLR}
}

@article{bhatnagar2016multiscale,
  title={Multiscale Q-learning with linear function approximation},
  author={Bhatnagar, Shalabh and Lakshmanan, K},
  journal={Discrete Event Dynamic Systems},
  volume={26},
  number={3},
  pages={477--509},
  year={2016},
  publisher={Springer}
}

@misc{patterson2021investigating,
  title={Investigating Objectives for Off-policy Value Estimation in Reinforcement Learning},
  author={Patterson, Andrew and Ghiassian, Sina and Gupta, D and White, A and White, M},
  year={2021},
  publisher={Preparation}
}

@article{xian2021faster,
  title={A faster decentralized algorithm for nonconvex minimax problems},
  author={Xian, Wenhan and Huang, Feihu and Zhang, Yanfu and Huang, Heng},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@incollection{roettger2021reviewing,
  title={Reviewing On-Policy/Off-Policy Critic Learning in the Context of Temporal Differences 
and Residual Learning},
  author={Roettger, Frederic},
  booktitle={Reinforcement Learning Algorithms: Analysis and Applications},
  pages={15--24},
  year={2021},
  publisher={Springer}
}

@article{lin2021decentralized,
  title={Decentralized TD (0) With Gradient Tracking},
  author={Lin, Qifeng and Ling, Qing},
  journal={IEEE Signal Processing Letters},
  volume={28},
  pages={723--727},
  year={2021},
  publisher={IEEE}
}

@phdthesis{sun2012generation,
  title={On the generation of representations for reinforcement learning},
  author={Sun, Yi},
  year={2012},
  school={Universit{\`a} della Svizzera italiana}
}

@article{bi2021gradient,
  title={Gradient compensation traces based temporal difference learning},
  author={Bi, Wang and Xuelian, Li and Zhiqiang, Gao and Yang, Chen},
  journal={Neurocomputing},
  volume={442},
  pages={221--235},
  year={2021},
  publisher={Elsevier}
}

@article{wang2021non,
  title={Non-Asymptotic Analysis for Two Time-scale TDC with General Smooth Function 
Approximation},
  author={Wang, Yue and Zou, Shaofeng and Zhou, Yi},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@phdthesis{xiong2021reinforcement,
  title={Reinforcement Learning Algorithms: Acceleration Design and Non-asymptotic Theory},
  author={Xiong, Huaqing},
  year={2021},
  school={The Ohio State University}
}

@article{cao2021gradient,
  title={Gradient temporal-difference learning for off-policy evaluation using emphatic 
weightings},
  author={Cao, Jiaqing and Liu, Quan and Zhu, Fei and Fu, Qiming and Zhong, Shan},
  journal={Information Sciences},
  volume={580},
  pages={311--330},
  year={2021},
  publisher={Elsevier}
}




@inproceedings{yang2021dhqn,
  title={{DHQN:} a Stable Approach to Remove Target Network from Deep Q-learning Network},
  author={Yang, Guang and Li, Yang and Huang, Tian and Li, Qingyun and Chen, Xingguo and others},
  booktitle={2021 IEEE 33rd International Conference on Tools with Artificial Intelligence (ICTAI)},
  pages={1474--1479},
  year={2021},
  organization={IEEE}
}
@inproceedings{tamar2012policy,
  title={Policy gradients with variance related risk criteria},
  author={Tamar, Aviv and Di Castro, Dotan and Mannor, Shie},
  booktitle={Proceedings of the 29th International Conference on Machine Learning},
  pages={1651--1658},
  year={2012}
}
@article{geibel2005risk,
  title={Risk-sensitive reinforcement learning applied to control under constraints},
  author={Geibel, Peter and Wysotzki, Fritz},
  journal={Journal of Artificial Intelligence Research},
  volume={24},
  pages={81--108},
  year={2005}
}
@article{chung1987discounted,
  title={Discounted MDP’s: Distribution functions and exponential utility maximization},
  author={Chung, Kun-Jen and Sobel, Matthew J},
  journal={SIAM journal on control and optimization},
  volume={25},
  number={1},
  pages={49--62},
  year={1987},
  publisher={SIAM}
}
@article{gaskett2003reinforcement,
  title={Reinforcement learning under circumstances beyond its control},
  author={Gaskett, Chris},
  year={2003}
}
@incollection{heger1994consideration,
  title={Consideration of risk in reinforcement learning},
  author={Heger, Matthias},
  booktitle={Machine Learning Proceedings 1994},
  pages={105--111},
  year={1994},
  publisher={Elsevier}
}
@inproceedings{lagoudakis2002value,
  title={Value function approximation in zero-sum markov games},
  author={Lagoudakis, Michail G and Parr, Ronald},
  booktitle={Proceedings of the Eighteenth conference on Uncertainty in artificial intelligence},
  pages={283--292},
  year={2002}
}
@article{szepesvari1999unified,
  title={A unified analysis of value-function-based reinforcement-learning algorithms},
  author={Szepesv{\'a}ri, Csaba and Littman, Michael L},
  journal={Neural computation},
  volume={11},
  number={8},
  pages={2017--2060},
  year={1999},
  publisher={MIT Press One Rogers Street, Cambridge, MA 02142-1209, USA journals-info~…}
}
@incollection{littman1994markov,
  title={Markov games as a framework for multi-agent reinforcement learning},
  author={Littman, Michael L},
  booktitle={Machine learning proceedings 1994},
  pages={157--163},
  year={1994},
  publisher={Elsevier}
}
@article{shapley1953stochastic,
  title={Stochastic games},
  author={Shapley, Lloyd S},
  journal={Proceedings of the national academy of sciences},
  volume={39},
  number={10},
  pages={1095--1100},
  year={1953},
  publisher={National Acad Sciences}
}
@article{zhang2021multi,
  title={Multi-agent reinforcement learning: A selective overview of theories and algorithms},
  author={Zhang, Kaiqing and Yang, Zhuoran and Ba{\c{s}}ar, Tamer},
  journal={Handbook of Reinforcement Learning and Control},
  pages={321--384},
  year={2021},
  publisher={Springer}
}
@inproceedings{zhang2020deep,
  title={Deep Residual Reinforcement Learning},
  author={Zhang, Shangtong and Boehmer, Wendelin and Whiteson, Shimon},
  booktitle={Proceedings of the 19th International Conference on Autonomous Agents and MultiAgent Systems},
  pages={1611--1619},
  year={2020}
}
@inproceedings{levine2017shallow,
  title={Shallow updates for deep reinforcement learning},
  author={Levine, Nir and Zahavy, Tom and Mankowitz, Daniel J and Tamar, Aviv and Mannor, Shie},
  booktitle={Proceedings of the 31st International Conference on Neural Information Processing Systems},
  pages={3138--3148},
  year={2017}
}

@inproceedings{xu2021sample,
  title={Sample complexity bounds for two timescale value-based reinforcement learning algorithms},
  author={Xu, Tengyu and Liang, Yingbin},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={811--819},
  year={2021},
  organization={PMLR}
}
@article{wai2019variance,
  title={Variance reduced policy evaluation with smooth function approximation},
  author={Wai, Hoi-To and Hong, Mingyi and Yang, Zhuoran and Wang, Zhaoran and Tang, Kexin},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  pages={5784--5795},
  year={2019}
}

@inproceedings{qu2019nonlinear,
  title={Nonlinear distributional gradient temporal-difference learning},
  author={Qu, Chao and Mannor, Shie and Xu, Huan},
  booktitle={International Conference on Machine Learning},
  pages={5251--5260},
  year={2019},
  organization={PMLR}
}
@inproceedings{dalal2020tale,
  title={A tale of two-timescale reinforcement learning with the tightest finite-time bound},
  author={Dalal, Gal and Szorenyi, Balazs and Thoppe, Gugan},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={34},
  number={04},
  pages={3701--3708},
  year={2020}
}
@inproceedings{chung2019two,
  title={Two-timescale networks for nonlinear value function approximation},
  author={Chung, Wesley and Nath, Somjit and Joseph, Ajin and White, Martha},
  booktitle={International conference on learning representations},
  year={2019}
}
@inproceedings{dalal2018finite,
  title={Finite sample analysis of two-timescale stochastic approximation with applications to reinforcement learning},
  author={Dalal, Gal and Thoppe, Gugan and Sz{\"o}r{\'e}nyi, Bal{\'a}zs and Mannor, Shie},
  booktitle={Conference On Learning Theory},
  pages={1199--1233},
  year={2018},
  organization={PMLR}
}

@InProceedings{zhang2021average,
  title = 	 {Average-Reward Off-Policy Policy Evaluation with Function Approximation},
  author =       {Zhang, Shangtong and Wan, Yi and Sutton, Richard S and Whiteson, Shimon},
  booktitle = 	 {Proceedings of the 38th International Conference on Machine Learning},
  pages = 	 {12578--12588},
  year = 	 {2021},
}

@inproceedings{zhang2020provably,
  title={Provably Convergent Two-Timescale Off-Policy Actor-Critic with Function Approximation},
  author={Zhang, Shangtong and Liu, Bo and Yao, Hengshuai and Whiteson, Shimon},
  booktitle={Proceedings of the 37th  International Conference on Machine Learning},
  pages={11204-11213},
  year={2020}
}
@inproceedings{yao2008preconditioned,
  title={Preconditioned temporal difference learning},
  author={Yao, Hengshuai and Liu, Zhi-Qiang},
  booktitle={Proceedings of the 25th International Conference on Machine learning},
  pages={1208--1215},
  year={2008}
}
@article{zhang2022truncated,
  title={Truncated emphatic temporal difference methods for prediction and control},
  author={Zhang, Shangtong and Whiteson, Shimon},
  journal={The Journal of Machine Learning Research},
  volume={23},
  number={1},
  pages={6859--6917},
  year={2022},
  publisher={JMLRORG}
}
@inproceedings{pan2017accelerated,
  title={Accelerated gradient temporal difference learning},
  author={Pan, Yangchen and White, Adam and White, Martha},
  booktitle={Proceedings of the 21st AAAI Conference on Artificial Intelligence},
  pages={2464--2470},
  year={2017}
}
@inproceedings{givchi2015quasi,
  title={Quasi newton temporal difference learning},
  author={Givchi, Arash and Palhang, Maziar},
  booktitle={Asian Conference on Machine Learning},
  pages={159--172},
  year={2015}
}
@misc{meyn2009markov,
  title={Markov Chains and Stochastic Stability},
  author={Meyn, Sean and Tweedie, Richard L},
  year={2009},
  publisher={Cambridge University Press}
}
@article{Bertsekas2009Projected,
  title={Projected equation methods for approximate solution of large linear systems},
  author={Bertsekas, Dimitri P and Yu, Huizhen},
  journal={Journal of Computational and Applied Mathematics},
  volume={227},
  number={1},
  pages={27--50},
  year={2009},
  publisher={Elsevier}
}
@inproceedings{zhu2021self,
  title={Self-correcting Q-learning},
  author={Zhu, Rong and Rigotti, Mattia},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={35},
  number={12},
  pages={11185--11192},
  year={2021}
}
@article{john2020generalized,
  title={Generalized speedy q-learning},
  author={John, Indu and Kamanchi, Chandramouli and Bhatnagar, Shalabh},
  journal={IEEE Control Systems Letters},
  volume={4},
  number={3},
  pages={524--529},
  year={2020},
  publisher={IEEE}
}
@article{yu2009convergence,
  title={Convergence results for some temporal difference methods based on least squares},
  author={Yu, Huizhen and Bertsekas, Dimitri P},
  journal={IEEE Transactions on Automatic Control},
  volume={54},
  number={7},
  pages={1515--1531},
  year={2009},
  publisher={IEEE}
}
@article{hallak2015emphatic,
  title={Emphatic TD Bellman operator is a contraction},
  author={Hallak, Assaf and Tamar, Aviv and Mannor, Shie},
  journal={arXiv preprint arXiv:1508.03411},
  year={2015}
}
@inproceedings{ng1999policy,
  title={Policy invariance under reward transformations: Theory and application to reward shaping},
  author={Ng, Andrew Y and Harada, Daishi and Russell, Stuart},
  booktitle={ICML},
  volume={99},
  pages={278--287},
  year={1999}
}
@article{bagheri2014online,
  title={Online adaptable learning rates for the game Connect-4},
  author={Bagheri, Samineh and Thill, Markus and Koch, Patrick and Konen, Wolfgang},
  journal={IEEE Transactions on Computational Intelligence and AI in Games},
  volume={8},
  number={1},
  pages={33--42},
  year={2014},
  publisher={IEEE}
}
@inproceedings{beal1999temporal,
  title={Temporal coherence and prediction decay in TD learning},
  author={Beal, Donald F and Smith, Martin C},
  booktitle={Proceedings of the International Joint Conference on Artificial Intelligence},
  pages={564--569},
  year={1999},
  organization={Citeseer}
}
@inproceedings{matsuzaki2017developing,
  title={Developing a 2048 player with backward temporal coherence learning and restart},
  author={Matsuzaki, Kiminori},
  booktitle={Advances in Computer Games},
  pages={176--187},
  year={2017},
  organization={Springer}
}
@article{jaskowski2017mastering,
  title={Mastering 2048 with delayed temporal coherence learning, multistage weight promotion, redundant encoding, and carousel shaping},
  author={Ja{\'s}kowski, Wojciech},
  journal={IEEE Transactions on Games},
  volume={10},
  number={1},
  pages={3--14},
  year={2017},
  publisher={IEEE}
}
@article{DONG2021327,
title = {Variance aware reward smoothing for deep reinforcement learning},
journal = {Neurocomputing},
volume = {458},
pages = {327-335},
year = {2021},
issn = {0925-2312},
doi = {https://doi.org/10.1016/j.neucom.2021.06.014},
url = {https://www.sciencedirect.com/science/article/pii/S0925231221009139},
author = {Yunlong Dong and Shengjun Zhang and Xing Liu and Yu Zhang and Tan Shen},
keywords = {Deep reinforcement learning, Rewards drop, Variance reduction, Rewards smoothing},
abstract = {A Reinforcement Learning (RL) agent interacts with the environment to learn a policy with high accumulated rewards through attempts and failures. However, RL suffers from its own trial-and-error learning nature, which results in an unstable learning process. In this paper, we investigate a common phenomenon called rewards drop at the late-stage RL training session, where the rewards trajectory oscillates dramatically. In order to solve such a problem, we propose a novel rewards shaping technique named Variance Aware Rewards Smoothing (VAR). We show that the proposed method reduces the variance of rewards and mitigates the rewards drop problem without changing the formulation of the value function. Furthermore, the theoretical analysis of convergence of VAR is provided, which is derived from the γ-contraction operator and the fixed point attribute of the value function. Finally, the theoretical results are illustrated by extensive results on various benchmarks and advanced algorithms across different random seeds to demonstrate the effectiveness and the compatibility of VAR.}
}
@article{meng2019qualitative,
  title={Qualitative measurements of policy discrepancy for return-based deep q-network},
  author={Meng, Wenjia and Zheng, Qian and Yang, Long and Li, Pengfei and Pan, Gang},
  journal={IEEE transactions on neural networks and learning systems},
  volume={31},
  number={10},
  pages={4374--4380},
  year={2019},
  publisher={IEEE}
}
@inproceedings{yang2018unified,
  title={A unified approach for multi-step temporal-difference learning with eligibility traces in reinforcement learning},
  author={Yang, Long and Shi, Minhao and Zheng, Qian and Meng, Wenjia and Pan, Gang},
  booktitle={Proceedings of the 27th International Joint Conference on Artificial Intelligence},
  pages={2984--2990},
  year={2018}
}
@article{kaelbling1996reinforcement,
  title={Reinforcement learning: A survey},
  author={Kaelbling, Leslie Pack and Littman, Michael L and Moore, Andrew W},
  journal={Journal of artificial intelligence research},
  volume={4},
  pages={237--285},
  year={1996}
}
@book{borkar2009stochastic,
  title={Stochastic approximation: a dynamical systems viewpoint},
  author={Borkar, Vivek S},
  volume={48},
  year={2009},
  publisher={Springer}
}
@article{gao2004,
  title={强化学习研究综述},
  author={高阳 and 陈世福 and 陆鑫},
  journal={自动化学报},
  volume={30},
  number={1},
  pages={86--100},
  year={2004}
}
@article{wang2008,
  title={强化学习中的迁移: 方法和进展},
  author={王皓 and 高阳 and 陈兴国},
  journal={电子学报},
  volume={36},
  number={B12},
  pages={39--43},
  year={2008}
}
@article{busoniu2008comprehensive,
  title={A comprehensive survey of multiagent reinforcement learning},
  author={Busoniu, Lucian and Babuska, Robert and De Schutter, Bart},
  journal={IEEE Transactions on Systems, Man, and Cybernetics, Part C (Applications and Reviews)},
  volume={38},
  number={2},
  pages={156--172},
  year={2008},
  publisher={IEEE}
}
@article{taylor2009transfer,
  title={Transfer Learning for Reinforcement Learning Domains: A Survey},
  author={Taylor, Matthew E and Stone, Peter},
  journal={Journal of Machine Learning Research},
  volume={10},
  pages={1633--1685},
  year={2009}
}
@article{garcia2015comprehensive,
  title={A comprehensive survey on safe reinforcement learning},
  author={Garc{\i}a, Javier and Fern{\'a}ndez, Fernando},
  journal={Journal of Machine Learning Research},
  volume={16},
  number={1},
  pages={1437--1480},
  year={2015}
}
@article{chen2016,
  title={强化学习及其在电脑围棋中的应用},
  author={陈兴国 and 俞扬},
  journal={自动化学报},
  volume={42},
  number={5},
  pages={685--695},
  year={2016}
}
@article{li2017deep,
  title={Deep reinforcement learning: An overview},
  author={Li, Yuxi},
  journal={arXiv preprint arXiv:1701.07274},
  year={2017}
}
@article{arulkumaran2017deep,
  title={Deep reinforcement learning: A brief survey},
  author={Arulkumaran, Kai and Deisenroth, Marc Peter and Brundage, Miles and Bharath, Anil Anthony},
  journal={IEEE Signal Processing Magazine},
  volume={34},
  number={6},
  pages={26--38},
  year={2017},
  publisher={IEEE}
}
@article{zhou2017,
  title={分层强化学习综述},
  author={周文吉 and 俞扬},
  journal={智能系统学报},
  volume={12},
  number={5},
  pages={590--594},
  year={2017}
}
@article{zhao2018,
  title={深度强化学习研究综述},
  author={赵星宇 and 丁世飞},
  journal={计算机科学},
  volume={45},
  number={7},
  pages={1--6},
  year={2018}
}
@article{liu2019,
  title={基于值函数和策略梯度的深度强化学习综述},
  author={刘建伟 and 高峰 and 罗雄麟 and others},
  journal={计算机学报},
  volume={42},
  number={6},
  pages={1406--1438},
  year={2019},
  publisher={中国计算机学会| 中国科学院计算技术研究所}
}
@article{du2019,
  title={多智能体强化学习综述},
  author={杜威 and 丁世飞},
  journal={计算机科学},
  volume={46},
  number={8},
  pages={1--8},
  year={2019}
}
@article{zhao2016,
  title={深度强化学习综述: 兼论计算机围棋的发展},
  author={赵冬斌 and 邵坤 and 朱圆恒 and 李栋 and 陈亚冉 and 王海涛 and 刘德荣 and 周彤 and 王成红},
  journal={控制理论与应用},
  volume={33},
  number={6},
  pages={701--717},
  year={2016}
}
@article{yau2017survey,
  title={A survey on reinforcement learning models and algorithms for traffic signal control},
  author={Yau, Kok-Lim Alvin and Qadir, Junaid and Khoo, Hooi Ling and Ling, Mee Hong and Komisarczuk, Peter},
  journal={ACM Computing Surveys (CSUR)},
  volume={50},
  number={3},
  pages={1--38},
  year={2017},
  publisher={ACM New York, NY, USA}
}
@article{liu2018,
  title={深度强化学习综述},
  author={刘全 and 翟建伟 and 章宗长 and 钟珊 and 周倩 and 章鹏 and 徐进},
  journal={计算机学报},
  volume={41},
  number={1},
  pages={1--27},
  year={2018}
}
@article{zhang2019,
  title={基于逆强化学习的示教学习方法综述},
  author={张凯峰 and 俞扬},
  journal={计算机研究与发展},
  volume={56},
  number={2},
  pages={254--261},
  year={2019}
}
@article{wan2019,
  title={深度强化学习理论及其应用综述},
  author={万里鹏 and 兰旭光 and 张翰博 and 郑南宁},
  journal={模式识别与人工智能},
  volume={32},
  number={1},
  pages={67--81},
  year={2019}
}
@article{ying2020,
  title={多智能体分层强化学习综述},
  author={殷昌盛 and 杨若鹏 and 朱巍 and 邹小飞 and 李峰 and others},
  journal={智能系统学报},
  volume={15},
  number={4},
  pages={646--655},
  year={2020}
}
@article{li2020,
  title={基于深度强化学习的组合优化研究进展},
  author={李凯文 and 张涛 and 王锐 and 覃伟健 and 贺惠晖 and 黄鸿},
  journal={自动化学报},
  volume={41},
  pages={1--17},
  year={2020},
  publisher={自动化学报}
}
@article{chen2021,
  title={深度强化学习的攻防与安全性分析综述},
  author={陈晋音 and 章燕 and 王雪柯 and 蔡鸿斌 and 王珏 and 纪守领},
  journal={自动化学报},
  volume={45},
  pages={1--19},
  year={2020},
  publisher={自动化学报}
}
@article{xiong2021,
  title={基于强化学习的综合能源系统管理综述},
  author={熊珞琳 and 毛帅 and 唐漾 and 孟科 and 董朝阳 and 钱锋},
  journal={自动化学报},
  pages={1--20},
  year={2021},
  publisher={自动化学报}
}
@article{kiran2021deep,
  title={Deep reinforcement learning for autonomous driving: A survey},
  author={Kiran, B Ravi and Sobh, Ibrahim and Talpaert, Victor and Mannion, Patrick and Al Sallab, Ahmad A and Yogamani, Senthil and P{\'e}rez, Patrick},
  journal={IEEE Transactions on Intelligent Transportation Systems},
  year={2021},
  publisher={IEEE}
}
@inproceedings{puiutta2020explainable,
  title={Explainable reinforcement learning: A survey},
  author={Puiutta, Erika and Veith, Eric MSP},
  booktitle={International Cross-Domain Conference for Machine Learning and Knowledge Extraction},
  pages={77--95},
  year={2020},
  organization={Springer}
}
@article{haydari2020deep,
  title={Deep reinforcement learning for intelligent transportation systems: A survey},
  author={Haydari, Ammar and Yilmaz, Yasin},
  journal={IEEE Transactions on Intelligent Transportation Systems},
  year={2020},
  publisher={IEEE}
}
@article{alharin2020reinforcement,
  title={Reinforcement Learning Interpretation Methods: A Survey},
  author={Alharin, Alnour and Doan, Thanh-Nam and Sartipi, Mina},
  journal={IEEE Access},
  volume={8},
  pages={171058--171077},
  year={2020},
  publisher={IEEE}
}
@article{coronato2020reinforcement,
  title={Reinforcement learning for intelligent healthcare applications: A survey},
  author={Coronato, Antonio and Naeem, Muddasar and De Pietro, Giuseppe and Paragliola, Giovanni},
  journal={Artificial Intelligence in Medicine},
  volume={109},
  pages={101964},
  year={2020},
  publisher={Elsevier}
}
@article{arora2021survey,
  title={A survey of inverse reinforcement learning: Challenges, methods and progress},
  author={Arora, Saurabh and Doshi, Prashant},
  journal={Artificial Intelligence},
  pages={103500},
  year={2021},
  publisher={Elsevier}
}
@article{mazyavkina2021reinforcement,
  title={Reinforcement learning for combinatorial optimization: A survey},
  author={Mazyavkina, Nina and Sviridov, Sergey and Ivanov, Sergei and Burnaev, Evgeny},
  journal={Computers \& Operations Research},
  pages={105400},
  year={2021},
  publisher={Elsevier}
}
@inproceedings{luketina2019survey,
  title={A Survey of Reinforcement Learning Informed by Natural Language},
  author={Luketina, J and Nardelli, N and Farquhar, G and Foerster, J and Andreas, J and Grefenstette, E and Whiteson, S and Rockt{\"a}schel, T},
  booktitle={Proceedings of the Twenty-Eighth International Joint Conference on Artificial Intelligence, IJCAI 2019, August 10-16 2019, Macao, China.},
  volume={57},
  pages={6309--6317},
  year={2019},
  organization={AAAI Press (Association for the Advancement of Artificial Intelligence)}
}
@article{sun2020,
  title={多智能体深度强化学习的若干关键科学问题},
  author={孙长银 and 穆朝絮},
  journal={自动化学报},
  volume={46},
  number={7},
  pages={1301--1312},
  year={2020}
}
@article{yang2020,
  title={深度强化学习中稀疏奖励问题研究综述},
  author={杨惟轶 and 白辰甲 and 蔡超 and 赵英男 and 刘鹏},
  journal={计算机科学},
  volume={47},
  number={3},
  pages={182--191},
  year={2020}
}
@inproceedings{gopalan2014thompson,
	title={Thompson sampling for complex online problems},
	author={Gopalan, Aditya and Mannor, Shie and Mansour, Yishay},
	booktitle={International Conference on Machine Learning},
	pages={100--108},
	year={2014}
}
@inproceedings{stachniss2005information,
	title={Information gain-based exploration using rao-blackwellized particle filters.},
	author={Stachniss, Cyrill and Grisetti, Giorgio and Burgard, Wolfram},
	booktitle={Robotics: Science and Systems},
	volume={2},
	pages={65--72},
	year={2005}
}
@article{LattimoreH14a,
	author    = {Tor Lattimore and
	Marcus Hutter},
	title     = {Near-optimal {PAC} bounds for discounted MDPs},
	journal   = {Theor. Comput. Sci.},
	volume    = {558},
	pages     = {125--143},
	year      = {2014}
}
@article{JakschOA10,
	author    = {Thomas Jaksch and
	Ronald Ortner and
	Peter Auer},
	title     = {Near-optimal Regret Bounds for Reinforcement Learning},
	journal   = {Journal of Machine Learning Research},
	volume    = {11},
	pages     = {1563--1600},
	year      = {2010}
}
@inproceedings{AuerO06,
	author    = {Peter Auer and
	Ronald Ortner},
	title     = {Logarithmic Online Regret Bounds for Undiscounted Reinforcement Learning},
	booktitle = {Advances in Neural Information Processing Systems},
	pages     = {49--56},
	year      = {2006}
}
@inproceedings{SzitaS10,
	author    = {Istvan Szita and
	Csaba Szepesv{\'{a}}ri},
	title     = {Model-based reinforcement learning with nearly tight exploration complexity
	bounds},
	booktitle = {Proceedings of the 27th International Conference on Machine Learning},
	pages     = {1031--1038},
	year      = {2010}
}
@inproceedings{StrehlL05,
	author    = {Alexander L. Strehl and
	Michael L. Littman},
	title     = {A theoretical analysis of Model-Based Interval Estimation},
	booktitle = {Proceedings of the 22nd International Conference Machine Learning},
	pages     = {856--863},
	year      = {2005}
}
@article{BrafmanT02,
	author    = {Ronen I. Brafman and
	Moshe Tennenholtz},
	title     = {{R-MAX} - {A} General Polynomial Time Algorithm for Near-Optimal Reinforcement
	Learning},
	journal   = {Journal of Machine Learning Research},
	volume    = {3},
	pages     = {213--231},
	year      = {2002}
}
@inproceedings{SinghBC04,
	author    = {Satinder P. Singh and
	Andrew G. Barto and
	Nuttapong Chentanez},
	title     = {Intrinsically Motivated Reinforcement Learning},
	booktitle = {Advances in Neural Information Processing Systems},
	pages     = {1281--1288},
	year      = {2004}
}
@inproceedings{tang2017exploration,
  title={\# Exploration: A study of count-based exploration for deep reinforcement learning},
  author={Tang, Haoran and Houthooft, Rein and Foote, Davis and Stooke, Adam and Chen, OpenAI Xi and Duan, Yan and Schulman, John and DeTurck, Filip and Abbeel, Pieter},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2753--2762},
  year={2017}
}

@article{kong2020rankmax,
  title={Rankmax: An Adaptive Projection Alternative to the Softmax Function},
  author={Kong, Weiwei and Krichene, Walid and Mayoraz, Nicolas and Rendle, Steffen and Zhang, Li},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={633--643},
  year={2020}
}
@inproceedings{song2019revisiting,
  title={Revisiting the softmax bellman operator: New benefits and new perspective},
  author={Song, Zhao and Parr, Ron and Carin, Lawrence},
  booktitle={International Conference on Machine Learning},
  pages={5916--5925},
  year={2019},
  organization={PMLR}
}
@inproceedings{kim2019removing,
  title={Removing the target network from deep Q-networks with the Mellowmax operator},
  author={Kim, Seungchan and Asadi, Kavosh and Littman, Michael and Konidaris, George},
  booktitle={Proceedings of the 18th International Conference on Autonomous Agents and MultiAgent Systems},
  pages={2060--2062},
  year={2019}
}

@inproceedings{papini2018stochastic,
  title={Stochastic variance-reduced policy gradient},
  author={Papini, Matteo and Binaghi, Damiano and Canonaco, Giuseppe and Pirotta, Matteo and Restelli, Marcello},
  booktitle={International conference on machine learning},
  pages={4026--4035},
  year={2018},
  organization={PMLR}
}
@article{veness2011variance,
  title={Variance reduction in monte-carlo tree search},
  author={Veness, Joel and Lanctot, Marc and Bowling, Michael},
  journal={Advances in Neural Information Processing Systems},
  volume={24},
  pages={1836--1844},
  year={2011},
  publisher={Citeseer}
}
@article{greensmith2004variance,
  title={Variance Reduction Techniques for Gradient Estimates in Reinforcement Learning},
  author={Greensmith, Evan and Bartlett, Peter L and Baxter, Jonathan},
  journal={Journal of Machine Learning Research},
  volume={5},
  number={Nov},
  pages={1471--1530},
  year={2004},
  publisher={Citeseer}
}

@article{johnson2013accelerating,
  title={Accelerating stochastic gradient descent using predictive variance reduction},
  author={Johnson, Rie and Zhang, Tong},
  journal={Advances in neural information processing systems},
  volume={26},
  pages={315--323},
  year={2013}
}

@article{clay2021learning,
  title={Learning sparse and meaningful representations through embodiment},
  author={Clay, Viviane and K{\"o}nig, Peter and K{\"u}hnberger, Kai-Uwe and Pipa, Gordon},
  journal={Neural Networks},
  volume={134},
  pages={23--41},
  year={2021},
  publisher={Elsevier}
}
@inproceedings{ghiassian2020improving,
  title={Improving Performance in Reinforcement Learning by Breaking Generalization in Neural Networks},
  author={Ghiassian, Sina and Rafiee, Banafsheh and Lo, Yat Long and White, Adam},
  booktitle={Proceedings of the 19th International Conference on Autonomous Agents and MultiAgent Systems},
  pages={438--446},
  year={2020}
}
@inproceedings{bengio2020interference,
  title={Interference and generalization in temporal difference learning},
  author={Bengio, Emmanuel and Pineau, Joelle and Precup, Doina},
  booktitle={International Conference on Machine Learning},
  pages={767--777},
  year={2020},
  organization={PMLR}
}

@inproceedings{makhzani2013k,
  title={K-sparse autoencoders},
  author={Makhzani, Alireza and Frey, Brendan},
  booktitle={Proc. 2nd Int. Conf. Learn. Repr.},
  address={Banff, AB, Canada},
  month={Apr.},
  year={2014}
}

@inproceedings{nair2010rectified,
  title={Rectified linear units improve restricted boltzmann machines},
  author={Nair, Vinod and Hinton, Geoffrey E},
  booktitle={Proc. 27th Int. Conf. Mach. Learn.},
  address={Haifa, Israel},
  month={Jun.},
  pages={807--814},
  year={2010}
}

@inproceedings{sutton1996generalization,
  title={Generalization in reinforcement learning: Successful examples using sparse coarse coding},
  author={Sutton, Richard S},
  booktitle={Proc. Adv. Neural Inf. Process. Syst. 9},
  address={Denver, CO, USA},
  month={Dec.},
  year={1996}
}
@inproceedings{javed2019meta,
  title={Meta-learning representations for continual learning},
  author={Javed, Khurram and White, Martha},
  booktitle={Proceedings of the 33rd International Conference on Neural Information Processing Systems},
  pages={1820--1830},
  year={2019}
}
@inproceedings{liu2019utility,
  title={The utility of sparse representations for control in reinforcement learning},
  author={Liu, Vincent and Kumaraswamy, Raksha and Le, Lei and White, Martha},
  booktitle={Proceedings of the aaai conference on artificial intelligence},
  volume={33},
  number={01},
  pages={4384--4391},
  year={2019}
}


@article{manela2021bias,
  title={Bias-reduced hindsight experience replay with virtual goal prioritization},
  author={Manela, Binyamin and Biess, Armin},
  journal={Neurocomputing},
  volume={451},
  pages={305--315},
  year={2021},
  publisher={Elsevier}
}
@article{liu2021experience,
  title={Experience replay is associated with efficient nonlocal learning},
  author={Liu, Y and Mattar, MG and Behrens, TEJ and Daw, ND and Dolan, RJ},
  journal={Science},
  volume={372},
  number={6544},
  year={2021},
  publisher={American Association for the Advancement of Science}
}
@inproceedings{luo2020dynamic,
  title={Dynamic experience replay},
  author={Luo, Jieliang and Li, Hui},
  booktitle={Conference on Robot Learning},
  pages={1191--1200},
  year={2020},
  organization={PMLR}
}
@article{daley2019reconciling,
  title={Reconciling $\lambda$-returns with experience replay},
  author={Daley, Brett and Amato, Christopher},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  pages={1133--1142},
  year={2019}
}
@inproceedings{sun2020attentive,
  title={Attentive experience replay},
  author={Sun, Peiquan and Zhou, Wengang and Li, Houqiang},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={34},
  number={04},
  pages={5900--5907},
  year={2020}
}
@inproceedings{liu2019competitive,
  title={Competitive experience replay},
  author={Liu, Hao and Trott, Alexander and Socher, Richard and Xiong, Caiming},
  booktitle={International Conference on Learning Representations},
  year={2019}
}
@inproceedings{fedus2020revisiting,
  title={Revisiting fundamentals of experience replay},
  author={Fedus, William and Ramachandran, Prajit and Agarwal, Rishabh and Bengio, Yoshua and Larochelle, Hugo and Rowland, Mark and Dabney, Will},
  booktitle={International Conference on Machine Learning},
  pages={3061--3071},
  year={2020},
  organization={PMLR}
}
@inproceedings{andrychowicz2017hindsight,
  title={Hindsight experience replay},
  author={Andrychowicz, Marcin and Wolski, Filip and Ray, Alex and Schneider, Jonas and Fong, Rachel and Welinder, Peter and McGrew, Bob and Tobin, Josh and Abbeel, Pieter and Zaremba, Wojciech},
  booktitle={Proceedings of the 31st International Conference on Neural Information Processing Systems},
  pages={5055--5065},
  year={2017}
}

@inproceedings{asadi2017alternative,
  title={An alternative softmax operator for reinforcement learning},
  author={Asadi, Kavosh and Littman, Michael L},
  booktitle={International Conference on Machine Learning},
  pages={243--252},
  year={2017}
}
@article{lv2019stochastic,
  title={Stochastic Double Deep Q-Network},
  author={Lv, Pingli and Wang, Xuesong and Cheng, Yuhu and Duan, Ziming},
  journal={IEEE Access},
  volume={7},
  pages={79446--79454},
  year={2019},
  publisher={IEEE}
}
@book{li2000book,
  title={数值计算原理},
  author={李庆扬},
  publisher={清华大学出版社},
  year={2000},
}
@book{tang2019book,
  author = {唐艳},
  publisher = {重庆大学出版社},
  title = {不动点与零点的迭代逼近及应用},
  year = {2019}
}
@inproceedings{fu2019diagnosing,
  title={Diagnosing Bottlenecks in Deep Q-learning Algorithms},
  author={Fu, Justin and Kumar, Aviral and Soh, Matthew and Levine, Sergey},
  booktitle={International Conference on Machine Learning},
  pages={2021--2030},
  year={2019}
}

@inproceedings{glorot2011deep,
	title={Deep Sparse Rectifier Neural Networks},
	author={Glorot, Xavier and Bordes, Antoine and Bengio, Yoshua},
	booktitle={Proceedings of the Fourteenth International Conference on Artificial Intelligence and Statistics},
	volume={15},
	pages={315--323},
	year={2011}
}
@article{srivastava2014dropout,
  title={Dropout: a simple way to prevent neural networks from overfitting},
  author={Srivastava, Nitish and Hinton, Geoffrey and Krizhevsky, Alex and Sutskever, Ilya and Salakhutdinov, Ruslan},
  journal={The journal of machine learning research},
  volume={15},
  number={1},
  pages={1929--1958},
  year={2014},
  publisher={JMLR. org}
}
@inproceedings{lee2007sparse,
	title={Sparse deep belief net model for visual area V2},
	author={Lee, Honglak and Ekanadham, Chaitanya and Ng, Andrew Y},
	booktitle={Advances in Neural Information Processing Systems 20},
	pages={873--880},
	year={2007}
}

@inproceedings{makhzani2015winner,
	title={Winner-take-all autoencoders},
	author={Makhzani, Alireza and Frey, Brendan J},
	booktitle={NIPS'15 Proceedings of the 28th International Conference on Neural Information Processing Systems - Volume 2},
  address={Montreal, Quebec, Canada},
  month={December},
	pages={2791--2799},
	year={2015}
}

@inproceedings{makhzani2014k,
	title={k-Sparse Autoencoders},
	author={Makhzani, Alireza and Frey, Brendan},
	booktitle={ICLR 2014 : International Conference on Learning Representations (ICLR) 2014},
	year={2014}
}

@inproceedings{le2017learning,
	title={Learning sparse representations in reinforcement learning with sparse coding},
	author={Le, Lei and Kumaraswamy, Raksha and White, Martha},
	booktitle={IJCAI'17 Proceedings of the 26th International Joint Conference on Artificial Intelligence},
	pages={2067--2073},
	year={2017}
}
@inproceedings{liang2016state,
  title={State of the Art Control of Atari Games Using Shallow Reinforcement Learning},
  author={Liang, Yitao and Machado, Marlos C and Talvitie, Erik and Bowling, Michael},
  booktitle={Proceedings of the 2016 International Conference on Autonomous Agents \& Multiagent Systems},
  pages={485--493},
  year={2016}
}

@inproceedings{ashiquzzaman2019compacting,
  title={Compacting deep neural networks for light weight iot \& scada based applications with node pruning},
  author={Ashiquzzaman, Akm and Van Ma, Linh and Kim, SangWoo and Lee, Dongsu and Um, Tai-Won and Kim, Jinsul},
  booktitle={2019 International Conference on Artificial Intelligence in Information and Communication (ICAIIC)},
  pages={082--085},
  year={2019},
  organization={IEEE}
}
@article{nagarajan2018deterministic,
  title={Deterministic implementations for reproducibility in deep reinforcement learning},
  author={Nagarajan, Prabhat and Warnell, Garrett and Stone, Peter},
  journal={arXiv preprint arXiv:1809.05676},
  year={2018}
}
@inproceedings{dong2017learning,
  title={Learning to prune deep neural networks via layer-wise optimal brain surgeon},
  author={Dong, Xin and Chen, Shangyu and Pan, Sinno},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4857--4867},
  year={2017}
}
@article{dickinson2020state,
  title={State of the Journal Editorial},
  author={Dickinson, Sven},
  journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},
  volume={42},
  number={2},
  pages={253--260},
  year={2020},
  publisher={IEEE}
}
@article{moravvcik2017deepstack,
  title={DeepStack: Expert-level artificial intelligence in heads-up no-limit poker},
  author={Morav{\v{c}}{\'\i}k, Matej and Schmid, Martin and Burch, Neil and Lis{\`y}, Viliam and Morrill, Dustin and Bard, Nolan and Davis, Trevor and Waugh, Kevin and Johanson, Michael and Bowling, Michael},
  journal={Science},
  volume={356},
  number={6337},
  pages={508--513},
  year={2017}
}
@inproceedings{nagarajan2019Deterministic,
  title={Deterministic Implementations for Reproducibility in Deep Reinforcement Learning},
  author={Nagarajan Prabhat, Garrett Warnell, and Peter Stone},
  booktitle={AAAI 2019 Workshop on Reproducible AI},
  pages={},
  year={2019}
}
@inproceedings{arulkumaran2019alphastar,
  title={Alphastar: An evolutionary computation perspective},
  author={Arulkumaran, Kai and Cully, Antoine and Togelius, Julian},
  booktitle={Proceedings of the Genetic and Evolutionary Computation Conference Companion},
  pages={314--315},
  year={2019}
}
@article{hernandez2019survey,
  title={A survey and critique of multiagent deep reinforcement learning},
  author={Hernandez-Leal, Pablo and Kartal, Bilal and Taylor, Matthew E},
  journal={Autonomous Agents and Multi-Agent Systems},
  volume={33},
  number={6},
  pages={750--797},
  year={2019},
  publisher={Springer}
}
@article{kondo2019playing,
  title={Playing game 2048 with deep convolutional neural networks trained by supervised learning},
  author={Kondo, Naoki and Matsuzaki, Kiminori},
  journal={Journal of Information Processing},
  volume={27},
  pages={340--347},
  year={2019},
  publisher={Information Processing Society of Japan}
}
@inproceedings{schrum2018evolving,
  title={Evolving indirectly encoded convolutional neural networks to play tetris with low-level features},
  author={Schrum, Jacob},
  booktitle={Proceedings of the Genetic and Evolutionary Computation Conference},
  pages={205--212},
  year={2018}
}
@inproceedings{henderson2018deep,
  title={Deep reinforcement learning that matters},
  author={Henderson, Peter and Islam, Riashat and Bachman, Philip and Pineau, Joelle and Precup, Doina and Meger, David},
  booktitle={Thirty-Second AAAI Conference on Artificial Intelligence},
   pages={3207--3214},
  year={2018}
}
@article{efron2004least,
  title={Least angle regression},
  author={Efron, Bradley and Hastie, Trevor and Johnstone, Iain and Tibshirani, Robert and others},
  journal={The Annals of statistics},
  volume={32},
  number={2},
  pages={407--499},
  year={2004},
  publisher={Institute of Mathematical Statistics}
}

@article{candes2007dantzig,
  title={The Dantzig selector: Statistical estimation when p is much larger than n},
  author={Candes, Emmanuel and Tao, Terence and others},
  journal={The annals of Statistics},
  volume={35},
  number={6},
  pages={2313--2351},
  year={2007},
  publisher={Institute of Mathematical Statistics}
}

@inproceedings{pati1993orthogonal,
  title={Orthogonal matching pursuit: Recursive function approximation with applications to wavelet decomposition},
  author={Pati, Yagyensh Chandra and Rezaiifar, Ramin and Krishnaprasad, Perinkulam Sambamurthy},
  booktitle={Proceedings of 27th Asilomar conference on signals, systems and computers},
  pages={40--44},
  year={1993},
  organization={IEEE}
}

@article{de2005tutorial,
  title={A tutorial on the cross-entropy method},
  author={De Boer, Pieter-Tjerk and Kroese, Dirk P and Mannor, Shie and Rubinstein, Reuven Y},
  journal={Annals of operations research},
  volume={134},
  number={1},
  pages={19--67},
  year={2005},
  publisher={Springer}
}

@article{zhao2009composite,
  title={The composite absolute penalties family for grouped and hierarchical variable selection},
  author={Zhao, Peng and Rocha, Guilherme and Yu, Bin and others},
  journal={The Annals of Statistics},
  volume={37},
  number={6A},
  pages={3468--3497},
  year={2009},
  publisher={Institute of Mathematical Statistics}
}

@inproceedings{jenatton2010proximal,
	title={Proximal Methods for Sparse Hierarchical Dictionary Learning},
	author={Jenatton, Rodolphe and Mairal, Julien and Obozinski, Guillaume and Bach, Francis R},
	booktitle={Proceedings of the 27th International Conference on Machine Learning},
	pages={487--494},
	year={2010}
}

@article{tesauro1994td,
  title={TD-Gammon, a self-teaching backgammon program, achieves master-level play},
  author={Tesauro, Gerald},
  journal={Neural computation},
  volume={6},
  number={2},
  pages={215--219},
  year={1994},
  publisher={MIT Press}
}
@inproceedings{sherstan2018comparing,
  title={Comparing Direct and Indirect Temporal-Difference Methods for Estimating the Variance of the Return.},
  author={Sherstan, Craig and Ashley, Dylan R and Bennett, Brendan and Young, Kenny and White, Adam and White, Martha and Sutton, Richard S},
  booktitle={UAI},
  pages={63--72},
  year={2018}
}
@inproceedings{yin2017knowledge,
  title={Knowledge transfer for deep reinforcement learning with hierarchical experience replay},
  author={Yin, Haiyan and Pan, Sinno Jialin},
  booktitle={Proceedings of the Thirty-First AAAI Conference on Artificial Intelligence},
  pages={1640--1646},
  year={2017}
}
@inproceedings{isele2018selective,
  title={Selective experience replay for lifelong learning},
  author={Isele, David and Cosgun, Akansel},
  booktitle={AAAI Conference on Artificial Intelligence 2018},
  pages={3302--3309},
  year={2018},
  organization={Association for the Advancement of Artificial Intelligence (AAAI)}
}
@inproceedings{zha2019experience,
  title={Experience replay optimization},
  author={Zha, Daochen and Lai, Kwei-Herng and Zhou, Kaixiong and Hu, Xia},
  booktitle={Proceedings of the 28th International Joint Conference on Artificial Intelligence},
  pages={4243--4249},
  year={2019},
  organization={AAAI Press}
}
@phdthesis{liu2019sparse,
  title={Sparse Representation Neural Networks for Online Reinforcement Learning},
  author={Liu, Vincent},
  year={2019},
  school={University of Alberta}
}
@inproceedings{pan2018asynchronous,
  title={Asynchronous value iteration network},
  author={Pan, Zhiyuan and Zhang, Zongzhang and Chen, Zixuan},
  booktitle={International Conference on Neural Information Processing},
  pages={169--180},
  year={2018},
  organization={Springer}
}
@article{elfwing2018sigmoid,
  title={Sigmoid-weighted linear units for neural network function approximation in reinforcement learning},
  author={Elfwing, Stefan and Uchibe, Eiji and Doya, Kenji},
  journal={Neural Networks},
  volume={107},
  pages={3--11},
  year={2018},
  publisher={Elsevier}
}
@inproceedings{zhang2020universal,
  title={Universal Value Iteration Networks: When Spatially-Invariant Is Not Universal},
  author={Zhang, Li and Li, Xin and Chen, Sen and Zang, Hongyu and Huang, Jie and Wang, Mingzhong},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={34},
  number={04},
  pages={6778--6785},
  year={2020}
}
@inproceedings{maei2009convergent,
  title={Convergent temporal-difference learning with arbitrary smooth function approximation.},
  author={Maei, Hamid Reza and Szepesvari, Csaba and Bhatnagar, Shalabh and Precup, Doina and Silver, David and Sutton, Richard S},
  booktitle={NIPS},
  pages={1204--1212},
  year={2009}
}
@inproceedings{mnih2016asynchronous,
  title={Asynchronous methods for deep reinforcement learning},
  author={Mnih, Volodymyr and Badia, Adria Puigdomenech and Mirza, Mehdi and Graves, Alex and Lillicrap, Timothy and Harley, Tim and Silver, David and Kavukcuoglu, Koray},
  booktitle={International conference on machine learning},
  pages={1928--1937},
  year={2016},
  organization={PMLR}
}
@article{schaul2015prioritized,
  title={Prioritized experience replay},
  author={Schaul, Tom and Quan, John and Antonoglou, Ioannis and Silver, David},
  journal={arXiv preprint arXiv:1511.05952},
  year={2015}
}
@inproceedings{cai2019neural,
  title={Neural temporal-difference learning converges to global optima},
  author={Cai, Qi and Yang, Zhuoran and Lee, Jason D and Wang, Zhaoran},
  booktitle={Advances in Neural Information Processing Systems},
  pages={11312--11322},
  year={2019}
}
@article{mahadevan2007proto,
  title={Proto-value functions: A Laplacian framework for learning representation and control in Markov decision processes},
  author={Mahadevan, Sridhar and Maggioni, Mauro},
  journal={Journal of Machine Learning Research},
  volume={8},
  number={Oct},
  pages={2169--2231},
  year={2007}
}

@article{albus1975a,
	title={A New Approach to Manipulator Control: The Cerebellar Model Articulation Controller (CMAC)},
	author={Albus, James S},
	journal={Journal of Dynamic Systems Measurement and Control-transactions of The Asme},
	volume={97},
	number={3},
	pages={220--227},
	year={1975}
}

@article{sutton2016emphatic,
  title={An emphatic approach to the problem of off-policy temporal-difference learning},
  author={Sutton, Richard S and Mahmood, A Rupam and White, Martha},
  journal={The Journal of Machine Learning Research},
  volume={17},
  number={1},
  pages={2603--2631},
  year={2016},
  publisher={JMLR. org}
}

@inproceedings{kolter2009regularization,
  title={Regularization and feature selection in least-squares temporal difference learning},
  author={Kolter, J Zico and Ng, Andrew Y},
  booktitle={Proceedings of the 26th annual international conference on machine learning},
  pages={521--528},
  year={2009}
}
@inproceedings{hallak2017consistent,
  title={Consistent On-Line Off-Policy Evaluation},
  author={Hallak, Assaf and Mannor, Shie},
  booktitle={Proceedings of the 34th International Conference on Machine Learning},
  pages={1372-1383},
  year={2017}
}

@inproceedings{parr2007analyzing,
  title={Analyzing feature generation for value-function approximation},
  author={Parr, Ronald and Painter-Wakefield, Christopher and Li, Lihong and Littman, Michael},
  booktitle={Proceedings of the 24th International Conference on Machine Learning},
  pages={737--744},
  year={2007}
}

@inproceedings{keller2006automatic,
  title={Automatic basis function construction for approximate dynamic programming and reinforcement learning},
  author={Keller, Philipp W and Mannor, Shie and Precup, Doina},
  booktitle={Proceedings of the 23rd international conference on Machine learning},
  pages={449--456},
  year={2006}
}

@inproceedings{kuss2004gaussian,
  title={Gaussian processes in reinforcement learning},
  author={Kuss, Malte and Rasmussen, Carl E},
  booktitle={Advances in neural information processing systems},
  pages={751--758},
  year={2004}
}

@article{menache2005basis,
  title={Basis function adaptation in temporal difference reinforcement learning},
  author={Menache, Ishai and Mannor, Shie and Shimkin, Nahum},
  journal={Annals of Operations Research},
  volume={134},
  number={1},
  pages={215--238},
  year={2005},
  publisher={Springer}
}

@inproceedings{engel2003bayes,
  title={Bayes meets Bellman: The Gaussian process approach to temporal difference learning},
  author={Engel, Yaakov and Mannor, Shie and Meir, Ron},
  booktitle={Proceedings of the 20th International Conference on Machine Learning (ICML-03)},
  pages={154--161},
  year={2003}
}

@inproceedings{petrik2010feature,
	title={Feature Selection Using Regularization in Approximate Linear Programs for Markov Decision Processes},
	author={Petrik, Marek and Taylor, Gavin and Parr, Ron and Zilberstein, Shlomo},
	booktitle={Proceedings of the 27th International Conference on Machine Learning},
	pages={871--878},
	year={2010}
}

@inproceedings{geramifard2011online,
  title={Online Discovery of Feature Dependencies.},
  author={Geramifard, Alborz and Doshi, Finale and Redding, Josh and Roy, Nicholas and How, Jonathan P},
  booktitle={ICML},
  pages={881--888},
  year={2011}
}


@inproceedings{painter-wakefield2012greedy,
	title={Greedy Algorithms for Sparse Reinforcement Learning},
	author={Painter-Wakefield, Christopher and Parr, Ronald},
	booktitle={Proceedings of the 29th International Conference on Machine Learning},
	pages={867--874},
	year={2012}
}

@inproceedings{johns2010linear,
  title={Linear complementarity for regularized policy evaluation and improvement},
  author={Johns, Jeffrey and Painter-Wakefield, Christopher and Parr, Ronald},
  booktitle={Advances in neural information processing systems},
  pages={1009--1017},
  year={2010}
}

@inproceedings{taylor2009kernelized,
  title={Kernelized value function approximation for reinforcement learning},
  author={Taylor, Gavin and Parr, Ronald},
  booktitle={Proceedings of the 26th annual international conference on machine learning},
  pages={1017--1024},
  year={2009}
}

@inproceedings{geramifard2006incremental,
  title={Incremental least-squares temporal difference learning},
  author={Geramifard, Alborz and Bowling, Michael and Sutton, Richard S},
  booktitle={Proceedings of the National Conference on Artificial Intelligence},
  volume={21},
  number={1},
  pages={356},
  year={2006},
  organization={Menlo Park, CA; Cambridge, MA; London; AAAI Press; MIT Press; 1999}
}


@inproceedings{liu2012regularized,
  title={Regularized off-policy TD-learning},
  author={Liu, Bo and Mahadevan, Sridhar and Liu, Ji},
  booktitle={Advances in Neural Information Processing Systems},
  pages={836--844},
  year={2012}
}

@inproceedings{geramifard2013batch,
	title={Batch-iFDD for representation expansion in large MDPs},
	author={Geramifard, Alborz and Walsh, Thomas J and Roy, Nicholas and How, Jonathan},
	booktitle={UAI'13 Proceedings of the Twenty-Ninth Conference on Uncertainty in Artificial Intelligence},
	pages={242--251},
	year={2013}
}

@article{farahmand2011model,
  title={Model selection in reinforcement learning},
  author={Farahmand, Amir-massoud and Szepesv{\'a}ri, Csaba},
  journal={Machine learning},
  volume={85},
  number={3},
  pages={299--332},
  year={2011},
  publisher={Springer}
}

@inproceedings{hachiya2010feature,
  title={Feature selection for reinforcement learning: Evaluating implicit state-reward dependency via conditional mutual information},
  author={Hachiya, Hirotaka and Sugiyama, Masashi},
  booktitle={Joint European Conference on Machine Learning and Knowledge Discovery in Databases},
  pages={474--489},
  year={2010},
  organization={Springer}
}

@inproceedings{geist2012a,
	title={A Dantzig Selector Approach to Temporal Difference Learning},
	author={Geist, Matthieu and Scherrer, Bruno and Lazaric, Alessandro and Ghavamzadeh, Mohammad},
	booktitle={Proceedings of the 29th International Conference on Machine Learning},
	pages={347--354},
	year={2012}
}

@inproceedings{kretchmar1997comparison,
  title={Comparison of CMACs and radial basis functions for local function approximators in reinforcement learning},
  author={Kretchmar, R Matthew and Anderson, Charles W},
  booktitle={Proceedings of International Conference on Neural Networks (ICNN'97)},
  volume={2},
  pages={834--837},
  year={1997},
  organization={IEEE}
}

@inproceedings{geist2011ℓ,
  title={ℓ 1-Penalized projected bellman residual},
  author={Geist, Matthieu and Scherrer, Bruno},
  booktitle={European Workshop on Reinforcement Learning},
  pages={89--101},
  year={2011},
  organization={Springer}
}

@inproceedings{jung2006least,
  title={Least squares SVM for least squares TD learning},
  author={Jung, Tobias and Polani, Daniel},
  booktitle={ECAI},
  pages={499--503},
  year={2006},
  organization={Citeseer}
}

@article{painter2012l1,
  title={L1 regularized linear temporal difference learning},
  author={Painter-Wakefield, Christopher and Parr, Ronald and Durham, NC},
  journal={Technical report: Department of Computer Science, Duke University, Durham, NC, TR-2012--01},
  year={2012}
}

@article{johns2009sparse,
  title={Sparse approximate policy evaluation using graph-based basis functions},
  author={Johns, Jeff and Mahadevan, Sridhar},
  journal={Dept. Comput. Sci., Univ. Massachusetts Amherst, Amherst, MA, USA, Tech. Rep. UM-CS-2009-041},
  year={2009}
}

@inproceedings{farahmand2009regularized,
  title={Regularized policy iteration},
  author={Farahmand, Amir M and Ghavamzadeh, Mohammad and Mannor, Shie and Szepesv{\'a}ri, Csaba},
  booktitle={Advances in Neural Information Processing Systems},
  pages={441--448},
  year={2009}
}

@article{pires2012statistical,
	title={Statistical analysis of L1-penalized linear estimation with applications},
	author={Bernardo Avila {Pires}},
	journal={},
	year={2012}
}
@book{Sutton2018book,
  author = {Sutton, Richard S. and Barto, Andrew G.},
  edition = {Second},
  publisher = {The MIT Press},
  title = {Reinforcement Learning: An Introduction},
  year = {2018 }
}
@conference{Parisi2017goal,
  title = {Goal-driven dimensionality reduction for reinforcement learning},
  author = {Parisi, S. and Ramstedt, S. and Peters, J.},
  booktitle = {IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},
  pages = {4634--4639},
  publisher = {IEEE},
  month = sep,
  year = {2017},
  month_numeric = {9}
}

@inproceedings{zhao2011analysis,
  title={Analysis and improvement of policy gradient estimation},
  author={Zhao, Tingting and Hachiya, Hirotaka and Niu, Gang and Sugiyama, Masashi},
  booktitle={Advances in Neural Information Processing Systems},
  pages={262--270},
  year={2011}
}
@inproceedings{weaver2001optimal,
  title={The optimal reward baseline for gradient-based reinforcement learning},
  author={Weaver, Lex and Tao, Nigel},
  booktitle={Proceedings of the Seventeenth conference on Uncertainty in artificial intelligence},
  pages={538--545},
  year={2001}
}
@inproceedings{precup2001off,
  title={Off-policy temporal-difference learning with function approximation},
  author={Precup, Doina and Sutton, Richard S and Dasgupta, Sanjoy},
  booktitle={Proceedings of the 18th International Conference on Machine Learning},
  pages={417--424},
  year={2001}
}
@article{geramifard2013tutorial,
  title={A tutorial on linear function approximators for dynamic programming and reinforcement learning},
  author={Geramifard, Alborz and Walsh, Thomas J and Tellex, Stefanie and Chowdhary, Girish and Roy, Nicholas and How, Jonathan P and others},
  journal={Foundations and Trends{\textregistered} in Machine Learning},
  volume={6},
  number={4},
  pages={375--451},
  year={2013},
  publisher={Now Publishers, Inc.}
}
@inproceedings{tiwari2019natural,
  title={Natural option critic},
  author={Tiwari, Saket and Thomas, Philip S},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={33},
  pages={5175--5182},
  year={2019}
}
@inproceedings{bacon2017option,
  title={The option-critic architecture},
  author={Bacon, Pierre-Luc and Harb, Jean and Precup, Doina},
  booktitle={Proceedings of the Thirty-First AAAI Conference on Artificial Intelligence},
  pages={1726--1734},
  year={2017}
}
@article{daniel2016probabilistic,
  title={Probabilistic inference for determining options in reinforcement learning},
  author={Daniel, Christian and Van Hoof, Herke and Peters, Jan and Neumann, Gerhard},
  journal={Machine Learning},
  volume={104},
  number={2-3},
  pages={337--357},
  year={2016},
  publisher={Springer}
}
@inproceedings{da2012learning,
  title={Learning parameterized skills},
  author={Da Silva, Bruno Castro and Konidaris, George and Barto, Andrew G},
  booktitle={Proceedings of the 29th International Coference on International Conference on Machine Learning},
  pages={1443--1450},
  year={2012}
}
@inproceedings{konidaris2009skill,
  title={Skill discovery in continuous reinforcement learning domains using skill chaining},
  author={Konidaris, George and Barto, Andrew G},
  booktitle={Advances in neural information processing systems},
  pages={1015--1023},
  year={2009}
}
@article{sutton1988learning,
  title={Learning to predict by the methods of temporal differences},
  author={Sutton, Richard S},
  journal={Machine learning},
  volume={3},
  number={1},
  pages={9--44},
  year={1988},
  publisher={Springer}
}

@inproceedings{hoffman2011regularized,
  title={Regularized least squares temporal difference learning with nested $l_2$ and $l_1$ penalization},
  author={Hoffman, Matthew W and Lazaric, Alessandro and Ghavamzadeh, Mohammad and Munos, R{\'e}mi},
  booktitle={European Workshop on Reinforcement Learning},
  pages={102--114},
  year={2011},
  organization={Springer}
}

@article{boyan2002technical,
  title={Technical update: Least-squares temporal difference learning},
  author={Boyan, Justin A},
  journal={Machine learning},
  volume={49},
  number={2-3},
  pages={233--246},
  year={2002},
  publisher={Springer}
}

@article{wu2017general,
  title={基于一般化斜投影的异策略时序差分学习算法},
  author={吴毓双 and 陈筱语 and 马静雯 and 陈兴国 and others},
  journal={南京大学学报 (自然科学版)},
  number={2017 年 06},
  pages={1052--1062},
  year={2017},
  publisher={南京大学}
}

@article{choi2006generalized,
  title={A generalized Kalman filter for fixed point approximation and efficient temporal-difference learning},
  author={Choi, David and Van Roy, Benjamin},
  journal={Discrete Event Dynamic Systems},
  volume={16},
  number={2},
  pages={207--239},
  year={2006},
  publisher={Springer}
}

@inproceedings{precup2000eligibility,
  title={Eligibility Traces for Off-Policy Policy Evaluation},
  author={Precup, Doina and Sutton, Richard S and Singh, Satinder P},
  booktitle={Proceedings of the 17th International Conference on Machine Learning},
  pages={759--766},
  year={2000}
}
@article{yu2018generalized,
  title={On generalized bellman equations and temporal-difference learning},
  author={Yu, Huizhen and Mahmood, A Rupam and Sutton, Richard S},
  journal={The Journal of Machine Learning Research},
  volume={19},
  number={1},
  pages={1864--1912},
  year={2018},
  publisher={JMLR. org}
}
@inproceedings{niu2018generalized,
  title={Generalized value iteration networks: Life beyond lattices},
  author={Niu, Sufeng and Chen, Siheng and Guo, Hanyu and Targonski, Colin and Smith, Melissa C and Kova{\v{c}}evi{\'c}, Jelena},
  booktitle={Thirty-Second AAAI Conference on Artificial Intelligence},
  year={2018}
}
@article{goyal2018graph,
  title={Graph embedding techniques, applications, and performance: A survey},
  author={Goyal, Palash and Ferrara, Emilio},
  journal={Knowledge-Based Systems},
  volume={151},
  pages={78--94},
  year={2018},
  publisher={Elsevier}
}
@article{cai2018comprehensive,
  title={A comprehensive survey of graph embedding: Problems, techniques, and applications},
  author={Cai, Hongyun and Zheng, Vincent W and Chang, Kevin Chen-Chuan},
  journal={IEEE Transactions on Knowledge and Data Engineering},
  volume={30},
  number={9},
  pages={1616--1637},
  year={2018},
  publisher={IEEE}
}
@inproceedings{niepert2016learning,
  title={Learning convolutional neural networks for graphs},
  author={Niepert, Mathias and Ahmed, Mohamed and Kutzkov, Konstantin},
  booktitle={International conference on machine learning},
  pages={2014--2023},
  year={2016}
}
@inproceedings{oh2015action,
  title={Action-conditional video prediction using deep networks in atari games},
  author={Oh, Junhyuk and Guo, Xiaoxiao and Lee, Honglak and Lewis, Richard L and Singh, Satinder},
  booktitle={Advances in neural information processing systems},
  pages={2863--2871},
  year={2015}
}
@article{lecun2015deep,
  title={Deep learning},
  author={LeCun, Yann and Bengio, Yoshua and Hinton, Geoffrey},
  journal={nature},
  volume={521},
  number={7553},
  pages={436},
  year={2015},
  publisher={Nature Publishing Group}
}

@article{Silver2017Mastering,
  title={Mastering the game of Go without human knowledge},
  author={Silver, D and Schrittwieser, J and Simonyan, K and Antonoglou, I and Huang, A. and Guez, A and Hubert, T and Baker, L and Lai, M. and Bolton, A},
  journal={Nature},
  volume={550},
  number={7676},
  pages={354-359},
  year={2017},
}
@article{bertsekas2018feature,
  title={Feature-based aggregation and deep reinforcement learning: A survey and some new implementations},
  author={Bertsekas, Dimitri P},
  journal={IEEE/CAA Journal of Automatica Sinica},
  volume={6},
  number={1},
  pages={1--31},
  year={2018},
  publisher={IEEE}
}

@article{johns2009hybrid,
  title={Hybrid least-squares algorithms for approximate policy evaluation},
  author={Johns, Jeff and Petrik, Marek and Mahadevan, Sridhar},
  journal={Machine learning},
  volume={76},
  number={2-3},
  pages={243--256},
  year={2009},
  publisher={Springer}
}
@inproceedings{tsitsiklis1997analysis,
  title={Analysis of temporal-diffference learning with function approximation},
  author={Tsitsiklis, John N and Van Roy, Benjamin},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1075--1081},
  year={1997}
}
@book{Kushner1997stochastic,
  author =	 {H. J. Kushner and G. G. Yin},
  title =	 {Stochastic Approximation Algorithms and Applications},
  year =	 {1997},
  publisher =	 {Springer-Verlag}
}
@article{devraj2017fastest,
  title={Fastest Convergence for Q-learning},
  author={Devraj, Adithya M and Meyn, Sean P},
  journal={arXiv preprint arXiv:1707.03770},
  year={2017}
}
@inproceedings{azar2011speedy,
  title={Speedy Q-learning},
  author={Azar, Mohammad Gheshlaghi and Munos, Remi and Ghavamzadeh, Mohammad and Kappen, Hilbert J},
  booktitle={Proceedings of the 24th International Conference on Neural Information Processing Systems},
  pages={2411--2419},
  year={2011},
  organization={Curran Associates Inc.}
}
@inproceedings{liu2016proximal,
  title={Proximal Gradient Temporal Difference Learning Algorithms.},
  author={Liu, Bo and Liu, Ji and Ghavamzadeh, Mohammad and Mahadevan, Sridhar and Petrik, Marek},
  booktitle={Proceedings of the International Joint Conference on Artificial Intelligence},
  pages={4195--4199},
  year={2016}
}
@inproceedings{ghiassian2020gradient,
  title={Gradient temporal-difference learning with regularized corrections},
  author={Ghiassian, Sina and Patterson, Andrew and Garg, Shivam and Gupta, Dhawal and White, Adam and White, Martha},
  booktitle={International Conference on Machine Learning},
  pages={3524--3534},
  year={2020},
  organization={PMLR}
}

@phdthesis{hackman2012faster,
  title={Faster Gradient-TD Algorithms},
  author={Hackman, Leah M},
  year={2012},
  school={University of Alberta}
}
@inproceedings{liu2015finite,
  title={Finite-sample analysis of proximal gradient TD algorithms},
  author={Liu, Bo and Liu, Ji and Ghavamzadeh, Mohammad and Mahadevan, Sridhar and Petrik, Marek},
  booktitle={Proceedings of the 21st Conference on Uncertainty in Artificial Intelligence},
  pages={504--513},
  year={2015}
}
@article{liu2018proximal,
  title={Proximal gradient temporal difference learning: Stable reinforcement learning with polynomial sample complexity},
  author={Liu, Bo and Gemp, Ian and Ghavamzadeh, Mohammad and Liu, Ji and Mahadevan, Sridhar and Petrik, Marek},
  journal={Journal of Artificial Intelligence Research},
  volume={63},
  pages={461--494},
  year={2018}
}
@inproceedings{melo2007convergence,
  title={Convergence of Q-learning with linear function approximation},
  author={Melo, Francisco S and Ribeiro, M Isabel},
  booktitle={Control Conference (ECC), 2007 European},
  pages={2671--2678},
  year={2007},
  organization={IEEE}
}
@inproceedings{leng2007convergence,
  title={Convergence analysis on approximate reinforcement learning},
  author={Leng, Jinsong and Jain, Lakhmi and Fyfe, Colin},
  booktitle={KSEM},
  pages={85--91},
  year={2007},
  organization={Springer}
}
@article{beggs2005convergence,
  title={On the convergence of reinforcement learning},
  author={Beggs, Alan W},
  journal={Journal of Economic Theory},
  volume={122},
  number={1},
  pages={1--36},
  year={2005},
  publisher={Elsevier}
}
@article{konda2004convergence,
  title={Convergence rate of linear two-time-scale stochastic approximation},
  author={Konda, Vijay R and Tsitsiklis, John N},
  journal={Annals of applied probability},
  pages={796--819},
  year={2004},
  publisher={JSTOR}
}
@article{borkar2000ode,
  title={The ODE method for convergence of stochastic approximation and reinforcement learning},
  author={Borkar, Vivek S and Meyn, Sean P},
  journal={SIAM Journal on Control and Optimization},
  volume={38},
  number={2},
  pages={447--469},
  year={2000},
  publisher={SIAM}
}
@article{even2003learning,
  title={Learning rates for Q-learning},
  author={Even-Dar, Eyal and Mansour, Yishay},
  journal={Journal of Machine Learning Research},
  volume={5},
  number={Dec},
  pages={1--25},
  year={2003}
}
@inproceedings{papavassiliou1999convergence,
  title={Convergence of reinforcement learning with general function approximators},
  author={Papavassiliou, Vassilis A and Russell, Stuart},
  booktitle={Proceedings of the International Joint Conference on Artificial Intelligence},
  pages={748--757},
  year={1999}
}

@inproceedings{szepesvari1998asymptotic,
  title={The asymptotic convergence-rate of Q-learning},
  author={Szepesv{\'a}ri, Csaba},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1064--1070},
  year={1998}
}
@article{borkar1997stochastic,
  title={Stochastic approximation with two time scales},
  author={Borkar, Vivek S},
  journal={Systems \& Control Letters},
  volume={29},
  number={5},
  pages={291--294},
  year={1997},
  publisher={Elsevier}
}

@inproceedings{jaakkola1994convergence,
  title={Convergence of stochastic iterative dynamic programming algorithms},
  author={Jaakkola, Tommi and Jordan, Michael I and Singh, Satinder P},
  booktitle={Advances in neural information processing systems},
  pages={703--710},
  year={1994}
}
@article{watkins1992q,
  title={Q-learning},
  author={Watkins, Christopher JCH and Dayan, Peter},
  journal={Machine learning},
  volume={8},
  number={3-4},
  pages={279--292},
  year={1992},
  publisher={Springer}
}
@inproceedings{omidshafiei2018crossmodal,
  title={Crossmodal attentive skill learner},
  author={Omidshafiei, Shayegan and Kim, Dong-Ki and Pazis, Jason and How, Jonathan P},
  booktitle={Proceedings of the 17th International Conference on Autonomous Agents and MultiAgent Systems},
  pages={139--146},
  year={2018},
  organization={International Foundation for Autonomous Agents and Multiagent Systems}
}
@inproceedings{akrour2018regularizing,
  title={Regularizing reinforcement learning with state abstraction},
  author={Akrour, Riad and Veiga, Filipe and Peters, Jan and Neumann, Gerhard},
  booktitle={2018 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},
  pages={534--539},
  year={2018},
  organization={IEEE}
}
@inproceedings{kulkarni2016hierarchical,
  title={Hierarchical deep reinforcement learning: Integrating temporal abstraction and intrinsic motivation},
  author={Kulkarni, Tejas D and Narasimhan, Karthik and Saeedi, Ardavan and Tenenbaum, Josh},
  booktitle={Advances in neural information processing systems},
  pages={3675--3683},
  year={2016}
}

@article{xu2013online,
  title={Online learning control using adaptive critic designs with sparse kernel machines},
  author={Xu, Xin and Hou, Zhongsheng and Lian, Chuanqiang and He, Haibo},
  journal={IEEE Trans. Neural Netw. Learn. Syst.},
  volume={24},
  number={5},
  pages={762--775},
  year={2013},
  publisher={IEEE}
}
@article{bertsekas2017value,
  title={Value and policy iterations in optimal control and adaptive dynamic programming},
  author={Bertsekas, Dimitri P},
  journal={IEEE Trans. Neural Netw. Learn. Syst.},
  year={2017},
  volume={28},
  number={3},
  pages={500 - 509},
  publisher={IEEE}
}

@inproceedings{harutyunyan2015multi,
  title={Multi-scale reward shaping via an off-policy ensemble},
  author={Harutyunyan, Anna and Brys, Tim and Vrancx, Peter and Now{\'e}, Ann},
  booktitle={Proc. 2015 Int. Conf. Autonomous Agents and Multiagent Systems},
  pages={1641--1642},
  year={2015},
  organization={International Foundation for Autonomous Agents and Multiagent Systems}
}
@inproceedings{harutyunyan2015expressing,
  title={Expressing Arbitrary Reward Functions as Potential-Based Advice.},
  author={Harutyunyan, Anna and Devlin, Sam and Vrancx, Peter and Now{\'e}, Ann},
  booktitle={AAAI},
  pages={2652--2658},
  year={2015}
}
@article{wiewiora2003potential,
  title={Potential-based shaping and Q-value initialization are equivalent},
  author={Wiewiora, Eric},
  journal={J. Artif. Intell. Res.},
  volume={19},
  pages={205--208},
  year={2003}
}
@article{grzes2010online,
  title={Online learning of shaping rewards in reinforcement learning},
  author={Grze{\'s}, Marek and Kudenko, Daniel},
  journal={Neural Netw.},
  volume={23},
  number={4},
  pages={541--550},
  year={2010},
  publisher={Elsevier}
}
@inproceedings{marthi2007automatic,
  title={Automatic shaping and decomposition of reward functions},
  author={Marthi, Bhaskara},
  booktitle={Proc. 24th Int. Conf. Mach. Learn.},
  pages={601--608},
  year={2007}
}
@inproceedings{laud2003influence,
  title={The Influence of Reward on the Speed of Reinforcement Learning: An Analysis of Shaping},
  author={Laud, Adam and Dejong, Gerald},
  booktitle={Proc. 20th Int. Conf. Mach. Learn.},
  pages={440--447},
  year={2003}
}
@phdthesis{laud2004theory,
  title={Theory and application of reward shaping in reinforcement learning},
  author={Laud, Adam Daniel},
  year={2004},
  school={University of Illinois at Urbana-Champaign}
}
@article{geist2013algorithmic,
  title={Algorithmic survey of parametric value function approximation},
  author={Geist, Matthieu and Pietquin, Olivier},
  journal={IEEE Trans. Neural Netw. Learn. Syst.},
  volume={24},
  number={6},
  pages={845--867},
  year={2013},
  publisher={IEEE}
}
@article{furmston2016approximate,
  title={Approximate Newton Methods for Policy Search in Markov Decision Processes},
  author={Furmston, Thomas and Lever, Guy and Barber, David},
  journal={J. Mach. Learn. Res.},
  volume={17},
  number={227},
  pages={1--51},
  year={2016}
}
@article{silver2016mastering,
  title={Mastering the game of Go with deep neural networks and tree search},
  author={Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and van den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and others},
  journal={Nature},
  volume={529},
  number={7587},
  pages={484--489},
  year={2016},
  publisher={Nature Publishing Group}
}

@article{zheng2020double,
  title={基于双估计器的改进 Speedy Q-learning 算法},
  author={郑帅 and 罗飞 and 顾春华 and 丁炜超 and 卢海峰},
  journal={计算机科学},
  volume={47},
  number={7},
  pages={179--185},
  year = {2020}
}
@article{mnih2015human,
  title={Human-level control through deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Rusu, Andrei A and Veness, Joel and Bellemare, Marc G and Graves, Alex and Riedmiller, Martin and Fidjeland, Andreas K and Ostrovski, Georg and others},
  journal={Nature},
  volume={518},
  number={7540},
  pages={529--533},
  year={2015},
  publisher={Nature Publishing Group}
}
@inproceedings{guo2014deep,
  title={Deep learning for real-time Atari game play using offline Monte-Carlo tree search planning},
  author={Guo, Xiaoxiao and Singh, Satinder and Lee, Honglak and Lewis, Richard L and Wang, Xiaoshi},
  booktitle={Advances in Neural Information Processing Systems},
  pages={3338--3346},
  publisher={Cambridge, MA: MIT Press},
  year={2014}
}
@inproceedings{scherrer2010should,
  title={Should one compute the Temporal Difference fix point or minimize the Bellman Residual? The unified oblique projection view},
  author={Scherrer, Bruno},
  booktitle={Proceedings of the 27th International Conference on Machine Learning},
  pages={959--966},
  year={2010}
}
@article{hirsch1989convergent,
  title={Convergent activation dynamics in continuous time networks},
  author={Hirsch, Morris W},
  journal={Neural Netw.},
  volume={2},
  number={5},
  pages={331--349},
  year={1989},
  publisher={Elsevier}
}

@article{ortner2013adaptive,
  title={Adaptive aggregation for reinforcement learning in average reward Markov decision processes},
  author={Ortner, Ronald},
  journal={Annals Oper. Res.},
  volume={208},
  number={1},
  pages={321--336},
  year={2013},
  publisher={Springer}
}
@article{jaksch2010near,
  title={Near-optimal regret bounds for reinforcement learning},
  author={Jaksch, Thomas and Ortner, Ronald and Auer, Peter},
  journal={J. Mach. Learn. Res.},
  volume={11},
  number={Apr},
  pages={1563--1600},
  year={2010}
}
@article{ortner2007logarithmic,
  title={Logarithmic online regret bounds for undiscounted reinforcement learning},
  author={Ortner, P and Auer, R},
  journal={Advances in Neural Information Processing Systems},
  publisher={Cambridge, MA: MIT Press},
  volume={19},
  pages={49},
  year={2007}
}
@article{das1999solving,
  title={Solving semi-Markov decision problems using average reward reinforcement learning},
  author={Das, Tapas K and Gosavi, Abhijit and Mahadevan, Sridhar and Marchalleck, Nicholas},
  journal={Management Science},
  volume={45},
  number={4},
  pages={560--574},
  year={1999},
  publisher={INFORMS}
}
@article{abounadi2001learning,
  title={Learning algorithms for Markov decision processes with average cost},
  author={Abounadi, Jinane and Bertsekas, D and Borkar, Vivek S},
  journal={SIAM J. Control Optim.},
  volume={40},
  number={3},
  pages={681--698},
  year={2001},
  publisher={SIAM}
}
@inproceedings{singh1994reinforcement,
  title={Reinforcement learning algorithms for average-payoff Markovian decision processes},
  author={Singh, Satinder P},
  booktitle={AAAI},
  volume={94},
  pages={700--705},
  year={1994}
}
@inproceedings{schwartz1993reinforcement,
  title={A reinforcement learning method for maximizing undiscounted rewards},
  author={Schwartz, Anton},
  booktitle={Proc. 10th Int. Conf. Mach. Learn.},
  volume={298},
  pages={298--305},
  year={1993}
}
@inproceedings{novati2019remember,
  title={Remember and Forget for Experience Replay},
  author={Novati, Guido and Koumoutsakos, Petros},
  booktitle={International Conference on Machine Learning},
  pages={4851--4860},
  year={2019}
}
@article{lagoudakis2003least,
  title={Least-squares policy iteration},
  author={Lagoudakis, Michail G and Parr, Ronald},
  journal={Journal of machine learning research},
  volume={4},
  number={Dec},
  pages={1107--1149},
  year={2003}
}
@inproceedings{yu2010convergence,
  title={Convergence of Least Squares Temporal Difference Methods Under General Conditions.},
  author={Yu, Huizhen},
  booktitle={ICML},
  pages={1207--1214},
  year={2010}
}
@article{yang2019expected,
  title={Expected Sarsa ($$\backslash$lambda $) with Control Variate for Variance Reduction},
  author={Yang, Long and Zhang, Yu},
  journal={arXiv preprint arXiv:1906.11058},
  year={2019}
}
@inproceedings{anschel2017averaged,
  title={Averaged-{DQN}: Variance reduction and stabilization for deep reinforcement learning},
  author={Anschel, Oron and Baram, Nir and Shimkin, Nahum},
  booktitle={Proceedings of the 34th International Conference on Machine Learning},
  pages={176--185},
  year={2017}
}
@inproceedings{hasselt2016deep,
  title={Deep reinforcement learning with double Q-Learning},
  author={Hasselt, Hado van and Guez, Arthur and Silver, David},
  booktitle={Proceedings of the Thirtieth AAAI Conference on Artificial Intelligence},
  pages={2094--2100},
  year={2016}
}
@inproceedings{hasselt2010double,
  title={Double Q-learning},
  author={Hasselt, Hado V},
  booktitle={Advances in neural information processing systems},
  pages={2613--2621},
  year={2010}
}
@inproceedings{lee2019target,
  title={Target-Based Temporal-Difference Learning},
  author={Lee, Donghwan and He, Niao},
  booktitle={International Conference on Machine Learning},
  pages={3713--3722},
  year={2019}
}
@inproceedings{korda2015td,
  title={On TD (0) with function approximation: Concentration bounds and a centered variant with exponential convergence},
  author={Korda, Nathaniel and La, Prashanth},
  booktitle={International Conference on Machine Learning},
  pages={626--634},
  year={2015}
}
@article{xu2020reanalysis,
  title={Reanalysis of Variance Reduced Temporal Difference Learning},
  author={Xu, Tengyu and Wang, Zhe and Zhou, Yi and Liang, Yingbin},
  journal={ICLR},
  year={2020}
}
@inproceedings{yang2018optimal,
  title={An Optimal Algorithm for the Stochastic Bandits with Knowing Near-optimal Mean Reward},
  author={Yang, Shangdong and Wang, Hao and Gao, Yang and Chen, Xingguo},
  booktitle={Proceedings of the 17th International Conference on Autonomous Agents and MultiAgent Systems},
  pages={2130--2132},
  year={2018},
  organization={International Foundation for Autonomous Agents and Multiagent Systems}
}
@inproceedings{yang2016efficient,
  title={Efficient Average Reward Reinforcement Learning Using Constant Shifting Values},
  author={Yang, Shangdong and Gao, Yang and An, Bo and Wang, Hao and Chen, Xingguo},
  booktitle={Thirtieth AAAI Conference on Artificial Intelligence},
  pages={2258-2264},
  year={2016}
}
@inproceedings{devlin2012dynamic,
  title={Dynamic potential-based reward shaping},
  author={Devlin, Sam and Kudenko, Daniel},
  booktitle={Proc. 11th Int. Conf. Autonomous Agents and Multiagent Systems},
  pages={433--440},
  year={2012}
}
@phdthesis{maei2011gradient,
  title={Gradient temporal-difference learning algorithms},
  author={Maei, Hamid Reza},
  year={2011},
  school={University of Alberta}
}
@phdthesis{baird1999reinforcement,
  title={Reinforcement learning through gradient descent},
  author={Baird III, Leemon C},
  year={1999},
  school={US Air Force Academy, US}
}
@article{tsitsiklis1996feature,
  title={Feature-based methods for large scale dynamic programming},
  author={Tsitsiklis, John N and Van Roy, Benjamin},
  journal={Mach. Learn.},
  volume={22},
  number={1-3},
  pages={59--94},
  year={1996},
  publisher={Springer}
}
@inproceedings{chen2009apply,
  title={Apply ant colony optimization to Tetris},
  author={Chen, X. and Wang, H. and Wang, W. and Shi, Y. and Gao, Y.},
  booktitle={Proceedings of the 11th Annual Conference on Genetic and Evolutionary Computation (GECCO)},
  pages={1741--1742},
  year={2009},
  organization={ACM}
}
@incollection{farias2006tetris,
  title={Tetris: A study of randomized constraint sampling},
  author={Farias, Vivek F and Van Roy, Benjamin},
  booktitle={Probabilistic and Randomized Methods for Design Under Uncertainty},
  pages={189--201},
  year={2006},
  publisher={Springer}
}
@article{bertsekas1996temporal,
  title={Temporal differences-based policy iteration and applications in neuro-dynamic programming},
  author={Bertsekas, Dimitri P and Ioffe, Sergey},
  journal={Lab. for Info. and Decision Systems Report LIDS-P-2349, MIT, Cambridge, MA},
  year={1996},
  publisher={Citeseer}
}
@inproceedings{kakade2001natural,
  title={A Natural Policy Gradient.},
  author={Kakade, Sham},
  booktitle={Advances in Neural Information Processing Systems},
  publisher={Cambridge, MA: MIT Press},
  volume={14},
  pages={1531--1538},
  year={2001}
}
@article{peters2008natural,
  title={Natural actor-critic},
  author={Peters, Jan and Schaal, Stefan},
  journal={Neurocomputing},
  volume={71},
  number={7},
  pages={1180--1190},
  year={2008},
  publisher={Elsevier}
}
@article{baxter2001infinite,
  title={Infinite-horizon policy-gradient estimation},
  author={Baxter, Jonathan and Bartlett, Peter L.},
  journal={J. Artif. Intell. Res.},
  pages={319--350},
  year={2001}
}
@inproceedings{sutton1999policy,
  title={Policy Gradient Methods for Reinforcement Learning with Function Approximation.},
  author={Sutton, Richard S and McAllester, David A and Singh, Satinder P and Mansour, Yishay and others},
  booktitle={Advances in Neural Information Processing Systems},
  publisher={Cambridge, MA: MIT Press},
  pages={1057--1063},
  year={1999}
}
@inproceedings{bohm2005evolutionary,
  title={An evolutionary approach to tetris},
  author={B{\"o}hm, Niko and K{\'o}kai, Gabriella and Mandl, Stefan},
  booktitle={Proc. 6th Metaheuristics Int. Conf.},
  pages={137-148},
  year={2005}
}
@article{szita2006learning,
  title={Learning Tetris using the noisy cross-entropy method},
  author={Szita, Istv{\'a}n and L{\"o}rincz, Andr{\'a}s},
  journal={Neural Comput.},
  volume={18},
  number={12},
  pages={2936--2941},
  year={2006},
  publisher={MIT Press}
}
@inproceedings{thiery2010least,
  title={Least-Squares $\lambda$ Policy Iteration: Bias-Variance Trade-off in Control Problems},
  author={Thiery, Christophe and Scherrer, Bruno},
  booktitle={Proc. 27th Int. Conf. Mach. Learn.},
  pages={1071--1078},
  year={2010}
}

@inproceedings{gabillon2013approximate,
  title={Approximate dynamic programming finally performs well in the game of Tetris},
  author={Gabillon, Victor and Ghavamzadeh, Mohammad and Scherrer, Bruno},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1754--1762},
  year={2013}
}
@article{scherrer2013performance,
  title={Performance bounds for $\lambda$ policy iteration and application to the game of Tetris},
  author={Scherrer, Bruno},
  journal={J. Mach. Learn. Res.},
  volume={14},
  number={1},
  pages={1181--1227},
  year={2013},
  publisher={JMLR. org}
}
@article{thiery2009improvements,
  title={Improvements on Learning Tetris with Cross Entropy},
  author={Thiery, Christophe and Scherrer, Bruno},
  journal={International Computer Games Association Journal},
  volume={32},
  number={1},
  pages={23--33},
  year={2009}
}
@article{scherrer2015approximate,
  title={Approximate Modified Policy Iteration and its Application to the Game of Tetris},
  author={Scherrer, Bruno and Ghavamzadeh, Mohammad and Gabillon, Victor and Lesner, Boris and Geist, Matthieu},
  journal={Journal of Machine Learning Research},
  volume={16},
  pages={1629--1676},
  year={2015}
}

@MASTERSTHESIS{Brzustowski1992,
  author ={John Brzustowski},
  title ={Can you win at tetris?},
  school = {University of British Columbia},
  year ={1992}
}
@Article{Breukelaar04,
  author =	 {Ron Breukelaar and Erik D. Demaine and Susan Hohenberger and Hendrik Jan Hoogeboom and Walter A. Kosters and David Liben-Nowell},
  title =	 {Tetris is Hard, Even to Approximate},
  journal =	 {International Journal of Computational Geometry and Applications},
  year =	 {2004},
  volume =	 {14},
  number =	 {1--2},
  pages =	 {41--68},
  month =	 {April},
}

@inproceedings{devraj2017zap,
  title={Zap Q-learning},
  author={Devraj, Adithya M and Meyn, Sean P},
  booktitle={Proceedings of the 31st International Conference on Neural Information Processing Systems},
  pages={2232--2241},
  year={2017}
}
@inproceedings{gehring2016incremental,
  title={Incremental truncated LSTD},
  author={Gehring, Clement and Pan, Yangchen and White, Martha},
  booktitle={Proceedings of the Twenty-Fifth International Joint Conference on Artificial Intelligence},
  pages={1505--1511},
  year={2016}
}
@article{ueno2011generalized,
  title={Generalized TD Learning},
  author={Ueno, Tsuyoshi and Maeda, Shin-ichi and Kawanabe, Motoaki and Ishii, Shin},
  journal={The Journal of Machine Learning Research},
  volume={12},
  pages={1977--2020},
  year={2011},
  publisher={JMLR. org}
}

@inproceedings{geist2011,
  title={l1-penalized projected bellman residual},
  author={Geist, Matthieu and Scherrer, Bruno},
  booktitle={European Workshop on Reinforcement Learning},
  pages={89--101},
  year={2011},
  organization={Springer}
}
@inproceedings{osogami2020uncorrected,
  title={Uncorrected Least-Squares Temporal Difference with Lambda-Return},
  author={Osogami, Takayuki},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={34},
  number={04},
  pages={5323--5330},
  year={2020}
}
@mastersthesis{amundsen2014comparison,
  title={A comparison of feature functions for Tetris strategies},
  author={Amundsen, Jonas Balgaard},
  year={2014},
  school={Institutt for datateknikk og informasjonsvitenskap}
}
@inproceedings{vanseijen2015deeper,
  title={A deeper look at planning as learning from replay},
  author={Vanseijen, Harm and Sutton, Rich},
  booktitle={International conference on machine learning},
  pages={2314--2322},
  year={2015},
  organization={PMLR}
}
@inproceedings{van2014off,
  title={Off-policy TD ($\lambda$) with a true online equivalence},
  author={van Hasselt, Hado and Mahmood, A Rupam and Sutton, Richard S},
  booktitle={Proceedings of the 30th Conference on Uncertainty in Artificial Intelligence, Quebec City, Canada},
  pages={330--339},
  year={2014}
}
@article{van2016true,
  title={True online temporal-difference learning},
  author={Van Seijen, Harm and Mahmood, A Rupam and Pilarski, Patrick M and Machado, Marlos C and Sutton, Richard S},
  journal={The Journal of Machine Learning Research},
  volume={17},
  number={1},
  pages={5057--5096},
  year={2016},
  publisher={JMLR. org}
}
@inproceedings{seijen2014true,
  title={True online TD (lambda)},
  author={Seijen, Harm and Sutton, Rich},
  booktitle={International Conference on Machine Learning},
  pages={692--700},
  year={2014},
  organization={PMLR}
}
@article{nedic2003least,
  title={Least squares policy evaluation algorithms with linear function approximation},
  author={Nedi{\'c}, A and Bertsekas, Dimitri P},
  journal={Discrete Event Dynamic Systems},
  volume={13},
  number={1},
  pages={79--110},
  year={2003},
  publisher={Springer}
}
@book{Bertsekas1996,
  author =	 {Bertsekas, D. and Tsitsiklis, J. N.},
  title =	 {Neuro-Dynamic Programming},
  year =	 {1996},
  publisher =	 {Athena Scientific},
}
@techreport{liu2011compressive,
  title={Compressive reinforcement learning with oblique random projections},
  author={Liu, Bo and Mahadevan, Sridhar},
  year={2011},
  institution={Technical Report UM-CS-2011-024}
}
@inproceedings{maei2010gq,
  title={GQ ($\lambda$): A general gradient algorithm for temporal-difference prediction learning with eligibility traces},
  author={Maei, Hamid Reza and Sutton, Richard S},
  booktitle={Proceedings of the Third Conference on Artificial General Intelligence},
  volume={1},
  pages={91--96},
  year={2010}
}
@inproceedings{maei2010toward,
  title={Toward off-policy learning control with function approximation},
  author={Maei, Hamid R and Szepesv{\'a}ri, Csaba and Bhatnagar, Shalabh and Sutton, Richard S},
  booktitle={Proc. 27th Int. Conf. Mach. Learn.},
  pages={719--726},
  year={2010}
}
@inproceedings{phua2007tracking,
  title={Tracking value function dynamics to improve reinforcement learning with piecewise linear function approximation},
  author={Phua, Chee Wee and Fitch, Robert},
  booktitle={Proc. 24th Int. Conf. Mach. Learn.},
  pages={751--758},
  year={2007},
  organization={ACM}
}
@inproceedings{krawiec2011learning,
  title={Learning n-tuple networks for Othello by coevolutionary gradient search},
  author={Krawiec, Krzysztof and Szubert, Marcin Grzegorz},
  booktitle={Proceedings of the 13th annual conference on Genetic and evolutionary computation},
  address={Dublin, Ireland},
  month={February},
  pages={355--362},
  year={2011}
}
@inproceedings{szubert2014temporal,
  title={Temporal difference learning of N-tuple networks for the game 2048},
  author={Szubert, Marcin and Jaskowski, Wojciech},
  booktitle={2014 IEEE Conference on Computational Intelligence and Games (CIG)},
  pages={1--8},
  year={2014},
  organization={IEEE}
}
@article{song2015kernel,
  title={Kernel-based least squares temporal difference with gradient correction},
  author={Song, Tianheng and Li, Dazi and Cao, Liulin and Hirasawa, Kotaro},
  journal={IEEE transactions on neural networks and learning systems},
  volume={27},
  number={4},
  pages={771--782},
  year={2015},
  publisher={IEEE}
}
@article{chen2013online,
  title={Online Selective Kernel-based Temporal Differece Learning},
  author={Chen, Xingguo and Gao, Yang and Wang, Ruili},
  journal={IEEE Trans. Neural Netw. Learn. Syst.},
  year={2013},
  volume={24},
  number={12},
  pages={1944--1956},
  publisher={IEEE}
}

@article{xu2007kernel,
  title={Kernel-based least squares policy iteration for reinforcement learning},
  author={Xu, Xin and Hu, Dewen and Lu, Xicheng},
  journal={IEEE Trans. Neural Netw.},
  volume={18},
  number={4},
  pages={973--992},
  year={2007},
  publisher={IEEE}
}
@INPROCEEDINGS{Engel03bayesmeets,
    author = {Yaakov Engel and Shie Mannor and Ron Meir},
    title = {Bayes meets {B}ellman: the {G}aussian process approach to temporal difference learning},
    booktitle = {Proc. 20th Int. Conf. Mach. Learn.},
    year = {2003},
    pages = {154--161},
    address={Washington, DC},
    month={Aug.},
}
@inproceedings{robards2011sparse,
  title={Sparse Kernel-SARSA ($\lambda$) with an eligibility trace},
  author={Robards, M. and Sunehag, P. and Sanner, S. and Marthi, B.},
  booktitle = {Proc. 22nd Eur. Conf. Mach. Learn.},
  pages={1--17},
  year={2011},
  month={Sept.},
   address = {Athens, Greece},
}
@conference{reisinger2008online,
  title={{Online kernel selection for {B}ayesian reinforcement learning}},
  author={Reisinger, J. and Stone, P. and Miikkulainen, R.},
  booktitle={Proc. 25th Int. Conf. Mach. Learn.},
  pages={816--823},
  year={2008},
  month={July},
  address={ Helsinki, Finland},
}
@article{jaskowski2018mastering,
  title={Mastering 2048 With Delayed Temporal Coherence Learning, Multistage Weight Promotion, Redundant Encoding, and Carousel Shaping},
  author={Ja{\'s}kowski, Wojciech},
  journal={IEEE Transactions on Games},
  volume={10},
  number={1},
  pages={3--14},
  year={2018},
  publisher={IEEE}
}
@article{yeh2017multistage,
  title={Multistage temporal difference learning for 2048-like games},
  author={Yeh, Kun-Hao and Wu, I-Chen and Hsueh, Chu-Hsuan and Chang, Chia-Chuan and Liang, Chao-Chin and Chiang, Han},
  journal={IEEE Transactions on Computational Intelligence and AI in Games},
  volume={9},
  number={4},
  pages={369--380},
  year={2017},
  publisher={IEEE}
}
@incollection{wu2014multi,
  title={Multi-stage temporal difference learning for 2048},
  author={Wu, I-Chen and Yeh, Kun-Hao and Liang, Chao-Chin and Chang, Chia-Chuan and Chiang, Han},
  booktitle={Technologies and Applications of Artificial Intelligence},
  pages={366--378},
  year={2014},
  publisher={Springer}
}
@book{Sutton1998,
  title={{Reinforcement learning: an introduction}},
  author={Sutton, R.S. and Barto, A.G.},
  year={1998},
  edition = {First},
  publisher={MIT Press},
  address={Cambridge, MA}
}
@article{song2019recursive,
  title={Recursive Least-Squares Temporal Difference With Gradient Correction},
  author={Song, Tianheng and Li, Dazi and Yang, Weimin and Hirasawa, Kotaro},
  journal={IEEE transactions on cybernetics},
  year={2019},
  volume={51},
  issue={8},
  pages={4251 - 4264},
  publisher={IEEE}
}
@article{xu2002efficient,
  title={Efficient reinforcement learning using recursive least-squares methods},
  author={Xu, Xin and He, Han-gen and Hu, Dewen},
  journal={Journal of Artificial Intelligence Research},
  volume={16},
  pages={259--292},
  year={2002}
}
@inproceedings{zinkevich2007ilstd,
  title={iLSTD: Eligibility traces and convergence analysis},
  author={Zinkevich, Alborz Geramifard Michael Bowling Martin and Sutton, Richard S},
  booktitle={Advances in Neural Information Processing Systems 19: Proceedings of the 2006 Conference},
  volume={19},
  pages={441},
  year={2007},
  organization={MIT Press}
}
@phdthesis{Bradtke1994phd,
  title={Incremental Dynamic Programming for On-line Adaptive Optimal Control},
  author={Bradtke, Steven J},
  year={1994},
  school={University of  Massachusetts},
  month={Sept.},
  address={Amherst},
}
@inproceedings{baird1995residual,
  title={Residual algorithms: Reinforcement learning with function approximation},
  author={Baird, Leemon and others},
  booktitle={Proceedings of the 12th International Conference on Machine Learning},
  pages={30--37},
  year={1995}
}

@article{mahmood2017multi,
  title={Multi-step off-policy learning without importance sampling ratios},
  author={Mahmood, Ashique Rupam and Yu, Huizhen and Sutton, Richard S},
  journal={arXiv preprint arXiv:1702.03006},
  year={2017}
}
@inproceedings{munos2016safe,
  title={Safe and efficient off-policy reinforcement learning},
  author={Munos, R{\'e}mi and Stepleton, Tom and Harutyunyan, Anna and Bellemare, Marc},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1054--1062},
  year={2016}
}
@inproceedings{mahmood2015off,
  title={Off-policy learning based on weighted importance sampling with linear computational complexity.},
  author={Mahmood, Ashique Rupam and Sutton, Richard S},
  booktitle={UAI},
  pages={552--561},
  year={2015},
  organization={Citeseer}
}

@inproceedings{sutton2014new,
  title={A new Q (lambda) with interim forward view and Monte Carlo equivalence},
  author={Sutton, Rich and Mahmood, Ashique Rupam and Precup, Doina and Hasselt, Hado},
  booktitle={International Conference on Machine Learning},
  pages={568--576},
  year={2014}
}

@inproceedings{sutton2009fast,
  title={Fast gradient-descent methods for temporal-difference learning with linear function approximation},
  author={Sutton, R.S. and Maei, H.R. and Precup, D. and Bhatnagar, S. and Silver, D. and Szepesv{\'a}ri, C. and Wiewiora, E.},
  booktitle={Proceedings of the 26th International Conference on Machine Learning},
  pages={993--1000},
  year={2009}
}
@article{bradtke1996linear,
  title={Linear least-squares algorithms for temporal difference learning},
  author={Bradtke, Steven J and Barto, Andrew G},
  journal={Machine learning},
  volume={22},
  number={1-3},
  pages={33--57},
  year={1996},
  publisher={Springer}
}
@inproceedings{sutton2008convergent,
  title={A Convergent ${O}(n)$ Temporal-difference Algorithm for Off-policy Learning with Linear Function Approximation},
  author={Sutton, Richard S and Maei, Hamid R and Szepesv{\'a}ri, Csaba},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1609--1616},
  year={2008}
}
@inproceedings{dabney2014natural,
  title={Natural Temporal Difference Learning},
  author={Dabney, William and Thomas, Philip},
  booktitle={Twenty-Eighth AAAI Conference on Artificial Intelligence},
  year={2014}
}
@inproceedings{mahmood2014weighted,
  title={Weighted importance sampling for off-policy learning with linear function approximation},
  author={Mahmood, A Rupam and van Hasselt, Hado P and Sutton, Richard S},
  booktitle={Advances in Neural Information Processing Systems},
  publisher={Cambridge, MA: MIT Press},
  pages={3014--3022},
  year={2014}
}

@article{ormoneit2002kernel,
  title={{Kernel-based reinforcement learning}},
  author={Ormoneit, D. and Sen, {\'S}.},
  journal={Mach. Learn.},
  volume={49},
  number={2-3},
  pages={161--178},
  issn={0885-6125},
  year={2002},
  publisher={Springer-Verlag },
  address = {Hingham, MA, USA},
}
@inproceedings{Ghavamzadeh2010lstd,
  author = {M. Ghavamzadeh and A. Lazaric and O. A. Maillard and R. Munos},
  title = {{LSTD} with Random Projections},
  BOOKTITLE={Advances in Neural Information Processing Systems},
  publisher={Cambridge, MA: MIT Press},
  volume = {23},
  pages = {721--729},
  Address = {Lake Tahoe, Nevada, USA},
  year = {2010}
}
@inproceedings{loth2007sparse,
  title={Sparse temporal difference learning using LASSO},
  author={Loth, M. and Davy, M. and Preux, P.},
  booktitle={Proc. IEEE Symp. Approx. Dynamic Program. Reinforce. Learn.},
  pages={352--359},
  year={2007},
  organization={IEEE}
}

@inproceedings{Ghavamzadeh2011finite,
  author = {M. Ghavamzadeh and A. Lazaric and R. Munos and M. Hoffman},
  title = {Finite-Sample Analysis of {Lasso-TD}},
  booktitle = {Proc. 28th Int. Conf. Mach. Learn.},
  year = {2011},
  month= {June},
  address={Bellevue, Washington, USA},
  pages={1177--1184},
}
@article{yang2022online,
  title={Online Attentive Kernel-Based Temporal Difference Learning},
  author={Yang, Guang and Chen, Xingguo and Yang, Shangdong and Wang, Huihui and Dong, Shaokang and Gao, Yang},
  journal={arXiv preprint arXiv:2201.09065},
  year={2022}
}