@article{huang2023reinforcement,
  title={Reinforcement Learning in Low-Rank MDPs with Density Features},
  author={Huang, Audrey and Chen, Jinglin and Jiang, Nan},
  journal={arXiv preprint arXiv:2302.02252},
  year={2023}
}
@inproceedings{chen2022offline,
  title={Offline reinforcement learning under value and density-ratio realizability: the power of gaps},
  author={Chen, Jinglin and Jiang, Nan},
  booktitle={Uncertainty in Artificial Intelligence},
  pages={378--388},
  year={2022},
  organization={PMLR}
}
@inproceedings{jiang2022learning,
  title={Learning expected emphatic traces for deep {RL}},
  author={Jiang, Ray and Zhang, Shangtong and Chelu, Veronica and White, Adam and van Hasselt, Hado},
  booktitle={{Proceedings of the AAAI Conference on Artificial Intelligence}},
  year={2022}
}
@article{sutton2022alberta,
  title={The {Alberta Plan} for {AI} Research},
  author={Sutton, Richard S and Bowling, Michael H and Pilarski, Patrick M},
  journal={arXiv preprint arXiv:2208.11173},
  year={2022}
}
@inproceedings{zhan2022offline,
  title={Offline reinforcement learning with realizability and single-policy concentrability},
  author={Zhan, Wenhao and Huang, Baihe and Huang, Audrey and Jiang, Nan and Lee, Jason},
  booktitle={Conference on Learning Theory},
  pages={2730--2775},
  year={2022},
  organization={PMLR}
}
@article{zhang2022truncated,
  author  = {Shangtong Zhang and Shimon Whiteson},
  title   = {Truncated Emphatic Temporal Difference Methods for Prediction and Control},
  journal = {Journal of Machine Learning Research},
  year    = {2022},
  volume  = {23},
  number  = {153},
  pages   = {1--59},
}
@inproceedings{agarwal2021deep,
 author = {Agarwal, Rishabh and Schwarzer, Max and Castro, Pablo Samuel and Courville, Aaron C and Bellemare, Marc},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {29304--29320},
 title = {Deep Reinforcement Learning at the Edge of the Statistical Precipice},
 volume = {34},
 year = {2021}
}
@article{ghiassian2021empirical2,
  title={An empirical comparison of off-policy prediction learning algorithms in the four rooms environment},
  author={Ghiassian, Sina and Sutton, Richard S},
  journal={arXiv preprint arXiv:2109.05110},
  year={2021}
}
@inproceedings{guan2021per,
title={{PER}-{ETD}: A Polynomially Efficient Emphatic Temporal Difference Learning Method},
author={Ziwei Guan and Tengyu Xu and Yingbin Liang},
booktitle={International Conference on Learning Representations},
year={2022},
}
@inproceedings{lee2021optidice,
  title={{OptiDICE}: Offline policy optimization via stationary distribution correction estimation},
  author={Lee, Jongmin and Jeon, Wonseok and Lee, Byungjun and Pineau, Joelle and Kim, Kee-Eung},
  booktitle={International Conference on Machine Learning},
  pages={6120--6130},
  year={2021},
  organization={PMLR}
}
@inproceedings{zhang2020gendice,
title={{GenDICE}: Generalized Offline Estimation of Stationary Values},
author={Ruiyi Zhang and Bo Dai and Lihong Li and Dale Schuurmans},
booktitle={International Conference on Learning Representations},
year={2020},
}
@inproceedings{zhang2020gradientdice,
  title={{GradientDICE}: Rethinking generalized offline estimation of stationary values},
  author={Zhang, Shangtong and Liu, Bo and Whiteson, Shimon},
  booktitle={{International Conference on Machine Learning}},
  pages={11194--11203},
  year={2020},
  organization={PMLR}
}
@inproceedings{zhang2020provably,
  title={Provably convergent two-timescale off-policy actor-critic with function approximation},
  author={Zhang, Shangtong and Liu, Bo and Yao, Hengshuai and Whiteson, Shimon},
  booktitle={{International Conference on Machine Learning}},
  pages={11204--11213},
  year={2020},
  organization={PMLR}
}
@inproceedings{gelada2019off,
  title={Off-policy deep reinforcement learning by bootstrapping the covariate shift},
  author={Gelada, Carles and Bellemare, Marc G},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  year={2019}
}
@inproceedings{nachum2019dualdice,
 author = {Nachum, Ofir and Chow, Yinlam and Dai, Bo and Li, Lihong},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {},
 title = {{DualDICE}: Behavior-Agnostic Estimation of Discounted Stationary Distribution Corrections},
 volume = {32},
 year = {2019}
}
@inproceedings{liu2018breaking,
 author = {Liu, Qiang and Li, Lihong and Tang, Ziyang and Zhou, Dengyong},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {},
 title = {Breaking the Curse of Horizon: Infinite-Horizon Off-Policy Estimation},
 volume = {31},
 year = {2018}
}
@book{sutton2018reinforcement,
  title={Reinforcement Learning: An Introduction},
  author={Sutton, Richard S and Barto, Andrew G},
  year={2018},
  publisher={MIT press}
}
@inproceedings{hallak2017consistent,
  title={Consistent on-line off-policy evaluation},
  author={Hallak, Assaf and Mannor, Shie},
  booktitle={{International Conference on Machine Learning}},
  pages={1372--1383},
  year={2017},
  organization={PMLR}
}
@inproceedings{hallak2016generalized,
  title={Generalized emphatic temporal difference learning: Bias-variance analysis},
  author={Hallak, Assaf and Tamar, Aviv and Munos, R{\'e}mi and Mannor, Shie},
  booktitle={{Proceedings of the AAAI Conference on Artificial Intelligence}},
  year={2016}
}
@article{sutton2016emphatic,
  title={An emphatic approach to the problem of off-policy temporal-difference learning},
  author={Sutton, Richard S and Mahmood, A Rupam and White, Martha},
  journal={Journal of Machine Learning Research},
  volume={17},
  number={1},
  pages={2603--2631},
  year={2016},
  publisher={JMLR. org}
}
@article{yu2016weak,
  title={Weak convergence properties of constrained emphatic temporal-difference learning with constant and slowly diminishing stepsize},
  author={Yu, Huizhen},
  journal={Journal of Machine Learning Research},
  volume={17},
  number={1},
  pages={7745--7802},
  year={2016},
  publisher={JMLR. org}
}
@inproceedings{white2012scaling,
  title={Scaling life-long off-policy learning},
  author={White, Adam and Modayil, Joseph and Sutton, Richard S},
  booktitle={2012 IEEE International Conference on Development and Learning and Epigenetic Robotics (ICDL)},
  pages={1--6},
  year={2012},
  organization={IEEE}
}
@inproceedings{kolter2011fixed,
 author = {Kolter, J.},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {},
 title = {The Fixed Points of Off-Policy {TD}},
 volume = {24},
 year = {2011}
}
@inproceedings{precup2001off,
  title={Off-policy temporal-difference learning with function approximation},
  author={Precup, Doina and Sutton, Richard S and Dasgupta, Sanjoy},
  booktitle={{International Conference on Machine Learning}},
  pages={417--424},
  year={2001},
}
@article{precup2000eligibility,
  title={Eligibility traces for off-policy policy evaluation},
  author={Precup, Doina},
  journal={Computer Science Department Faculty Publication Series},
  pages={80},
  year={2000}
}
@article{sutton1999between,
  title={Between {MDPs} and {semi-MDPs}: A framework for temporal abstraction in reinforcement learning},
  author={Sutton, Richard S and Precup, Doina and Singh, Satinder},
  journal={Artificial Intelligence},
  volume={112},
  number={1-2},
  pages={181--211},
  year={1999},
  publisher={Elsevier}
}
@book{varga1999matrix,
  title={Matrix Iterative Analysis},
  author={Varga, R.S.},
  lccn={99050196},
  year={1999},
  publisher={Springer Berlin Heidelberg}
}
@inproceedings{tsitsiklis1996analysis,
 author = {Tsitsiklis, John and Van Roy, Benjamin},
 booktitle = {Advances in Neural Information Processing Systems},
 title = {Analysis of Temporal-Diffference Learning with Function Approximation},
 volume = {9},
 year = {1996}
}
@incollection{baird1995residual,
title = {Residual Algorithms: Reinforcement Learning with Function Approximation},
booktitle = {Machine Learning Proceedings 1995},
publisher = {Morgan Kaufmann},
pages = {30-37},
year = {1995},
author = {Leemon Baird},
}
@article{sutton1988learning,
  title={Learning to predict by the methods of temporal differences},
  author={Sutton, Richard S},
  journal={Machine learning},
  volume={3},
  pages={9--44},
  year={1988},
  publisher={Springer}
}