@inproceedings{si2021autoregressive,
	title        = {Autoregressive Quantile Flows for Predictive Uncertainty Estimation},
	author       = {Si, Phillip and Bishop, Allan and Kuleshov, Volodymyr},
	year         = 2021,
	booktitle    = {ICLR}
}
@article{densityestimation,
	title        = {Density estimation by dual ascent of the log-likelihood},
	author       = {Tabak, Esteban and Vanden-Eijnden, Eric},
	year         = 2010,
	month        = {03},
	journal      = {Communications in Mathematical Sciences - COMMUN MATH SCI},
	volume       = 8,
}
@inproceedings{touati2020randomized,
	title        = {Randomized value functions via multiplicative normalizing flows},
	author       = {Touati, Ahmed and Satija, Harsh and Romoff, Joshua and Pineau, Joelle and Vincent, Pascal},
	year         = 2020,
	booktitle    = {UAI},
	pages        = {422--432},
	organization = {PMLR}
}
@inproceedings{mazoure2020leveraging,
	title        = {Leveraging exploration in off-policy algorithms via normalizing flows},
	author       = {Mazoure, Bogdan and Doan, Thang and Durand, Audrey and Pineau, Joelle and Hjelm, R Devon},
	year         = 2020,
	booktitle    = {Conference on Robot Learning},
	pages        = {430--444},
	organization = {PMLR}
}
@article{ward2019improving,
	title        = {Improving exploration in soft-actor-critic with normalizing flows policies},
	author       = {Ward, Patrick Nadeem and Smofsky, Ariella and Bose, Avishek Joey},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1906.02771}
}
@article{koenker1978regression,
	title        = {Regression quantiles},
	author       = {Koenker, Roger and Bassett Jr, Gilbert},
	year         = 1978,
	journal      = {Econometrica: journal of the Econometric Society},
	publisher    = {JSTOR},
	pages        = {33--50}
}
@article{expbible,
	title        = {{Interpreting Expectiles}},
	author       = {Philipps, Collin},
	year         = 2021,
	journal      = {SSRN},
}
@inproceedings{er-dqn,
	title        = {{Statistics and Samples in Distributional Reinforcement Learning}},
	author       = {Rowland, Mark and Dadashi, Robert and Kumar, Saurabh and Munos, Rémi and Bellemare, Marc G. and Dabney, Will},
	year         = 2019,
	booktitle    = {ICML},
	pages        = {5528--5536},
	organization = {PMLR}
}
@inproceedings{mnih2016asynchronous,
	title        = {Asynchronous methods for deep reinforcement learning},
	author       = {Mnih, Volodymyr and Badia, Adria Puigdomenech and Mirza, Mehdi and Graves, Alex and Lillicrap, Timothy and Harley, Tim and Silver, David and Kavukcuoglu, Koray},
	year         = 2016,
	booktitle    = {ICML},
	pages        = {1928--1937},
	organization = {PMLR}
}
@article{li2018distributional,
	title        = {Distributional advantage actor-critic},
	author       = {Li, Shangda and Bing, Selina and Yang, Steven},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1806.06914}
}
@inproceedings{gmac,
	title        = {Gmac: A distributional perspective on actor-critic framework},
	author       = {Nam, Daniel W and Kim, Younghoon and Park, Chan Y},
	year         = 2021,
	booktitle    = {ICML},
	pages        = {7927--7936},
	organization = {PMLR}
}
@article{dsac,
	title        = {DSAC: distributional soft actor critic for risk-sensitive reinforcement learning},
	author       = {Ma, Xiaoteng and Xia, Li and Zhou, Zhengyuan and Yang, Jun and Zhao, Qianchuan},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2004.14547}
}
@article{barth2018distributed,
	title        = {Distributed distributional deterministic policy gradients},
	author       = {Barth-Maron, Gabriel and Hoffman, Matthew W and Budden, David and Dabney, Will and Horgan, Dan and Tb, Dhruva and Muldal, Alistair and Heess, Nicolas and Lillicrap, Timothy},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1804.08617}
}
@incollection{pytorch,
	title        = {PyTorch: An Imperative Style, High-Performance Deep Learning Library},
	author       = {Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia and Antiga, Luca and Desmaison, Alban and Kopf, Andreas and Yang, Edward and DeVito, Zachary and Raison, Martin and Tejani, Alykhan and Chilamkurthy, Sasank and Steiner, Benoit and Fang, Lu and Bai, Junjie and Chintala, Soumith},
	year         = 2019,
	booktitle    = {NeurIPS},
	publisher    = {Curran Associates, Inc.},
	pages        = {8024--8035}
}
@article{ie2019recsim,
	title        = {RecSim: A Configurable Simulation Platform for Recommender Systems},
	author       = {Eugene Ie and Chih-wei Hsu and Martin Mladenov and Vihan Jain and Sanmit Narvekar and Jing Wang and Rui Wu and Craig Boutilier},
	year         = 2019,
	eprint       = {1909.04847},
	archiveprefix = {arXiv},
	primaryclass = {cs.LG}
}
@inproceedings{danesh2021oodd,
	title        = {Out-of-Distribution Dynamics Detection: RL-Relevant Benchmarks and Results},
	author       = {Danesh, Mohamad H and Fern, Alan},
	year         = 2021,
	journal      = {},
	booktitle    = {ICML, Uncertainty & Robustness in Deep Learning Workshop}
}
@article{kuo2021quantum,
	title        = {Quantum architecture search via deep reinforcement learning},
	author       = {Kuo, En-Jui and Fang, Yao-Lung L and Chen, Samuel Yen-Chi},
	year         = 2021,
	journal      = {arXiv preprint arXiv:2104.07715}
}
@article{quantileinvariance,
	title        = {Invariance to Quantile Selection in Distributional Continuous Control},
	author       = {Grün, Felix and Saif ur Rehman, Muhammad and Glasmachers, Tobias and Iossifidis, Ioannis},
	year         = 2022,
	month        = 12,
}
@article{fqf,
	title        = {{Fully Parameterized Quantile Function for Distributional Reinforcement Learning}},
	author       = {Yang, Derek and Zhao, Li and Lin, Zichuan and Qin, Tao and Bian, Jiang and Liu, Tie-Yan},
	year         = 2019,
	journal      = {NeurIPS},
	volume       = 32
}
@inproceedings{iqn,
	title        = {{Implicit Quantile Networks for Distributional Reinforcement Learning}},
	author       = {Dabney, Will and Ostrovski, Georg and Silver, David and Munos, Rémi},
	year         = 2018,
	booktitle    = {ICML},
	pages        = {1096--1105},
	organization = {PMLR}
}
@inproceedings{qr-dqn,
	title        = {{Distributional Reinforcement Learning with Quantile Regression}},
	author       = {Dabney, Will and Rowland, Mark and Bellemare, Marc and Munos, Rémi},
	year         = 2018,
	booktitle    = {Proceedings of the AAAI Conference on Artificial Intelligence},
	volume       = 32
}
@inproceedings{c51,
	title        = {{A Distributional Perspective on Reinforcement Learning}},
	author       = {Bellemare, Marc G. and Dabney, Will and Munos, Rémi},
	year         = 2017,
	booktitle    = {ICML},
	pages        = {449--458},
	organization = {PMLR}
}
@article{crossing,
 author = {Xuming He},
 journal = {The American Statistician},
 number = {2},
 pages = {186--192},
 publisher = {[American Statistical Association, Taylor & Francis, Ltd.]},
 title = {Quantile Curves without Crossing},
 urldate = {2023-04-04},
 volume = {51},
 year = {1997}
}
@book{sutton2018reinforcement,
  title={Reinforcement Learning: An Introduction},
  author={Sutton, Richard S. and Barto, Andrew G.},
  year={2018},
  publisher={MIT press}
}
@inproceedings{strens2000bayesian,
  title={{A {Bayesian} Framework for Reinforcement Learning}},
  author={Strens, Malcolm},
  booktitle={ICML},
  volume={2000},
  pages={943--950},
  year={2000}
}
@article{vlassis2012bayesian,
  title={{Bayesian Reinforcement Learning}},
  author={Vlassis, Nikos and Ghavamzadeh, Mohammad and Mannor, Shie and Poupart, Pascal},
  journal={Reinforcement Learning: State-of-the-Art},
  pages={359--386},
  year={2012},
  publisher={Springer}
}
@inproceedings{mavrin2019distributional,
  title={{Distributional Reinforcement Learning for Efficient Exploration}},
  author={Mavrin, Borislav and Yao, Hengshuai and Kong, Linglong and Wu, Kaiwen and Yu, Yaoliang},
  booktitle={ICML},
  pages={4424--4434},
  year={2019},
  organization={PMLR}
}

@InProceedings{pmlr-v139-kostas21a,
  title = 	 {High Confidence Generalization for Reinforcement Learning},
  author =       {Kostas, James and Chandak, Yash and Jordan, Scott M and Theocharous, Georgios and Thomas, Philip},
  booktitle = 	 {ICML},
  pages = 	 {5764--5773},
  year = 	 {2021},
  publisher =    {PMLR},
}

@article{chow2017risk,
  title={Risk-constrained reinforcement learning with percentile risk criteria},
  author={Chow, Yinlam and Ghavamzadeh, Mohammad and Janson, Lucas and Pavone, Marco},
  journal={JMLR},
  volume={18},
  number={1},
  pages={6070--6120},
  year={2017},
}

@article{expectile-blue,
    title = {{When is an Expectile the Best Linear Unbiased Estimator?}},
    author = {Philipps, Colin},
    year = {2021},
    journal = {SSRN},
}

@article{expectilesoriginal,
 author = {Whitney K. Newey and James L. Powell},
 journal = {Econometrica},
 number = {4},
 pages = {819--847},
 publisher = {[Wiley, Econometric Society]},
 title = {{Asymmetric Least Squares Estimation and Testing}},
 volume = {55},
 year = {1987}
}

@article{rowland2023analysis,
      title={{An Analysis of Quantile Temporal-Difference Learning}}, 
      author={Mark Rowland and Rémi Munos and Mohammad Gheshlaghi Azar and Yunhao Tang and Georg Ostrovski and Anna Harutyunyan and Karl Tuyls and Marc G. Bellemare and Will Dabney},
      year={2023},
      journal={arXiv preprint arXiv:2301.04462},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@article{tang2022nature,
      title={The Nature of Temporal Difference Errors in Multi-step Distributional Reinforcement Learning}, 
      author={Yunhao Tang and Mark Rowland and Rémi Munos and Bernardo Ávila Pires and Will Dabney and Marc G. Bellemare},
      year={2022},
      journal={arXiv preprint arXiv:2207.07570},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@book{distributional-book,
    title={Distributional Reinforcement Learning},
    author={Marc G. Bellemare and Will Dabney and Mark Rowland},
    publisher={MIT Press},
    year={2023}
}

@article{dqn,
      title={{Playing {Atari} with Deep Reinforcement Learning}}, 
      author={Volodymyr Mnih and Koray Kavukcuoglu and David Silver and Alex Graves and Ioannis Antonoglou and Daan Wierstra and Martin Riedmiller},
      year={2013},
      journal={arXiv preprint arXiv:1312.5602},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@article{sac,
      title={{Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor}}, 
      author={Tuomas Haarnoja and Aurick Zhou and Pieter Abbeel and Sergey Levine},
      year={2018},
      journal={arXiv preprint arXiv:1801.01290},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@article{ridgeoriginal,
    title={Application of Ridge Analysis to Regression Problems},
    journal={Chemical Engineering Progress},
    volume={58},
    year={1962},
    author={Arthur E. Hoerl},
    pages = {54--59},
}
@inproceedings{quota,
author = {Zhang, Shangtong and Yao, Hengshuai},
title = {QUOTA: The Quantile Option Architecture for Reinforcement Learning},
year = {2019},
publisher = {AAAI Press},
articleno = {711},
numpages = {8},
booktitle = {AAAI}
}
@ARTICLE{dsac2,
  author={Duan, Jingliang and Guan, Yang and Li, Shengbo Eben and Ren, Yangang and Sun, Qi and Cheng, Bo},
  journal={IEEE Transactions on Neural Networks and Learning Systems}, 
  title={Distributional Soft Actor-Critic: Off-Policy Reinforcement Learning for Addressing Value Estimation Errors}, 
  year={2022},
  volume={33},
  number={11},
  pages={6584-6598},
  }
@article{cleanrl,
  author  = {Shengyi Huang and Rousslan Fernand Julien Dossa and Chang Ye and Jeff Braga and Dipam Chakraborty and Kinal Mehta and João G.M. Araújo},
  title   = {{{CleanRL}: High-quality Single-file Implementations of Deep Reinforcement Learning Algorithms}},
  journal = {JMLR},
  year    = {2022},
  volume  = {23},
  number  = {274},
  pages   = {1--18},
}

@InProceedings{sddrl,
  title = 	 {{Stochastically Dominant Distributional Reinforcement Learning}},
  author =       {Martin, John and Lyskawinski, Michal and Li, Xiaohu and Englot, Brendan},
  booktitle = 	 {ICML},
  pages = 	 {6745--6754},
  year = 	 {2020},
  volume = 	 {119},
  month = 	 {13--18 Jul},
  publisher =    {PMLR},
}
@article{
gtdqn,
title={{A Simulation Environment and Reinforcement Learning Method for Waste Reduction}},
author={Sami Jullien and Mozhdeh Ariannezhad and Paul Groth and Maarten de Rijke},
journal={TMLR},
year={2023},
note={}
}
@inproceedings{mujoco,
  title={MuJoCo: A physics engine for model-based control},
  author={Todorov, Emanuel and Erez, Tom and Tassa, Yuval},
  booktitle={2012 IEEE/RSJ International Conference on Intelligent Robots and Systems},
  pages={5026--5033},
  year={2012},
  organization={IEEE},
}

@InProceedings{meanestimator,
  title = 	 {A Tale of Sampling and Estimation in Discounted Reinforcement Learning},
  author =       {Metelli, Alberto Maria and Mutti, Mirco and Restelli, Marcello},
  booktitle = 	 {Proceedings of The 26th International Conference on Artificial Intelligence and Statistics},
  pages = 	 {4575--4601},
  year = 	 {2023},
  volume = 	 {206},
  month = 	 {25--27 Apr},
  publisher =    {PMLR},
}
@inproceedings{
kostrikov2022offline,
title={{Offline Reinforcement Learning with Implicit Q-Learning}},
author={Ilya Kostrikov and Ashvin Nair and Sergey Levine},
booktitle={ICLR},
year={2022},
}

@article{expectile-location-scale,
author = {Yao, Qiwli  and  Tong, Howell},
title = {{Asymmetric Least Squares Regression Estimation: A Nonparametric Approach}},
journal = {Journal of Nonparametric Statistics},
volume = {6},
number = {2-3},
pages = {273-292},
year  = {1996},
publisher = {Taylor & Francis},
}

@article{rlpd,
      title={Efficient Online Reinforcement Learning with Offline Data}, 
      author={Philip J. Ball and Laura Smith and Ilya Kostrikov and Sergey Levine},
      year={2023},
      journal={arXiv preprint arXiv:2302.02948},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@inproceedings{droq,
    title={Dropout Q-Functions for Doubly Efficient Reinforcement Learning},
    author={Takuya Hiraoka and Takahisa Imagawa and Taisei Hashimoto and Takashi Onishi and Yoshimasa Tsuruoka},
    booktitle={ICLR},
      year={2022},
}

@article{polyak,
	author = {Polyak, Boris T. and Juditsky, Anatoli B.},
	journal = {SIAM Journal on Control and Optimization},
	number = {4},
	pages = {838-855},
	title = {{Acceleration of Stochastic Approximation by Averaging}},
	volume = {30},
	year = {1992}}

@article{exp-quant-david-goliath,
    author = {Linda Schulze Waltrup and Fabian Sobotka and Thomas Kneib and Göran Kauermann},
    title ={{Expectile and Quantile Regression—David and Goliath?}},
    journal = {Statistical Modelling},
    volume = {15},
    number = {5},
    pages = {433-456},
    year = {2015},
}
@inproceedings{fan2022welfare,
author = {Fan, Zimeng and Peng, Nianli and Tian, Muhang and Fain, Brandon},
title = {Welfare and Fairness in Multi-objective Reinforcement Learning},
year = {2023},
booktitle = {AAMAS}
}
@article{minorityfaces,
  title={Accuracy comparison across face recognition algorithms: Where are we on measuring race bias?},
  author={Cavazos, Jacqueline G. and Phillips, P. Jonathon and Castillo, Carlos D. and O'Toole, Alice J.},
  journal={IEEE Transactions on Biometrics, Behavior, and Identity Science},
  volume={3},
  number={1},
  pages={101--111},
  year={2020},
  publisher={IEEE}
}
@inproceedings{codac,
 author = {Ma, Yecheng and Jayaraman, Dinesh and Bastani, Osbert},
 booktitle = {NeurIPS},
 pages = {19235--19247},
 publisher = {Curran Associates, Inc.},
 title = {Conservative Offline Distributional Reinforcement Learning},
 volume = {34},
 year = {2021}
}
@inproceedings{redq,
title={Randomized Ensembled Double Q-Learning: Learning Fast Without a Model},
author={Xinyue Chen and Che Wang and Zijian Zhou and Keith W. Ross},
booktitle={ICLR},
year={2021},
}
@misc{gymnasium, 
        title={Gymnasium},
        author={Farama foundation},
        year={2023},
url={https://github.com/Farama-Foundation/Gymnasium}}

@misc{jax2018github,
  author = {James Bradbury and Roy Frostig and Peter Hawkins and Matthew James Johnson and Chris Leary and Dougal Maclaurin and George Necula and Adam Paszke and Jake Vander{P}las and Skye Wanderman-{M}ilne and Qiao Zhang},
  title = {{{JAX}: Composable Transformations of {P}ython+{N}um{P}y Programs}},
  url = {http://github.com/google/jax},
  version = {0.3.13},
  year = {2018},
}
@Article{bellemare13arcade,
    author = {Bellemare, Marc G. and Naddaf, Yavar and Veness, Joel and Bowling, Michael},
    title = {{The {Arcade Learning Environment}: An Evaluation Platform for General Agents}},
    journal = {Journal of Artificial Intelligence Research},
    year = "2013",
    month = "jun",
    volume = "47",
    pages = "253--279",
}
@Article{machado18arcade,
    author = {Marlos C. Machado and Marc G. Bellemare and Erik Talvitie and Joel Veness and Matthew J. Hausknecht and Michael Bowling},
    title = {{Revisiting the {Arcade Learning Environment}: Evaluation Protocols and Open Problems for General Agents}},
    journal = {Journal of Artificial Intelligence Research},
    volume = {61},
    pages = {523--562},
    year = {2018}
}


@InProceedings{atari5,
  title = 	 {{{A}tari-5: Distilling the Arcade Learning Environment down to Five Games}},
  author =       {Aitchison, Matthew and Sweetser, Penny and Hutter, Marcus},
  booktitle = 	 {Proceedings of the 40th International Conference on Machine Learning},
  pages = 	 {421--438},
  year = 	 {2023},
  editor = 	 {Krause, Andreas and Brunskill, Emma and Cho, Kyunghyun and Engelhardt, Barbara and Sabato, Sivan and Scarlett, Jonathan},
  volume = 	 {202},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {23--29 Jul},
  publisher =    {PMLR},
}

@article{agarwal2021deep,
    title={{Deep Reinforcement Learning at the Edge of the Statistical Precipice}},
  author={Agarwal, Rishabh and Schwarzer, Max and Castro, Pablo Samuel and Courville, Aaron C. and Bellemare, Marc},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@article{german-paper,
	author = {Hajo Holzmann and Bernhard Klar},
	journal = {Electronic Journal of Statistics},
	keywords = {asymptotic normality, convergence to stable distributions, expectiles, M-estimator, uniform central limit theorem},
	number = {2},
	pages = {2355 -- 2371},
	publisher = {Institute of Mathematical Statistics and Bernoulli Society},
	title = {{Expectile Asymptotics}},
	volume = {10},
	year = {2016},}

@phdthesis{luzin,
author = {Nikolai Luzin},
journal = {Doctoral Thesis},
title ={The Integral and Trigonometric Series (Russian)},
school = {Moscow State University},
year = 1915}

@article{zaretsky,
author = {Moisej A. Zaretsky},
journal = {Doklady Rossiiskoi Akademii Nauk},
title ={{On One Theorem on Absolutely Continuous Functions (Russian)}},
year = 1925}

@phdthesis{WeiEA2014,
title={Parameter Estimation and Prediction Interval Construction for Location-Scale Models with Nuclear Applications},
author={Xingli Wei},
school={McMaster University},
year={2014},
}   


@InProceedings{pmlr-v139-fei21a,
  title = 	 {{Risk-Sensitive Reinforcement Learning with Function Approximation: A Debiasing Approach}},
  author =       {Fei, Yingjie and Yang, Zhuoran and Wang, Zhaoran},
  booktitle = 	 {Proceedings of the 38th International Conference on Machine Learning},
  pages = 	 {3198--3207},
  year = 	 {2021},
  editor = 	 {Meila, Marina and Zhang, Tong},
  volume = 	 {139},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {18--24 Jul},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v139/fei21a/fei21a.pdf},
  abstract = 	 {We study function approximation for episodic reinforcement learning with entropic risk measure. We first propose an algorithm with linear function approximation. Compared to existing algorithms, which suffer from improper regularization and regression biases, this algorithm features debiasing transformations in backward induction and regression procedures. We further propose an algorithm with general function approximation, which features implicit debiasing transformations. We prove that both algorithms achieve a sublinear regret and demonstrate a trade-off between generality and efficiency. Our analysis provides a unified framework for function approximation in risk-sensitive reinforcement learning, which leads to the first sublinear regret bounds in the setting.}
}

@inproceedings{NEURIPS2022_c88a2bd0,
 author = {Lim, Shiau Hong and Malik, Ilyas},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {S. Koyejo and S. Mohamed and A. Agarwal and D. Belgrave and K. Cho and A. Oh},
 pages = {30977--30989},
 publisher = {Curran Associates, Inc.},
 title = {{Distributional Reinforcement Learning for Risk-Sensitive Policies}},
 volume = {35},
 year = {2022}
}

@inproceedings{NEURIPS2022_d2511dfb,
 author = {Greenberg, Ido and Chow, Yinlam and Ghavamzadeh, Mohammad and Mannor, Shie},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {S. Koyejo and S. Mohamed and A. Agarwal and D. Belgrave and K. Cho and A. Oh},
 pages = {32639--32652},
 publisher = {Curran Associates, Inc.},
 title = {{Efficient Risk-Averse Reinforcement Learning}},
 volume = {35},
 year = {2022}
}