@InProceedings{Hutter:23selfaixi,
  author =       "Elliot Catt and Jordi Grau-Moya and Marcus Hutter and Matthew Aitchison and Tim Genewein and Gregoire Deletang and Li Kevin Wenliang and Joel Veness",
  title =        "Self-Predictive Universal {AI}",
  booktitle =    "37th Conf. on Neural Information Processing Systems ({NeurIPS'23})",
  pages =        "1--18",
  address =      "New Orleans, USA",
  _month =        dec,
  year =         "2023",
  bibtex =       "http://www.hutter1.net/official/bib.htm#selfaixi",
  http =         "https://openreview.net/forum?id=psXVkKO9No",
  pdf =          "http://www.hutter1.net/publ/selfaixi.pdf",
  poster =       "http://www.hutter1.net/publ/pselfaixi.pdf",
  project =      "http://www.hutter1.net/official/projects.htm#uai",
  keywords =     "reinforcement learning; universal; self-prediction; Bayes; planning; on-policy; intelligence; AIXI",
}

@Book{Hutter:24uaibook2,
  author =       "Marcus Hutter and David Quarel and Elliot Catt",
  title =        "An Introduction to Universal Artificial Intelligence",
  series =       "Chapman \& Hall/CRC Artificial Intelligence and Robotics Series",
  publisher =    "Taylor and Francis",
  xaddress =      "Berlin",
  year =         "2024",
  _note =         "500 pages, http://www.hutter1.net/ai/uaibook2.htm",
  http =         "https://www.routledge.com/An-Introduction-to-Universal-Artificial-Intelligence/Hutter-Catt-Quarel/p/book/9781032607023",
  slides =       "http://www.hutter1.net/ai/suaibook.pdf",
  video =        "http://vimeo.com/14888930",
  keywords =     "Artificial general intelligence; algorithmic information theory;
                  Bayes mixture distributions; universal sequence prediction;
                  context tree weighting; rational agents; sequential decision theory;
                  universal intelligent agents; reinforcement learning;
                  games and multi-agent systems; approximation/implementation/application;
                  AGI-safety; philosophy of AI.",
}

@InProceedings{Hutter:17corruptrl,
  author =       "Tom Everitt and Victoria Krakovna and Laurent Orseau and Marcus Hutter and Shane Legg",
  title =        "Reinforcement Learning with a Corrupted Reward Channel",
  booktitle =    "Proc. 26th International Joint Conf. on Artificial Intelligence ({IJCAI'17})",
  address =      "Melbourne, Australia",
  _editor =       "Carles Sierra",
  _publisher =    "IJCAI",
  pages =        "4705--4713",
  _month =        aug,
  year =         "2017",
  bibtex =       "http://www.hutter1.net/official/bib.htm#corruptrl",
  pdf =          "http://www.hutter1.net/publ/corruptrl.pdf",
  slides =       "http://www.hutter1.net/publ/scorruptrl.pdf",
  project =      "http://www.hutter1.net/official/projects.htm#safe",
  keywords =     "decoupled reinforcement learning; reward corruption;
                  quantilisation; robustness; value learning.",
}
@inproceedings{Hutter:24trainsol,
  title        = {Learning Universal Predictors},
  author       = {Grau-Moya, Jordi and Genewein, Tim and Hutter, Marcus and Orseau, Laurent and Deletang, Gregoire and Catt, Elliot and Ruoss, Anian and Wenliang, Li Kevin and Mattern, Christopher and Aitchison, Matthew and Veness, Joel},
  year         = 2024,
  booktitle    = {41st International Conference on Machine Learning},
  _month       = jul,
  bibtex       = {http://www.hutter1.net/official/bib.htm#trainsol},
  arxiv        = {http://arxiv.org/abs/2401.14953},
  press        = {http://medium.com/syncedreview/neural-networks-on-the-brink-of-universal-prediction-with-deepminds-cutting-edge-approach-2de9af5b4e3f}
}

@inproceedings{abbeel2004apprenticeship,
  title={Apprenticeship learning via inverse reinforcement learning},
  author={Abbeel, Pieter and Ng, Andrew Y},
  booktitle={Proceedings of the twenty-first international conference on Machine learning},
  pages={1},
  year={2004},
  organization={ACM}
}

@article{amodei_olah_2016, title={Concrete Problems in {AI} Safety}, journal={arXiv preprint arXiv:1606.06565}, author={Amodei, Dario and Olah, Chris and Steinhardt, Jacob and Christiano, Paul and Schulman, John and Mané, Dan}, year={2016}}

@article{armstrong_sandberg_bostrom_2012, title={Thinking Inside the Box: Controlling and Using an Oracle {AI}}, volume={22}, DOI={10.1007/s11023-012-9282-2}, number={4}, journal={Minds and Machines}, author={Armstrong, Stuart and Sandberg, Anders and Bostrom, Nick}, year={2012}, month={Jun}, pages={299–324}}

@InProceedings{bell_tate_1985, title={Using temporal constraints to restrict search in a planner}, booktitle={Proc. Third Alvey IKBS SIG Workshop}, author={Bell, Colin E. and Tate, Austin}, year={1985}}

@booklet{bengio_2017,
author = {Bengio, Yoshua},
title = {Deep Learning for the Future of {AI}},
howpublished = {Presented at Open Philanthropy Workshop},
year = {2017},
url={https://drive.google.com/file/d/0BwMz1xbo9yErWldPWWNlS0NnOE0/view?fbclid=IwAR0sBQ5eBCZ3slOS5plUd-ZM6-vqAzOKVLvT_d0xBPe1OLrl5fxf9CdJAy4}
}

@article{blackwell1962merging,
  title={Merging of opinions with increasing information},
  author={Blackwell, David and Dubins, Lester},
  journal={The Annals of Mathematical Statistics},
  volume={33},
  number={3},
  pages={882--886},
  year={1962},
  publisher={JSTOR}
}

@book{bostrom_2014, place={Oxford}, title={Superintelligence: paths, dangers, strategies}, publisher={Oxford University Press}, author={Bostrom, Nick}, year={2014}}

@misc{christiano_2016, title={What does the universal prior actually look like?}, howpublished="\path{https://ordinaryideas.wordpress.com/2016/11/30/what-does-the-universal-prior-actually-look-like/}", journal={Ordinary Ideas}, author={Christiano, Paul F.}, year={2016}}
  
@article{carey2020incentives,
  title={The Incentives that Shape Behaviour},
  author={Carey, Ryan and Langlois, Eric and Everitt, Tom and Legg, Shane},
  journal={arXiv preprint arXiv:2001.07118},
  year={2020}
}
  
@inproceedings{cohen2019asymptotically,
  title={Asymptotically Unambitious Artificial General Intelligence},
  author={Cohen, Michael K and Vellambi, Badri and Hutter, Marcus},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  year={2020}
}

@article{cohen2020curiosity,
    title={Curiosity Killed the Cat and the Asymptotically Optimal Agent},
    author={Michael K. Cohen and Marcus Hutter},
    journal={arXiv preprint arXiv:2006.03357},
    year={2020},
    eprint={2006.03357},
    archivePrefix={arXiv},
    primaryClass={cs.LG},
}
  
@phdthesis{clouse1997integrating,
  author       = {Jeffery A Clouse}, 
  title        = {On integrating apprentice learning and reinforcement learning},
  school       = {University of Massachusetts Amherst},
  year         = 1997
}
  
@book{durrett2010probability,
  title={Probability: Theory and Examples},
  author={Durrett, R},
  year={2010},
  publisher={Cambridge University Press}
}
  
@article{everitt2019understanding,
  title={Understanding Agent Incentives using Causal Influence Diagrams, Part I: Single Action Settings},
  author={Everitt, Tom and Ortega, Pedro A and Barnes, Elizabeth and Legg, Shane},
  journal={arXiv preprint arXiv:1902.09980},
  year={2019}
}

@inproceedings{finn2016guided,
  title={Guided cost learning: Deep inverse optimal control via policy optimization},
  author={Finn, Chelsea and Levine, Sergey and Abbeel, Pieter},
  booktitle={International Conference on Machine Learning},
  pages={49--58},
  year={2016}
}

@article{garcia2015comprehensive,
  title={A comprehensive survey on safe reinforcement learning},
  author={Garc{\'\i}a, Javier and Fern{\'a}ndez, Fernando},
  journal={Journal of Machine Learning Research},
  volume={16},
  number={1},
  pages={1437--1480},
  year={2015}
}

@article{garcia2012safe,
  title={Safe exploration of state and action spaces in reinforcement learning},
  author={Garc{\'\i}a, Javier and Fern{\'a}ndez, Fernando},
  journal={Journal of Artificial Intelligence Research},
  volume={45},
  pages={515--564},
  year={2012}
}

@article{garcia2013safe,
  title={Safe reinforcement learning through probabilistic policy reuse},
  author={Garc{\'\i}a, Javier and Acera, Daniel and Fern{\'a}ndez, Fernando},
  journal={RLDM 2013},
  pages={14},
  year={2013},
  publisher={Citeseer}
}

@inproceedings{gaskett2003reinforcement,
  title={Reinforcement learning under circumstances beyond its control},
  author={Gaskett, Chris},
  booktitle={Proceedings of the International Conference on Computational Intelligence for Modelling Control and Automation},
  year={2003}
}

@article{goodhart_1984, title={Problems of Monetary Management: The {UK} Experience}, DOI={10.1007/978-1-349-17295-5_4}, journal={Monetary Theory and Practice}, author={Goodhart, C. A. E.}, year={1984}, pages={91–121}}

@article{goodman1965new,
  title={The new riddle of induction},
  author={Goodman, Nelson},
  year={1965}
}

@article{grice1975logic,
  title={Logic and conversation (pp. 41--58)},
  author={Grice, HP},
  journal={Syntax and semantics},
  volume={3},
  year={1975}
}

@inproceedings{hadfield2017inverse,
  title={Inverse reward design},
  author={Hadfield-Menell, Dylan and Milli, Smitha and Abbeel, Pieter and Russell, Stuart J and Dragan, Anca},
  booktitle={Advances in Neural Information Processing Systems},
  pages={6765--6774},
  year={2017}
}

@inproceedings{hadfield2016cooperative,
  title={Cooperative inverse reinforcement learning},
  author={Hadfield-Menell, Dylan and Russell, Stuart J and Abbeel, Pieter and Dragan, Anca},
  booktitle={Advances in neural information processing systems},
  pages={3909--3917},
  year={2016}
}

@inproceedings{hans2008safe,
  title={Safe exploration for reinforcement learning.},
  author={Hans, Alexander and Schneega{\ss}, Daniel and Sch{\"a}fer, Anton Maximilian and Udluft, Steffen},
  booktitle={ESANN},
  pages={143--148},
  year={2008}
}

@article{hibbard_2012, title={Avoiding Unintended {AI} Behaviors}, DOI={10.1007/978-3-642-35506-6_12}, journal={Artificial General Intelligence Lecture Notes in Computer Science}, author={Hibbard, Bill}, year={2012}, pages={107–116}}

@incollection{heger1994consideration,
  title={Consideration of risk in reinforcement learning},
  author={Heger, Matthias},
  booktitle={Machine Learning Proceedings 1994},
  pages={105--111},
  year={1994},
  publisher={Elsevier}
}

@InProceedings{hessel_2018,
  author={Hessel, M. and Modayil, J. and van Hasselt, H. and Schaul, T. and
          Ostrovski, G. and Dabney, W. and Horgan, D. and Piot, B. and
          Azar, M. and Silver, D.},
  year={2018},
  title={Rainbow: Combining improvements in
  deep reinforcement learning},
  booktitle={Proc. of AAAI Conference on Artificial Intelligence}
}

@inproceedings{ho2016generative,
  title={Generative adversarial imitation learning},
  author={Ho, Jonathan and Ermon, Stefano},
  booktitle={Advances in neural information processing systems},
  pages={4565--4573},
  year={2016}
}

@article{huang2017enabling,
  title={Enabling robots to communicate their objectives},
  author={Huang, Sandy H and Held, David and Abbeel, Pieter and Dragan, Anca D},
  journal={Autonomous Robots},
  pages={1--18},
  year={2017},
  publisher={Springer}
}

@Book{Hutter:04uaibook,
  author =       "Marcus Hutter",
  title =        "Universal Artificial Intelligence:
                  Sequential Decisions based on Algorithmic Probability",
  _series =       "EATCS",
  publisher =    "Springer",
  address =      "Berlin",
  year =         "2005",
  isbn =         "3-540-22139-5",
  isbn-online =  "978-3-540-26877-2",
  doi =          "10.1007/b138233",
  review1 =      "AIJ: http://dx.doi.org/10.1016/j.artint.2006.10.005",
  review2 =      "ACM: http://www.reviews.com/review/review_review.cfm?review_id=131175",
  reviews =      "Amazon: http://www.amazon.com/exec/obidos/redirect?tag=homepageofm0a-20&path=ASIN/3540221395",
  keywords =     "Artificial intelligence; algorithmic probability;
                  sequential decision theory; Solomonoff induction;
                  Kolmogorov complexity; Bayes mixture distributions;
                  reinforcement learning; universal sequence prediction;
                  tight loss and error bounds; Levin search;
                  strategic games; function minimization; supervised learning.",
  abstract =     "This book presents sequential decision theory from a
                  novel algorithmic information theory perspective. While the former
                  theory is suited for active agents in known environments, the
                  latter is suited for passive prediction of unknown environments.
                  The book introduces these two well-known but very different ideas
                  and removes the limitations by unifying them to one parameter-free
                  theory of an optimal reinforcement learning agent interacting with
                  an arbitrary unknown world. Most if not all AI problems can easily
                  be formulated within this theory, which reduces the conceptual
                  problems to pure computational ones. Considered problem classes
                  include sequence prediction, strategic games, function
                  minimization, reinforcement and supervised learning. Formal
                  definitions of intelligence order relations, the horizon problem
                  and relations to other approaches to AI are discussed. One
                  intention of this book is to excite a broader AI audience about
                  abstract algorithmic information theory concepts, and conversely
                  to inform theorists about exciting applications to AI.",
  support =      "SNF grant 2000-61847",
}

@Article{Hutter:09aixiopen,
  author =       "Marcus Hutter",
  title =        "Open Problems in Universal Induction \& Intelligence",
  journal =      "Algorithms",
  volume =       "3",
  number =       "2",
  pages =        "879--906",
  _month =        jul,
  year =         "2009",
  bibtex =       "http://www.hutter1.net/official/bib.htm#aixiopen",
  pdf =          "http://www.hutter1.net/ai/aixiopen.pdf",
  ps =           "http://www.hutter1.net/ai/aixiopen.ps",
  latex =        "http://www.hutter1.net/ai/aixiopen.tex",
  project =      "http://www.hutter1.net/official/projects.htm#uai",
  doi =          "10.3390/a2030879",
  issn =         "1999-4893",
  keywords =     "Kolmogorov complexity; information theory;
                  sequential decision theory; reinforcement learning;
                  artificial intelligence; universal Solomonoff induction;
                  rational agents.",
  abstract =     "Specialized intelligent systems can be found everywhere: finger
                  print, handwriting, speech, and face recognition, spam filtering,
                  chess and other game programs, robots, et al. This decade the first
                  presumably complete {\em mathematical} theory of artificial
                  intelligence based on universal induction-prediction-decision-action
                  has been proposed. This information-theoretic approach solidifies
                  the foundations of inductive inference and artificial intelligence.
                  Getting the foundations right usually marks a significant progress
                  and maturing of a field. The theory provides a gold standard and
                  guidance for researchers working on intelligent algorithms. The
                  roots of universal induction have been laid exactly half-a-century
                  ago and the roots of universal intelligence exactly one decade ago.
                  So it is timely to take stock of what has been achieved and what
                  remains to be done. Since there are already good recent surveys, I
                  describe the state-of-the-art only in passing and refer the reader
                  to the literature. This article concentrates on the open problems in
                  universal induction and its extension to universal intelligence.",
}

@InProceedings{Hutter:11asyoptag,
  author =       "Tor Lattimore and Marcus Hutter",
  title =        "Asymptotically Optimal Agents",
  booktitle =    "Proc. 22nd International Conf. on Algorithmic Learning Theory ({ALT'11})",
  address =      "Espoo, Finland",
  series =       "LNAI",
  volume =       "6925",
  _editor =       "J. Kivinen and C. Szepesv{\'a}ri and E. Ukkonen and T. Zeugmann",
  publisher =    "Springer",
  pages =        "368--382",
  _month =        oct,
  year =         "2011",
  bibtex =       "http://www.hutter1.net/official/bib.htm#asyoptag",
  conf =         "http://www-alg.ist.hokudai.ac.jp/~thomas/ALT11/alt11c.html",
  pdf =          "http://www.hutter1.net/publ/asyoptag.pdf",
  latex =        "http://www.hutter1.net/publ/asyoptag.tex",
  slides =       "http://www.hutter1.net/publ/sasyoptag.pdf",
  project =      "http://www.hutter1.net/official/projects.htm#uai",
  doi =          "10.1007/978-3-642-24412-4_29",
  issn =         "0302-9743",
  isbn =         "3-642-24411-4",
  keywords =     "Rational agents; sequential decision theory;
                  artificial general intelligence; reinforcement learning;
                  asymptotic optimality; general discounting.",
  abstract =     "Artificial general intelligence aims to create agents capable
                  of learning to solve arbitrary interesting problems. We define
                  two versions of asymptotic optimality and prove that no agent
                  can satisfy the strong version while in some cases, depending
                  on discounting, there does exist a non-computable weak
                  asymptotically optimal agent.",
  support =      "ARC grant DP0988049",
  for =          "080101(100%)",
  znote =        "Acceptance rate: 28/61 = 46\%",
}

@Article{Hutter:14tcdiscx,
  author =       "Tor Lattimore and Marcus Hutter",
  title =        "General Time Consistent Discounting",
  journal =      "Theoretical Computer Science",
  volume =       "519",
  pages =        "140--154",
  publisher =    "Elsevier",
  _month =        jan,
  year =         "2014",
  bibtex =       "http://www.hutter1.net/official/bib.htm#tcdiscx",
  pdf =          "http://www.hutter1.net/publ/tcdiscx.pdf",
  slides =       "http://www.hutter1.net/publ/stcdisc.pdf",
  project =      "http://www.hutter1.net/official/projects.htm#agents",
  issn =         "0304-3975",
  doi =          "10.1016/j.tcs.2013.09.022",
  keywords =     "Rational agents; sequential decision theory;
                  general discounting; time-consistency; game theory.",
  abstract =     "Modeling inter-temporal choice is a key problem in both computer
                  science and economic theory. The discounted utility model of
                  Samuelson is currently the most popular model for measuring the
                  global utility of a time-series of local utilities. The model is
                  limited by not allowing the discount function to change with the
                  age of the agent. This is despite the fact that many agents, in
                  particular humans, are best modelled with age-dependent discount
                  functions. It is well known that discounting can lead to
                  time-inconsistent behaviour where agents change their preferences
                  over time. In this paper we generalise the discounted utility model
                  to allow age-dependent discount functions. We then extend previous
                  work in time-inconsistency to our new setting, including a complete
                  characterisation of time-(in)consistent discount functions, the
                  existence of sub-game perfect equilibrium policies where the
                  discount function is time-inconsistent and a continuity result
                  showing that ``nearly'' time-consistent discount rates lead to
                  ``nearly'' time-consistent behaviour.",
  for =          "010405(20%),080101(40%),140104(20%),170202(20%)",
  seo =          "970108(40%),970114(30%),970117(30%)",
}

@InProceedings{Hutter:16thompgrl,
  author =       "Jan Leike and Tor Lattimore and Laurent Orseau and Marcus Hutter",
  title =        "Thompson Sampling is Asymptotically Optimal in General Environments",
  booktitle =    "Proc. 32nd International Conf. on Uncertainty in Artificial Intelligence ({UAI'16})",
  address =      "New Jersey, USA",
  _editor =       "Alexander Ihler and Dominik Janzing",
  publisher =    "AUAI Press",
  pages =        "417--426",
  _month =        jun,
  year =         "2016",
  bibtex =       "http://www.hutter1.net/official/bib.htm#thompgrl",
  http =         "http://auai.org/uai2016/proceedings/papers/20.pdf",
  pdf =          "http://www.hutter1.net/publ/thompgrl.pdf",
  latex =        "http://www.hutter1.net/publ/thompgrl.tex",
  slides =       "http://www.hutter1.net/publ/sthompgrl.pdf",
  award =        "http://auai.org/uai2016/program.php",
  project =      "http://www.hutter1.net/official/projects.htm#uai",
  isbn =         "978-0-9966431-1-5",
  keywords =     "General reinforcement learning; Thompson sampling;
                  asymptotic optimality; regret; discounting; recoverability; AIXI",
  abstract =     "We discuss a variant of Thompson sampling for nonparametric
                  reinforcement learning in countable classes of general stochastic
                  environments. These environments can be non-Markov, nonergodic, and
                  partially observable. We show that Thompson sampling learns the
                  environment class in the sense that (1) asymptotically its value
                  converges to the optimal value in mean and (2) given a
                  recoverability assumption regret is sublinear.",
  support =      "ARC grant DP150104590",
  for =          "080101(60%),010404(40%)",
  seo =          "970108(100%)",
  znote =        "Acceptance rate: 26/275 = 9\% (oral!) [85/275 = 31\% incl. poster]",
}

@Article{Hutter:18aixicplexx,
  author =       "Jan Leike and Marcus Hutter",
  title =        "On the computability of {S}olomonoff induction and {AIXI}",
  journal =      "Theoretical Computer Science",
  volume =       "716",
  pages =        "28--49",
  publisher =    "Elsevier",
  year =         "2018",
  bibtex =       "http://www.hutter1.net/official/bib.htm#aixicplexx",
  pdf =          "http://www.hutter1.net/publ/aixicplexx.pdf",
  slides =       "http://www.hutter1.net/publ/saixicplex.pdf",
  project =      "http://www.hutter1.net/official/projects.htm#uai",
  issn =         "????-????",
  doi =          "10.1016/j.tcs.2017.11.020",
  keywords =     "Solomonoff induction; AIXI; General reinforcement learning;
                  Knowledge-seeking agents; Computability; Arithmetical hierarchy.",
  abstract =     "How could we solve the machine learning and the artificial
                  intelligence problem if we had infinite computation? Solomonoff
                  induction and the reinforcement learning agent AIXI are proposed
                  answers to this question. Both are known to be incomputable. We
                  quantify this using the arithmetical hierarchy, and prove upper and
                  in most cases corresponding lower bounds for incomputability.
                  Moreover, we show that AIXI is not limit computable, thus it cannot
                  be approximated using finite computation. However there are limit
                  computable epsilon-optimal approximations to AIXI. We also derive
                  computability bounds for knowledge-seeking agents, and give a limit
                  computable weakly asymptotically optimal reinforcement learning
                  agent.",
  support =      "ARC grant DP150104590",
  for =          "080101(50%),080201(50%)",
  seo =          "970108(100%)",
}

@InProceedings{Hutter:13ksaprob,
  author =       "Laurent Orseau and Tor Lattimore and Marcus Hutter",
  title =        "Universal Knowledge-Seeking Agents for Stochastic Environments",
  booktitle =    "Proc. 24th International Conf. on Algorithmic Learning Theory ({ALT'13})",
  address =      "Singapore",
  series =       "LNAI",
  volume =       "8139",
  _editor =       "S. Jain and R. Munos and F. Stephan and Th. Zeugmann",
  publisher =    "Springer",
  pages =        "158--172",
  _month =        oct,
  year =         "2013",
  bibtex =       "http://www.hutter1.net/official/bib.htm#ksaprob",
  conf =         "http://www-alg.ist.hokudai.ac.jp/~thomas/ALT13/",
  pdf =          "http://www.hutter1.net/publ/ksaprob.pdf",
  slides =       "http://www.hutter1.net/publ/sksaprob.pdf",
  project =      "http://www.hutter1.net/official/projects.htm#uai",
  doi =          "10.1007/978-3-642-40935-6_12",
  issn =         "0302-9743",
  isbn =         "978-3-642-40934-9",
  keywords =     "Universal artificial intelligence; exploration; reinforcement learning;
                  algorithmic information theory; Solomonoff induction.",
  abstract =     "We define an optimal Bayesian knowledge-seeking agent, KL-KSA,
                  designed for countable hypothesis classes of stochastic
                  environments and whose goal is to gather as much information about
                  the unknown world as possible. Although this agent works for
                  arbitrary countable classes and priors, we focus on the especially
                  interesting case where all stochastic computable environments are
                  considered and the prior is based on Solomonoff's universal prior.
                  Among other properties, we show that KL-KSA learns the true
                  environment in the sense that it learns to predict the consequences
                  of actions it does not take. We show that it does not consider
                  noise to be information and avoids taking actions leading to
                  inescapable traps. We also present a variety of toy experiments
                  demonstrating that KL-KSA behaves according to expectation.",
  for =          "080101(50%),080199(50%)",
  seo =          "970108(100%)",
  znote =        "Acceptance rate: 23/39 = 59\%",
}

@Article{Hutter:15ratagentx,
  author =       "Peter Sunehag and Marcus Hutter",
  title =        "Rationality, Optimism and Guarantees in General Reinforcement Learning",
  journal =      "Journal of Machine Learning Research",
  volume =       "16",
  pages =        "1345--1390",
  publisher =    "Microtome",
  _address =     "Princeton, NJ, USA",
  _month =        aug,
  year =         "2015",
  bibtex =       "http://www.hutter1.net/official/bib.htm#ratagentx",
  pdf =          "http://www.hutter1.net/publ/ratagentx.pdf",
  slides =       "http://www.hutter1.net/publ/sagproblaws.pdf",
  slides =       "http://www.hutter1.net/publ/soptcog.pdf",
  slides =       "http://www.hutter1.net/publ/sagscilaws.pdf",
  slides =       "http://www.hutter1.net/publ/saixiopt.pdf",
  slides =       "http://www.hutter1.net/publ/soptopt.pdf",
  slides =       "http://www.hutter1.net/publ/saixiaxiom.pdf",
  slides =       "http://www.hutter1.net/publ/saixiaxiom2.pdf",
  project =      "http://www.hutter1.net/official/projects.htm#uai",
  issn =         "1532-4435",
  keywords =     "Reinforcement Learning, Rationality, Optimism, Optimality, Error bounds",
  abstract =     "In this article, we present a top-down theoretical study of general
                  reinforcement learning agents. We begin with rational agents with
                  unlimited resources and then move to a setting where an agent can
                  only maintain a limited number of hypotheses and optimizes plans
                  over a horizon much shorter than what the agent designer actually
                  wants. We axiomatize what is rational in such a setting in a manner
                  that enables optimism, which is important to achieve systematic
                  explorative behavior. Then, within the class of agents deemed
                  rational, we achieve convergence and finite-error bounds. Such
                  results are desirable since they imply that the agent learns well
                  from its experiences, but the bounds do not directly guarantee good
                  performance and can be achieved by agents doing things one should
                  obviously not. Good performance cannot in fact be guaranteed for
                  any agent in fully general settings. Our approach is to design
                  agents that learn well from experience and act rationally. We
                  introduce a framework for general reinforcement learning agents
                  based on rationality axioms for a decision function and an
                  hypothesis-generating function designed so as to achieve guarantees
                  on the number errors. We will consistently use an optimistic
                  decision function but the hypothesis-generating function needs to
                  change depending on what is known/assumed. We investigate a number
                  of natural situations having either a frequentist or Bayesian flavor,
                  deterministic or stochastic environments and either finite or
                  countable hypothesis class. Further, to achieve sufficiently good
                  bounds as to hold promise for practical success we introduce a
                  notion of a class of environments being generated by a set of laws.
                  None of the above has previously been done for fully general
                  reinforcement learning environments.",
  support =      "ARC grant DP120100950",
  for =          "080101(60%),010404(30%),220302(10%)",
  seo =          "970108(90%),970122(10%)",
}

@InProceedings{Hutter:16speedprior,
  author =       "Daniel Filan and Jan Leike and Marcus Hutter",
  title =        "Loss Bounds and Time Complexity for Speed Priors",
  booktitle =    "Proc. 19th International Conf. on Artificial Intelligence and Statistics ({AISTATS'16})",
  address =      "Cadiz, Spain",
  volume =       "51",
  _editor =       "Arthur Gretton and Christian Robert",
  publisher =    "Microtome",
  pages =        "1394--1402",
  _month =        may,
  year =         "2016",
  bibtex =       "http://www.hutter1.net/official/bib.htm#speedprior",
  http =         "http://jmlr.org/proceedings/papers/v51/",
  url =          "http://arxiv.org/abs/1604.03343",
  pdf =          "http://www.hutter1.net/publ/speedprior.pdf",
  latex =        "http://www.hutter1.net/publ/speedprior.tex",
  project =      "http://www.hutter1.net/official/projects.htm#ait",
  issn =         "0302-9743",
  keywords =     "universal distribution; speed prior; computational complexity; predictive performance; upper bounds.",
  abstract =     "This paper establishes for the first time the predictive
                  performance of speed priors and their computational complexity. A
                  speed prior is essentially a probability distribution that puts low
                  probability on strings that are not efficiently computable. We
                  propose a variant to the original speed prior (Schmidhuber, 2002),
                  and show that our prior can predict sequences drawn from
                  probability measures that are estimable in polynomial time. Our
                  speed prior is computable in doubly-exponential time, but not in
                  polynomial time. On a polynomial time computable sequence our speed
                  prior is computable in exponential time. We show better upper
                  complexity bounds for Schmidhuber's speed prior under the same
                  conditions, and that it predicts deterministic sequences that are
                  computable in polynomial time; however, we also show that it is not
                  computable in polynomial time, and the question of its predictive
                  properties for stochastic sequences remains open.",
  support =      "ARC grant DP150104590",
  for =          "080101(20%),080201(30%),080401(20%),010404(30%)",
  seo =          "970108(100%)",
  znote =        "Acceptance rate: 165/537 = 31\%",
}

@InProceedings{Hutter:09mdltvp,
  author =       "Marcus Hutter",
  title =        "Discrete {MDL} Predicts in Total Variation",
  booktitle =    "Advances in Neural Information Processing Systems 22 ({NIPS'09})",
  pages =        "817--825",
  _editor =      "Y. Bengio and D. Schuurmans and J. Lafferty and C. K. I. Williams and A. Culotta",
  publisher =    "Curran Associates",
  address =      "Cambridge, MA, USA",
  _month =        dec,
  year =         "2009",
  bibtex =       "http://www.hutter1.net/official/bib.htm#mdltvp",
  pdf =          "http://www.hutter1.net/ai/mdltvp.pdf",
  ps =           "http://www.hutter1.net/ai/mdltvp.ps",
  latex =        "http://www.hutter1.net/ai/mdltvp.tex",
  slides =       "http://www.hutter1.net/ai/smdltvp.pdf",
  project =      "http://www.hutter1.net/official/projects.htm#mdl",
  isbn =         "1615679111",
  keywords =     "minimum description length; countable model class;
                  total variation distance; sequence prediction;
                  discriminative learning; reinforcement learning.",
  abstract =     "The Minimum Description Length (MDL) principle selects the model
                  that has the shortest code for data plus model. We show that for a
                  countable class of models, MDL predictions are close to the true
                  distribution in a strong sense. The result is completely general. No
                  independence, ergodicity, stationarity, identifiability, or other
                  assumption on the model class need to be made. More formally, we
                  show that for any countable class of models, the distributions
                  selected by MDL (or MAP) asymptotically predict (merge
                  with) the true measure in the class in total variation distance.
                  Implications for non-i.i.d. domains like time-series forecasting,
                  discriminative learning, and reinforcement learning are discussed.",
  for =          "080401(30%),010405(50%),080198(20%)",
  znote =        "Acceptance rate: 263/1105 = 24\%",
}

@article{irving2018ai,
  title={{AI} safety via debate},
  author={Irving, Geoffrey and Christiano, Paul and Amodei, Dario},
  journal={arXiv preprint arXiv:1805.00899},
  year={2018}
}

@article{iyengar2005robust,
  title={Robust dynamic programming},
  author={Iyengar, Garud N},
  journal={Mathematics of Operations Research},
  volume={30},
  number={2},
  pages={257--280},
  year={2005},
  publisher={INFORMS}
}

@inproceedings{jiang1998minimax,
  title={Minimax-based reinforcement learning with state aggregation},
  author={Jiang, Guofei and Wu, Cang-Pu and Cybenko, George},
  booktitle={Proceedings of the 37th IEEE Conference on Decision and Control (Cat. No. 98CH36171)},
  volume={2},
  pages={1236--1241},
  year={1998},
  organization={IEEE}
}

@article{khani2016unanimous,
  title={Unanimous prediction for 100\% precision with application to learning semantic mappings},
  author={Khani, Fereshte and Rinard, Martin and Liang, Percy},
  journal={arXiv preprint arXiv:1606.06368},
  year={2016}
}

@article{kirkpatrick2017overcoming,
  title={Overcoming catastrophic forgetting in neural networks},
  author={Kirkpatrick, James and Pascanu, Razvan and Rabinowitz, Neil and Veness, Joel and Desjardins, Guillaume and Rusu, Andrei A and Milan, Kieran and Quan, John and Ramalho, Tiago and Grabska-Barwinska, Agnieszka and others},
  journal={Proceedings of the national academy of sciences},
  pages={201611835},
  year={2017},
  publisher={National Acad Sciences}
}

@inproceedings{klyubin2005empowerment,
  title={Empowerment: A universal agent-centric measure of control},
  author={Klyubin, Alexander S and Polani, Daniel and Nehaniv, Chrystopher L},
  booktitle={2005 IEEE Congress on Evolutionary Computation},
  volume={1},
  pages={128--135},
  year={2005},
  organization={IEEE}
}

@article{kosoy2019safeml,
  Author = {Kosoy, Vanessa},
  Journal = {Safe Machine Learning workshop at {ICLR}},
  Title = {Delegative Reinforcement Learning: learning to avoid traps with a little help},
  Year = {2019}}


@misc{krakovna_2018, title={Specification Gaming Examples in {AI}}, howpublished="\path{https://vkrakovna.wordpress.com/2018/04/02/specification-gaming-examples-in-ai/}", author={Krakovna, Victoria}, year={2018}}

@article{lai1985asymptotically,
  title={Asymptotically efficient adaptive allocation rules},
  author={Lai, Tze Leung and Robbins, Herbert},
  journal={Advances in applied mathematics},
  volume={6},
  number={1},
  pages={4--22},
  year={1985},
  publisher={Academic Press}
}

@misc{leaderboard,
    author       = {OpenAI},
    title        = {{Leaderboard}},
    month        = Sep,
    year         = 2019,
    publisher    = {Github},
    howpublished = "\path{https://github.com/openai/gym/wiki/Leaderboard}"
    }

@article{legg2007universal,
  title={Universal intelligence: A definition of machine intelligence},
  author={Legg, Shane and Hutter, Marcus},
  journal={Minds and machines},
  volume={17},
  number={4},
  pages={391--444},
  year={2007},
  publisher={Springer}
}

@inproceedings{mandel2017add,
  title={Where to Add Actions in Human-in-the-Loop Reinforcement Learning.},
  author={Mandel, Travis and Liu, Yun-En and Brunskill, Emma and Popovic, Zoran},
  booktitle={AAAI},
  pages={2322--2328},
  year={2017}
}

@book{mansfield1971technological,
  title={Technological change:an introduction to a vital area of modern economics},
  author={Mansfield, Edwin},
  year={1971},
  publisher={Norton}
}

@InProceedings{minsky_1961, edition={1}, title={Steps toward artificial intelligence}, volume={49}, booktitle={Proceedings of the IRE}, author={Minsky, Marvin}, year={1961}, pages={8–30}}

@article{mnih2015human,
  title={Human-level control through deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Rusu, Andrei A and Veness, Joel and Bellemare, Marc G and Graves, Alex and Riedmiller, Martin and Fidjeland, Andreas K and Ostrovski, Georg and others},
  journal={Nature},
  volume={518},
  number={7540},
  pages={529},
  year={2015},
  publisher={Nature Publishing Group}
}

@InProceedings{mnih2016asynchronous,
  title={Asynchronous methods for deep reinforcement learning},
  author={Mnih, Volodymyr and Badia, Adria Puigdomenech and Mirza, Mehdi and Graves, Alex and Lillicrap, Timothy and Harley, Tim and Silver, David and Kavukcuoglu, Koray},
  booktitle={International conference on machine learning},
  pages={1928--1937},
  year={2016}
}

@incollection{muller2016future,
  title={Future progress in artificial intelligence: A survey of expert opinion},
  author={M{\"u}ller, Vincent C and Bostrom, Nick},
  booktitle={Fundamental issues of artificial intelligence},
  pages={555--572},
  year={2016},
  publisher={Springer}
}

@inproceedings{ng2000algorithms,
  title={Algorithms for inverse reinforcement learning.},
  author={Ng, Andrew Y and Russell, Stuart J},
  booktitle={Icml},
  pages={663--670},
  year={2000}
}

@article{nilim2005robust,
  title={Robust control of {M}arkov decision processes with uncertain transition matrices},
  author={Nilim, Arnab and El Ghaoui, Laurent},
  journal={Operations Research},
  volume={53},
  number={5},
  pages={780--798},
  year={2005},
  publisher={INFORMS}
}

@InProceedings{omohundro_2008, title={The basic {AI} drives}, volume={171}, booktitle={Artificial General Intelligence}, author={Omohundro, Steve M.}, year={2008}, pages={483–492}}

@InProceedings{ring_orseau_2011, title={Delusion, survival, and intelligent agents}, booktitle={Artificial General Intelligence}, publisher={Springer}, author={Ring, Mark and Orseau, Laurent}, year={2011}, pages={11–20}}

@article{russell_dewey_tegmark_2015, title={Research Priorities for Robust and Beneficial Artificial Intelligence: An Open Letter}, volume={36}, number={4}, journal={AI Magazine}, author={Rusell, Stuart and Dewey, Daniel and Tegmark, Max}, year={2015}, pages={105–114}}

@article{russell2014myths,
  title={Of myths and moonshine},
  author={Russell, Stuart},
  journal={Edge (blog comment)},
  url={http://edge.org/conversation/the-myth-of-ai#26015},
  year={2014}
}

@inproceedings{ross2011reduction,
  title={A reduction of imitation learning and structured prediction to no-regret online learning},
  author={Ross, St{\'e}phane and Gordon, Geoffrey and Bagnell, Drew},
  booktitle={Proceedings of the fourteenth international conference on artificial intelligence and statistics},
  pages={627--635},
  year={2011}
}

@inproceedings{sadigh2016information,
  title={Information gathering actions over human internal state},
  author={Sadigh, Dorsa and Sastry, S Shankar and Seshia, Sanjit A and Dragan, Anca},
  booktitle={Intelligent Robots and Systems (IROS), 2016 IEEE/RSJ International Conference on},
  pages={66--73},
  year={2016},
  organization={IEEE}
}

@inproceedings{saunders2018trial,
  title={Trial without error: Towards safe reinforcement learning via human intervention},
  author={Saunders, William and Sastry, Girish and Stuhlmueller, Andreas and Evans, Owain},
  booktitle={Proceedings of the 17th International Conference on Autonomous Agents and MultiAgent Systems},
  pages={2067--2069},
  year={2018},
  organization={International Foundation for Autonomous Agents and Multiagent Systems}
}

@inproceedings{schmidhuber2002speed,
  title={The Speed Prior: a new simplicity measure yielding near-optimal computable predictions},
  author={Schmidhuber, J{\"u}rgen},
  booktitle={International Conference on Computational Learning Theory},
  pages={216--228},
  year={2002},
  organization={Springer}
}

@book{shanahan_2015, place={Cambridge, MA}, title={The technological singularity}, publisher={The MIT Press}, author={Shanahan, Murray}, year={2015}}

@book{shannon_weaver_1949, place={Urbana}, title={The mathematical theory of communication}, publisher={University of Illinois Press}, author={Shannon, Claude Elwood and Weaver, Warren}, year={1949}}

@article{silver2017mastering,
  title={Mastering chess and shogi by self-play with a general reinforcement learning algorithm},
  author={Silver, David and Hubert, Thomas and Schrittwieser, Julian and Antonoglou, Ioannis and Lai, Matthew and Guez, Arthur and Lanctot, Marc and Sifre, Laurent and Kumaran, Dharshan and Graepel, Thore and others},
  journal={arXiv preprint arXiv:1712.01815},
  year={2017}
}

@InProceedings{soares_fallenstein_yudkowsky_2015, title={Corrigibility}, booktitle={Workshops at the Twenty-Ninth AAAI Conference on Artificial Intelligence}, author={Soares, Nate and Fallenstein, Benja and Yudkowsky, Eliezer}, year={2015}}

@article{solomonoff_1964, title={A formal theory of inductive inference. Part I}, volume={7}, DOI={10.1016/s0019-9958(64)90131-7}, number={1}, journal={Information and Control}, author={Solomonoff, Ray J.}, year={1964}, pages={1–22}}

@book{sutton_barto_1998, place={Cambridge}, title={Reinforcement learning: An introduction}, publisher={MIT Press}, author={Sutton, Richard S. and Barto, Andrew G.}, year={1998}}

@article{tamar2013scaling,
  title={Scaling up robust {MDP}s by reinforcement learning},
  author={Tamar, Aviv and Xu, Huan and Mannor, Shie},
  journal={arXiv preprint arXiv:1306.6189},
  year={2013}
}

@article{taylor2016alignment,
  title={Alignment for advanced machine learning systems},
  author={Taylor, Jessica and Yudkowsky, Eliezer and LaVictoire, Patrick and Critch, Andrew},
  journal={Machine Intelligence Research Institute},
  year={2016}
}

@inproceedings{thomas2016data,
  title={Data-efficient off-policy policy evaluation for reinforcement learning},
  author={Thomas, Philip and Brunskill, Emma},
  booktitle={International Conference on Machine Learning},
  pages={2139--2148},
  year={2016}
}

@article{thompson1933likelihood,
  title={On the likelihood that one unknown probability exceeds another in view of the evidence of two samples},
  author={Thompson, William R},
  journal={Biometrika},
  volume={25},
  number={3/4},
  pages={285--294},
  year={1933},
  publisher={JSTOR}
}

@article{veness2011monte,
  title={A monte-carlo aixi approximation},
  author={Veness, Joel and Ng, Kee Siong and Hutter, Marcus and Uther, William and Silver, David},
  journal={Journal of Artificial Intelligence Research},
  volume={40},
  pages={95--142},
  year={2011}
}

@article{neumann1944theory,
  title={Theory of games and economic behavior},
  author={Neumann Von J, Morgenstern O},
  journal={Princeton University Prress, Princeton},
  year={1944}
}

@book{wasserman1994social,
  title={Social network analysis: Methods and applications},
  author={Wasserman, Stanley and Faust, Katherine and others},
  volume={8},
  year={1994},
  publisher={Cambridge university press}
}

@article{whittle_1979,
  author={Whittle, Peter},
  title={Discussion of {D}r {G}ittins' paper},
  year={1979},
  journal={Journal of the Royal Statistical Society},
  volume={41},
  pages={164-177}
}

@article{yeung2001validating,
  title={Validating clustering for gene expression data},
  author={Yeung, Ka Yee and Haynor, David R. and Ruzzo, Walter L.},
  journal={Bioinformatics},
  volume={17},
  number={4},
  pages={309--318},
  year={2001},
  publisher={Oxford University Press}
}

@article{yudkowsky_2008,
  title={Artificial intelligence as a positive and negative factor in global risk},
  author={Yudkowsky, Eliezer},
  journal={Global catastrophic risks},
  volume={1},
  number={303},
  pages={184},
  year={2008},
  publisher={Oxford University Press Oxford, UK}
}

@misc{yudkowsky_2002,
  title={The AI-box experiment},
  author={Yudkowsky, Eliezer},
  url={http://yudkowsky.net/singularity/aibox},
  year={2002}
}

@article{zvonkin_1970,
  title={The complexity of finite objects and the development of the concepts of information and randomness by means of the theory of algorithms},
  author={Zvonkin, Alexander K and Levin, Leonid A},
  journal={Russian Mathematical Surveys},
  volume={25},
  number={6},
  pages={83--124},
  year={1970},
  publisher={Turpion Ltd}
}

@phdthesis{kraft1949device,
  title={A device for quantizing, grouping, and coding amplitude-modulated pulses},
  author={Kraft, Leon Gordon},
  year={1949},
  school={Massachusetts Institute of Technology}
}

@incollection{de2011luckiness,
  title={Luckiness and regret in minimum description length inference},
  author={De Rooij, Steven and Gr{\"u}nwald, Peter D},
  booktitle={Philosophy of Statistics},
  pages={865--900},
  year={2011},
  publisher={Elsevier}
}

@inproceedings{cohen2020pessimism,
  title={Pessimism About Unknown Unknowns Inspires Conservatism},
  author={Cohen, Michael K and Hutter, Marcus},
  booktitle={Conference on Learning Theory},
  pages={1344--1373},
  year={2020}
}

@article{cohen2022fully,
  title={Fully general online imitation learning},
  author={Cohen, Michael K and Hutter, Marcus and Nanda, Neel},
  journal={The Journal of Machine Learning Research},
  volume={23},
  number={1},
  pages={15066--15095},
  year={2022},
  publisher={JMLRORG}
}

@article{moskovitz2023confronting,
  title={Confronting Reward Model Overoptimization with Constrained {RLHF}},
  author={Moskovitz, Ted and Singh, Aaditya K and Strouse, DJ and Sandholm, Tuomas and Salakhutdinov, Ruslan and Dragan, Anca D and McAleer, Stephen},
  journal={arXiv preprint arXiv:2310.04373},
  year={2023}
}

@inproceedings{laidlaw2023preventing,
  title={Preventing reward hacking with occupancy measure regularization},
  author={Laidlaw, Cassidy and Singhal, Shivam and Dragan, Anca},
  booktitle={ICML Workshop on New Frontiers in Learning, Control, and Dynamical Systems},
  year={2023}
}

@inproceedings{yang2021accelerating,
  title={Accelerating safe reinforcement learning with constraint-mismatched baseline policies},
  author={Yang, Tsung-Yen and Rosca, Justinian and Narasimhan, Karthik and Ramadge, Peter J},
  booktitle={International Conference on Machine Learning},
  pages={11795--11807},
  year={2021},
  organization={PMLR}
}

@article{vieillard2020leverage,
  title={Leverage the average: an analysis of kl regularization in reinforcement learning},
  author={Vieillard, Nino and Kozuno, Tadashi and Scherrer, Bruno and Pietquin, Olivier and Munos, R{\'e}mi and Geist, Matthieu},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={12163--12174},
  year={2020}
}

@article{stiennon2020learning,
  title={Learning to summarize with human feedback},
  author={Stiennon, Nisan and Ouyang, Long and Wu, Jeffrey and Ziegler, Daniel and Lowe, Ryan and Voss, Chelsea and Radford, Alec and Amodei, Dario and Christiano, Paul F},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={3008--3021},
  year={2020}
}

@article{ouyang2022training,
  title={Training language models to follow instructions with human feedback},
  author={Ouyang, Long and Wu, Jeffrey and Jiang, Xu and Almeida, Diogo and Wainwright, Carroll and Mishkin, Pamela and Zhang, Chong and Agarwal, Sandhini and Slama, Katarina and Ray, Alex and others},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={27730--27744},
  year={2022}
}

@article{bai2022training,
  title={Training a helpful and harmless assistant with reinforcement learning from human feedback},
  author={Bai, Yuntao and Jones, Andy and Ndousse, Kamal and Askell, Amanda and Chen, Anna and DasSarma, Nova and Drain, Dawn and Fort, Stanislav and Ganguli, Deep and Henighan, Tom and others},
  journal={arXiv preprint arXiv:2204.05862},
  year={2022}
}

@inproceedings{taylor2016quantilizers,
  title={Quantilizers: A Safer Alternative to Maximizers for Limited Optimization.},
  author={Taylor, Jessica},
  booktitle={AAAI Workshop: AI, Ethics, and Society},
  year={2016}
}

@article{carey2019how,
  Author = {Carey, Ryan},
  Journal = {Safe Machine Learning workshop at {ICLR}},
  Title = {How useful is quantilization for mitigating specification-gaming?},
  Year = {2019}}

@inproceedings{gao2023scaling,
  title={Scaling laws for reward model overoptimization},
  author={Gao, Leo and Schulman, John and Hilton, Jacob},
  booktitle={International Conference on Machine Learning},
  pages={10835--10866},
  year={2023},
  organization={PMLR}
}

@inproceedings{korbak2022rl,
  title={{RL} with {KL} penalties is better viewed as {B}ayesian inference},
  author={Korbak, Tomasz and Perez, Ethan and Buckley, Christopher},
  booktitle={Findings of the Association for Computational Linguistics: EMNLP 2022},
  pages={1083--1091},
  year={2022}
}

@article{ziegler2019fine,
  title={Fine-tuning language models from human preferences},
  author={Ziegler, Daniel M and Stiennon, Nisan and Wu, Jeffrey and Brown, Tom B and Radford, Alec and Amodei, Dario and Christiano, Paul and Irving, Geoffrey},
  journal={arXiv preprint arXiv:1909.08593},
  year={2019}
}

@article{perez2022red,
  title={Red teaming language models with language models},
  author={Perez, Ethan and Huang, Saffron and Song, Francis and Cai, Trevor and Ring, Roman and Aslanides, John and Glaese, Amelia and McAleese, Nat and Irving, Geoffrey},
  journal={arXiv preprint arXiv:2202.03286},
  year={2022}
}

@inproceedings{jaques2017sequence,
  title={Sequence tutor: Conservative fine-tuning of sequence generation models with kl-control},
  author={Jaques, Natasha and Gu, Shixiang and Bahdanau, Dzmitry and Hern{\'a}ndez-Lobato, Jos{\'e} Miguel and Turner, Richard E and Eck, Douglas},
  booktitle={International Conference on Machine Learning},
  pages={1645--1654},
  year={2017},
  organization={PMLR}
}

@article{jaques2019way,
  title={Way off-policy batch deep reinforcement learning of implicit human preferences in dialog},
  author={Jaques, Natasha and Ghandeharioun, Asma and Shen, Judy Hanwen and Ferguson, Craig and Lapedriza, Agata and Jones, Noah and Gu, Shixiang and Picard, Rosalind},
  journal={arXiv preprint arXiv:1907.00456},
  year={2019}
}

@article{schulman2017proximal,
  title={Proximal policy optimization algorithms},
  author={Schulman, John and Wolski, Filip and Dhariwal, Prafulla and Radford, Alec and Klimov, Oleg},
  journal={arXiv preprint arXiv:1707.06347},
  year={2017}
}

@article{schmitt2018kickstarting,
  title={Kickstarting deep reinforcement learning},
  author={Schmitt, Simon and Hudson, Jonathan J and Zidek, Augustin and Osindero, Simon and Doersch, Carl and Czarnecki, Wojciech M and Leibo, Joel Z and Kuttler, Heinrich and Zisserman, Andrew and Simonyan, Karen and others},
  journal={arXiv preprint arXiv:1803.03835},
  year={2018}
}

@misc{Sutskever_2018, title={Meta Learning and Self Play}, url={https://www.youtube.com/watch?v=RvEwFvl-TrY&amp;t=196s}, journal={Berkeley EECS Colloquium}, author={Sutskever, Ilya}, year={2018}, month={Jan}} 

@misc{Sutskever_2023, title={An Observation on Generalization
}, url={https://simons.berkeley.edu/talks/ilya-sutskever-openai-2023-08-14}, journal={Large Language Models and Transformers}, author={Sutskever, Ilya}, year={2023}, month={Aug}} 

@inproceedings{zhuang2020consequences,
 author = {Zhuang, Simon and Hadfield-Menell, Dylan},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M.F. Balcan and H. Lin},
 pages = {15763--15773},
 publisher = {Curran Associates, Inc.},
 title = {Consequences of Misaligned {AI}},
 volume = {33},
 year = {2020}
}

@inproceedings{lee2021pebble,
  title={PEBBLE: Feedback-Efficient Interactive Reinforcement Learning via Relabeling Experience and Unsupervised Pre-training},
  author={Lee, Kimin and Smith, Laura and Abbeel, Pieter},
  booktitle={38th International Conference on Machine Learning, ICML 2021},
  year={2021},
  organization={International Machine Learning Society (IMLS)}
}

@inproceedings{mishra2017prediction,
  title={Prediction and control with temporal segment models},
  author={Mishra, Nikhil and Abbeel, Pieter and Mordatch, Igor},
  booktitle={International conference on machine learning},
  pages={2459--2468},
  year={2017},
  organization={PMLR}
}

@inproceedings{baker2019emergent,
  title={Emergent Tool Use From Multi-Agent Autocurricula},
  author={Baker, Bowen and Kanitscheider, Ingmar and Markov, Todor and Wu, Yi and Powell, Glenn and McGrew, Bob and Mordatch, Igor},
  booktitle={International Conference on Learning Representations},
  year={2019}
}

@article{cohen2024regulating,
  title={Regulating advanced artificial agents},
  author={Cohen, Michael K. and Kolt, Noam and Bengio, Yoshua and Hadfield, Gillian K. and Russell, Stuart},
  journal={Science},
  volume={384},
  number={6691},
  pages={36--38},
  year={2024}
}


@article{cohen2022advanced,
  title={Advanced artificial agents intervene in the provision of reward},
  author={Cohen, Michael K. and Hutter, Marcus and Osborne, Michael A.},
  journal={AI magazine},
  volume={43},
  number={3},
  pages={282--293},
  year={2022}
}

@inproceedings{turner2021optimal,
 author = {Turner, Alex and Smith, Logan and Shah, Rohin and Critch, Andrew and Tadepalli, Prasad},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {M. Ranzato and A. Beygelzimer and Y. Dauphin and P.S. Liang and J. Wortman Vaughan},
 pages = {23063--23074},
 publisher = {Curran Associates, Inc.},
 title = {Optimal Policies Tend To Seek Power},
 volume = {34},
 year = {2021}
}

@article{chaitin1975theory,
  title={A theory of program size formally identical to information theory},
  author={Chaitin, Gregory J},
  journal={Journal of the ACM (JACM)},
  volume={22},
  number={3},
  pages={329--340},
  year={1975},
  publisher={ACM New York, NY, USA}
}

@inproceedings{solomonoff1960preliminary,
  title={A preliminary report on a general theory of inductive inference},
  author={Solomonoff, Ray J},
  year={1960},
  organization={Citeseer}
}

@book{li2008introduction,
  title={An introduction to Kolmogorov complexity and its applications},
  author={Li, Ming and Vit{\'a}nyi, Paul and others},
  volume={3},
  year={2008},
  publisher={Springer}
}

@inproceedings{zhang2017query,
  title={Query-efficient imitation learning for end-to-end simulated driving},
  author={Zhang, Jiakai and Cho, Kyunghyun},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={31},
  number={1},
  year={2017}
}

@inproceedings{menda2019ensembledagger,
  title={Ensembledagger: A {B}ayesian approach to safe imitation learning},
  author={Menda, Kunal and Driggs-Campbell, Katherine and Kochenderfer, Mykel J},
  booktitle={2019 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},
  pages={5041--5048},
  year={2019},
  organization={IEEE}
}

@inproceedings{brown2018risk,
  title={Risk-aware active inverse reinforcement learning},
  author={Brown, Daniel S and Cui, Yuchen and Niekum, Scott},
  booktitle={Conference on Robot Learning},
  pages={362--372},
  year={2018},
  organization={PMLR}
}

@article{brown2020bayesian,
  title={Bayesian robust optimization for imitation learning},
  author={Brown, Daniel and Niekum, Scott and Petrik, Marek},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={2479--2491},
  year={2020}
}

@article{kolmogorov1963tables,
  title={On tables of random numbers},
  author={Kolmogorov, Andrei N},
  journal={Sankhy{\=a}: The Indian Journal of Statistics, Series A},
  pages={369--376},
  year={1963},
  publisher={JSTOR}
}

@inproceedings{chan2023harms,
  title={Harms from increasingly agentic algorithmic systems},
  author={Chan, Alan and Salganik, Rebecca and Markelius, Alva and Pang, Chris and Rajkumar, Nitarshan and Krasheninnikov, Dmitrii and Langosco, Lauro and He, Zhonghao and Duan, Yawen and Carroll, Micah and others},
  booktitle={Proceedings of the 2023 ACM Conference on Fairness, Accountability, and Transparency},
  pages={651--666},
  year={2023}
}

@article{zvonkin1970complexity,
  title={The complexity of finite objects and the development of the concepts of information and randomness by means of the theory of algorithms},
  author={Zvonkin, Alexander K and Levin, Leonid A},
  journal={Russian Mathematical Surveys},
  volume={25},
  number={6},
  pages={83},
  year={1970},
  publisher={IOP Publishing}
}

@article{sanh2019distilbert,
  title={Distil{BERT}, a distilled version of {BERT}: smaller, faster, cheaper and lighter},
  author={Sanh, Victor and Debut, Lysandre and Chaumond, Julien and Wolf, Thomas},
  journal={arXiv preprint arXiv:1910.01108},
  year={2019}
}

@article{jiang2024mixtral,
  title={Mixtral of experts},
  author={Jiang, Albert Q and Sablayrolles, Alexandre and Roux, Antoine and Mensch, Arthur and Savary, Blanche and Bamford, Chris and Chaplot, Devendra Singh and Casas, Diego de las and Hanna, Emma Bou and Bressand, Florian and others},
  journal={arXiv preprint arXiv:2401.04088},
  year={2024}
}

@article{liu2018intriguing,
  title={An intriguing failing of convolutional neural networks and the coordconv solution},
  author={Liu, Rosanne and Lehman, Joel and Molino, Piero and Petroski Such, Felipe and Frank, Eric and Sergeev, Alex and Yosinski, Jason},
  journal={Advances in neural information processing systems},
  volume={31},
  year={2018}
}