
@article{auer2002finite,
  title={Finite-time analysis of the multiarmed bandit problem},
  author={Auer, Peter and Cesa-Bianchi, Nicolo and Fischer, Paul},
  journal={Machine Learning},
  volume={47},
  number={2-3},
  pages={235--256},
  year={2002},
  publisher={Springer}
}

@inproceedings{nakhleh2020neurwin,
  title={Neur{WIN}: Neural {W}hittle Index Network for Restless Bandits via Deep {RL}},
  author={Nakhleh, K. and Ganji, S. and Hsieh, P.-C. and Hou, I.-H. and Shakkottai, S.},
  booktitle={NeurIPS},
  year={2021}
}

@inproceedings{adam2021double,
  title={Double Oracle Algorithm for Computing Equilibria in Continuous Games},
  author={Adam, Luk{\'a}{\v{s}} and Hor{\v{c}}{\'\i}k, Rostislav and Kasl, Tom{\'a}{\v{s}} and Kroupa, Tom{\'a}{\v{s}}},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  year={2021}
}


@article{gilbert2017double,
  title={A double oracle approach to minmax regret optimization problems with interval data},
  author={Gilbert, H. and Spanjaard, O.},
  journal={Eur. J. Oper. Res.},
  volume={262},
  number={3},
  pages={929--943},
  year={2017},
  publisher={Elsevier}
}

@article{yao2009evacuation,
  title={Evacuation transportation planning under uncertainty: a robust optimization approach},
  author={Yao, Tao and Mandala, Supreet Reddy and Do Chung, Byung},
  journal={Networks and Spatial Economics},
  volume={9},
  number={2},
  pages={171--189},
  year={2009},
  publisher={Springer}
}

@article{shortridge2017robust,
  title={Robust decision making in data scarce contexts: addressing data and model limitations for infrastructure planning under transient climate change},
  author={Shortridge, Julie and Guikema, Seth and Zaitchik, Ben},
  journal={Climatic Change},
  volume={140},
  number={2},
  pages={323--337},
  year={2017},
  publisher={Springer}
}

@article{visconti2015building,
  title={Building robust conservation plans},
  author={Visconti, Piero and Joppa, Lucas},
  journal={Conservation Biology},
  volume={29},
  number={2},
  pages={503--512},
  year={2015},
  publisher={Wiley Online Library}
}

@article{regan2005robust,
  title={Robust decision-making under severe uncertainty for conservation management},
  author={Regan, Helen M and Ben-Haim, Yakov and Langford, Bill and Wilson, William G and Lundberg, Per and Andelman, Sandy J and Burgman, Mark A},
  journal={Ecological applications},
  volume={15},
  number={4},
  pages={1471--1477},
  year={2005},
  publisher={Wiley Online Library}
}

@article{ghassemi2019practical,
  title={Practical guidance on artificial intelligence for health-care data},
  author={Ghassemi, Marzyeh and Naumann, Tristan and Schulam, Peter and Beam, Andrew L and Chen, Irene Y and Ranganath, Rajesh},
  journal={The Lancet Digital Health},
  volume={1},
  number={4},
  pages={e157--e159},
  year={2019},
  publisher={Elsevier}
}

@article{begoli2019need,
  title={The need for uncertainty quantification in machine-assisted medical decision making},
  author={Begoli, Edmon and Bhattacharya, Tanmoy and Kusnezov, Dimitri},
  journal={Nature Machine Intelligence},
  volume={1},
  number={1},
  pages={20--23},
  year={2019},
  publisher={Nature Publishing Group}
}

@inproceedings{jung2019regret,
  author    = {Y. H. Jung and
               A. Tewari},
  title     = {Regret Bounds for Thompson Sampling in Episodic Restless Bandit Problems},
  booktitle = {NeurIPS},
  pages     = {9005--9014},
  year      = {2019}
}

@article{moilanen2006planning,
  title={Planning for robust reserve networks using uncertainty analysis},
  author={Moilanen, Atte and Runge, Michael C and Elith, Jane and Tyre, Andrew and Carmel, Yohay and Fegraus, Eric and Wintle, Brendan A and Burgman, Mark and Ben-Haim, Yakov},
  journal={Ecological Modelling},
  volume={199},
  number={1},
  pages={115--124},
  year={2006},
  publisher={Elsevier}
}

@inproceedings{braziunas2007minimax,
  title={Minimax regret based elicitation of generalized additive utilities},
  author={Braziunas, D. and Boutilier, C.},
  year={2007},
  booktitle={UAI}
}

@inproceedings{nguyen2014regret,
  title={Regret-Based Optimization and Preference Elicitation for {Stackelberg} Security Games with Uncertainty.},
  author={Nguyen, T. H. and Yadav, A. and An, B. and Tambe, M. and Boutilier, C.},
  booktitle={AAAI},
  pages={756--762},
  year={2014}
}

@inproceedings{li2019robust,
  title={Robust multi-agent reinforcement learning via minimax deep deterministic policy gradient},
  author={Li, S. and Wu, Y. and Cui, X. and Dong, H. and Fang, F. and Russell, S.},
  booktitle={AAAI},
  volume={33},
  issue={01},
  pages={4213--4220},
  year={2019}
}

@inproceedings{pinto2017robust,
  title={Robust adversarial reinforcement learning},
  author={Pinto, L. and Davidson, J. and Sukthankar, R. and Gupta, A.},
  booktitle={ICML},
  pages={2817--2826},
  year={2017},
  organization={PMLR}
}

@article{silver2017mastering,
  title={Mastering the game of go without human knowledge},
  author={Silver, David and Schrittwieser, Julian and Simonyan, Karen and Antonoglou, Ioannis and Huang, Aja and Guez, Arthur and Hubert, Thomas and Baker, Lucas and Lai, Matthew and Bolton, Adrian and others},
  journal={Nature},
  volume={550},
  number={7676},
  pages={354--359},
  year={2017},
  publisher={Nature Publishing Group}
}

@article{mnih2015human,
  title={Human-level control through deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Rusu, Andrei A. and Veness, Joel and Bellemare, Marc G. and Graves, Alex and Riedmiller, Martin and Fidjeland, Andreas K. and Ostrovski, Georg and others},
  journal={Nature},
  volume={518},
  number={7540},
  pages={529--533},
  year={2015},
  publisher={Nature Publishing Group}
}

@inproceedings{garivier2016maximin,
  title={Maximin action identification: A new bandit framework for games},
  author={Garivier, Aur{\'e}lien and Kaufmann, Emilie and Koolen, Wouter M},
  booktitle={Conference on Learning Theory},
  pages={1028--1050},
  year={2016},
  organization={PMLR}
}

@inproceedings{ghosh2020learning,
  title={Learning and Fairness in Energy Harvesting: A Maximin Multi-Armed Bandits Approach},
  author={Ghosh, Debamita and Verma, Arun and Hanawal, Manjesh K},
  booktitle={2020 International Conference on Signal Processing and Communications (SPCOM)},
  pages={1--5},
  year={2020},
  organization={IEEE}
}

@article{lanctot2017unified,
  title={A unified game-theoretic approach to multiagent reinforcement learning},
  author={Lanctot, M. and Zambaldi, V. and Gruslys, A. and Lazaridou, A. and Tuyls, K. and P{\'e}rolat, J. and others},
  journal={NeurIPS},
  volume={30},
  pages={4190--4203},
  year={2017}
}

@inproceedings{xu2021robust,
  title={Robust Reinforcement Learning Under Minimax Regret for Green Security},
  author={Xu, L. and Perrault, A. and Fang, F. and Chen, H. and Tambe, M.},
  booktitle={UAI},
  year={2021},
}

% Journals

% First the Full Name is given, then the abbreviation used in the AMS Math
% Reviews, with an indication if it could not be found there.
% Note the 2nd overwrites the 1st, so swap them if you want the full name.


@inproceedings{killian2021Q,
  title={Q-Learning {L}agrange Policies for Multi-action Restless Bandits},
  author={Killian, J. A. and Biswas, A. and Shah, S. and Tambe, M.},
  booktitle={KDD},
  year={2021}
}

@inproceedings{biswas2021learn,
  title={Learn to Intervene: An Adaptive Learning Policy for Restless Bandits in
Application to Preventive Healthcare},
  author={Biswas, A. and Aggarwal, G. and Varakantham, P. and Tambe, M.},
  booktitle={IJCAI},
  year={2021}
}

@inproceedings{mate2021risk,
  title={Risk-Aware Interventions in Public Health: Planning with Restless Multi-Armed Bandits},
  author={Mate, Aditya and Perrault, Andrew and Tambe, Milind},
  booktitle={Proceedings of the 20th International Conference on Autonomous Agents and MultiAgent Systems},
  pages={880--888},
  year={2021}
}
%---------------------------------

@article{wei2021nonstationary,
  title={Nonstationary Stochastic Multiarmed Bandits: {UCB} Policies and Minimax Regret},
  author={Wei, Lai and Srivastava, Vaibhav},
  journal={arXiv preprint arXiv:2101.08980},
  year={2021}
}

@article{lakshminarayanan2017stability,
  title={A stability criterion for two timescale stochastic approximation schemes},
  author={Lakshminarayanan, Chandrashekar and Bhatnagar, Shalabh},
  journal={Automatica},
  volume={79},
  pages={108--114},
  year={2017},
  publisher={Elsevier}
}

@misc{notes,
  title = {Convex Functions},
  author = {Robert Gardner},
  howpublished = {\url{https://faculty.etsu.edu/gardnerr/5210/Beamer-Proofs/Proofs-6-6-print.pdf}},
  note = {Accessed: 2021-01-15}
}

@article{watkins1989learning,
  title={Learning from delayed rewards},
  author={Watkins, Christopher John Cornish Hellaby},
  year={1989},
  publisher={King's College, Cambridge}
}

@article{watkins1992q,
  title={Q-learning},
  author={Watkins, Christopher JCH and Dayan, Peter},
  journal={Machine learning},
  volume={8},
  number={3-4},
  pages={279--292},
  year={1992},
  publisher={Springer}
}

@inproceedings{iannello2012optimality,
  title={Optimality of myopic scheduling and whittle indexability for energy harvesting sensors},
  author={Iannello, Fabio and Simeone, Osvaldo and Spagnolini, Umberto},
  booktitle={2012 46th Annual Conference on Information Sciences and Systems (CISS)},
  pages={1--6},
  year={2012},
  organization={IEEE}
}

@article{avrachenkov2020whittle,
  title={Whittle index based {Q}-learning for restless bandits with average reward},
  author={Avrachenkov, K. and Borkar, V. S.},
  journal={arXiv preprint arXiv:2004.14427},
  year={2020}
}

@article{abounadi2001learning,
  title={Learning algorithms for {M}arkov decision processes with average cost},
  author={Abounadi, Jinane and Bertsekas, Dimitrib and Borkar, Vivek S.},
  journal={SIAM Journal on Control and Optimization},
  volume={40},
  number={3},
  pages={681--698},
  year={2001},
  publisher={SIAM}
}

@article{xiong2021learning,
  title={Learning Augmented Index Policy for Optimal Service Placement at the Network Edge},
  author={Xiong, Guojun and Singh, Rahul and Li, Jian},
  journal={arXiv preprint arXiv:2101.03641},
  year={2021}
}

@article{Liu2010,
  title={ Indexability of restless bandit problems and optimality of Whittle index for dynamic multichannel access},
  author={Liu, K. and Zhao, Q. },
  journal={IEEE Transactions on Information Theory},
  pages = {5547-5567},
  year={2010}
}

@inproceedings{Sombabu2020,
  title={Whittle index for AoI-aware scheduling},
  author={Sombabu, B. and Mate, A. and Manjunath, D. and Moharir, S.},
  booktitle={IEEE International Conference on Communication Systems \& Networks (COSMSNETS)},
  pages={},
  year={2020},
  organization={IEEE}
}

@article{jacko2016resource,
  title={Resource capacity allocation to stochastic dynamic competitors: knapsack problem for perishable items and index-knapsack heuristic},
  author={Jacko, Peter},
  journal={Annals of Operations Research},
  volume={241},
  number={1},
  pages={83--107},
  year={2016},
  publisher={Springer}
}

 %{AMS}
 @String{AMSTrans = "American Mathematical Society Translations" }
 @String{AMSTrans = "Amer. Math. Soc. Transl." }
 @String{BullAMS = "Bulletin of the American Mathematical Society" }
 @String{BullAMS = "Bull. Amer. Math. Soc." }
 @String{ProcAMS = "Proceedings of the American Mathematical Society" }
 @String{ProcAMS = "Proc. Amer. Math. Soc." }
 @String{TransAMS = "Transactions of the American Mathematical Society" }
 @String{TransAMS = "Trans. Amer. Math. Soc." }

 %ACM
 @String{CACM = "Communications of the {ACM}" }
 @String{CACM = "Commun. {ACM}" }
 @String{CompServ = "Comput. Surveys" }
 @String{JACM = "J. ACM" }
 @String{ACMMathSoft = "{ACM} Transactions on Mathematical Software" }
 @String{ACMMathSoft = "{ACM} Trans. Math. Software" }
 @String{SIGNUM = "{ACM} {SIGNUM} Newsletter" }
 @String{SIGNUM = "{ACM} {SIGNUM} Newslett." }

 @String{AmerSocio = "American Journal of Sociology" }
 @String{AmerStatAssoc = "Journal of the American Statistical Association" }
 @String{AmerStatAssoc = "J. Amer. Statist. Assoc." }
 @String{ApplMathComp = "Applied Mathematics and Computation" }
 @String{ApplMathComp = "Appl. Math. Comput." }
 @String{AmerMathMonthly = "American Mathematical Monthly" }
 @String{AmerMathMonthly = "Amer. Math. Monthly" }
 @String{BIT = "{BIT}" }
 @String{BritStatPsych = "British Journal of Mathematical and Statistical
          Psychology" }
 @String{BritStatPsych = "Brit. J. Math. Statist. Psych." }
 @String{CanMathBull = "Canadian Mathematical Bulletin" }
 @String{CanMathBull = "Canad. Math. Bull." }
 @String{CompApplMath = "Journal of Computational and Applied Mathematics" }
 @String{CompApplMath = "J. Comput. Appl. Math." }
 @String{CompPhys = "Journal of Computational Physics" }
 @String{CompPhys = "J. Comput. Phys." }
 @String{CompStruct = "Computers and Structures" }
 @String{CompStruct = "Comput. \& Structures" }
 @String{CompJour = "The Computer Journal" }
 @String{CompJour = "Comput. J." }
 @String{CompSysSci = "Journal of Computer and System Sciences" }
 @String{CompSysSci = "J. Comput. System Sci." }
 @String{Computing = "Computing" }
 @String{ContempMath = "Contemporary Mathematics" }
 @String{ContempMath = "Contemp. Math." }
 @String{Crelle = "Crelle's Journal" }
 @String{GiornaleMath = "Giornale di Mathematiche" }
 @String{GiornaleMath = "Giorn. Mat." } % didn't find in AMS MR., ibid.

 %IEEE
 @String{Computer = "{IEEE} Computer" }
 @String{IEEETransComp = "{IEEE} Transactions on Computers" }
 @String{IEEETransComp = "{IEEE} Trans. Comput." }
 @String{IEEETransAC = "{IEEE} Transactions on Automatic Control" }
 @String{IEEETransAC = "{IEEE} Trans. Automat. Control" }
 @String{IEEESpec = "{IEEE} Spectrum" } % didn't find in AMS MR
 @String{ProcIEEE = "Proceedings of the {IEEE}" }
 @String{ProcIEEE = "Proc. {IEEE}" } % didn't find in AMS MR
 @String{IEEETransAeroElec = "{IEEE} Transactions on Aerospace and Electronic
     Systems" }
 @String{IEEETransAeroElec = "{IEEE} Trans. Aerospace Electron. Systems" }

 @String{IMANumerAna = "{IMA} Journal of Numerical Analysis" }
 @String{IMANumerAna = "{IMA} J. Numer. Anal." }
 @String{InfProcLet = "Information Processing Letters" }
 @String{InfProcLet = "Inform. Process. Lett." }
 @String{InstMathApp = "Journal of the Institute of Mathematics and
     its Applications" }
 @String{InstMathApp = "J. Inst. Math. Appl." }
 @String{IntControl = "International Journal of Control" }
 @String{IntControl = "Internat. J. Control" }
 @String{IntNumerEng = "International Journal for Numerical Methods in
     Engineering" }
 @String{IntNumerEng = "Internat. J. Numer. Methods Engrg." }
 @String{IntSuper = "International Journal of Supercomputing Applications" }
 @String{IntSuper = "Internat. J. Supercomputing Applic." } % didn't find
%% in AMS MR
 @String{Kibernetika = "Kibernetika" }
 @String{JResNatBurStand = "Journal of Research of the National Bureau
     of Standards" }
 @String{JResNatBurStand = "J. Res. Nat. Bur. Standards" }
 @String{LinAlgApp = "Linear Algebra and its Applications" }
 @String{LinAlgApp = "Linear Algebra Appl." }
 @String{MathAnaAppl = "Journal of Mathematical Analysis and Applications" }
 @String{MathAnaAppl = "J. Math. Anal. Appl." }
 @String{MathAnnalen = "Mathematische Annalen" }
 @String{MathAnnalen = "Math. Ann." }
 @String{MathPhys = "Journal of Mathematical Physics" }
 @String{MathPhys = "J. Math. Phys." }
 @String{MathComp = "Mathematics of Computation" }
 @String{MathComp = "Math. Comp." }
 @String{MathScand = "Mathematica Scandinavica" }
 @String{MathScand = "Math. Scand." }
 @String{TablesAidsComp = "Mathematical Tables and Other Aids to Computation" }
 @String{TablesAidsComp = "Math. Tables Aids Comput." }
 @String{NumerMath = "Numerische Mathematik" }
 @String{NumerMath = "Numer. Math." }
 @String{PacificMath = "Pacific Journal of Mathematics" }
 @String{PacificMath = "Pacific J. Math." }
 @String{ParDistComp = "Journal of Parallel and Distributed Computing" }
 @String{ParDistComp = "J. Parallel and Distrib. Comput." } % didn't find
%% in AMS MR
 @String{ParComputing = "Parallel Computing" }
 @String{ParComputing = "Parallel Comput." }
 @String{PhilMag = "Philosophical Magazine" }
 @String{PhilMag = "Philos. Mag." }
 @String{ProcNAS = "Proceedings of the National Academy of Sciences
                    of the USA" }
 @String{ProcNAS = "Proc. Nat. Acad. Sci. U. S. A." }
 @String{Psychometrika = "Psychometrika" }
 @String{QuartMath = "Quarterly Journal of Mathematics, Oxford, Series (2)" }
 @String{QuartMath = "Quart. J. Math. Oxford Ser. (2)" }
 @String{QuartApplMath = "Quarterly of Applied Mathematics" }
 @String{QuartApplMath = "Quart. Appl. Math." }
 @String{RevueInstStat = "Review of the International Statisical Institute" }
 @String{RevueInstStat = "Rev. Inst. Internat. Statist." }

 %SIAM
 @String{JSIAM = "Journal of the Society for Industrial and Applied
     Mathematics" }
 @String{JSIAM = "J. Soc. Indust. Appl. Math." }
 @String{JSIAMB = "Journal of the Society for Industrial and Applied
     Mathematics, Series B, Numerical Analysis" }
 @String{JSIAMB = "J. Soc. Indust. Appl. Math. Ser. B Numer. Anal." }
 @String{SIAMAlgMeth = "{SIAM} Journal on Algebraic and Discrete Methods" }
 @String{SIAMAlgMeth = "{SIAM} J. Algebraic Discrete Methods" }
 @String{SIAMAppMath = "{SIAM} Journal on Applied Mathematics" }
 @String{SIAMAppMath = "{SIAM} J. Appl. Math." }
 @String{SIAMComp = "{SIAM} Journal on Computing" }
 @String{SIAMComp = "{SIAM} J. Comput." }
 @String{SIAMMatrix = "{SIAM} Journal on Matrix Analysis and Applications" }
 @String{SIAMMatrix = "{SIAM} J. Matrix Anal. Appl." }
 @String{SIAMNumAnal = "{SIAM} Journal on Numerical Analysis" }
 @String{SIAMNumAnal = "{SIAM} J. Numer. Anal." }
 @String{SIAMReview = "{SIAM} Review" }
 @String{SIAMReview = "{SIAM} Rev." }
 @String{SIAMSciStat = "{SIAM} Journal on Scientific and Statistical
     Computing" }
 @String{SIAMSciStat = "{SIAM} J. Sci. Statist. Comput." }

 @String{SoftPracExp = "Software Practice and Experience" }
 @String{SoftPracExp = "Software Prac. Experience" } % didn't find in AMS MR
 @String{StatScience = "Statistical Science" }
 @String{StatScience = "Statist. Sci." }
 @String{Techno = "Technometrics" }
 @String{USSRCompMathPhys = "{USSR} Computational Mathematics and Mathematical
     Physics" }
 @String{USSRCompMathPhys = "{U. S. S. R.} Comput. Math. and Math. Phys." }
 @String{VLSICompSys = "Journal of {VLSI} and Computer Systems" }
 @String{VLSICompSys = "J. {VLSI} Comput. Syst." }
 @String{ZAngewMathMech = "Zeitschrift fur Angewandte Mathematik und
     Mechanik" }
 @String{ZAngewMathMech = "Z. Angew. Math. Mech." }
 @String{ZAngewMathPhys = "Zeitschrift fur Angewandte Mathematik und Physik" }
 @String{ZAngewMathPhys = "Z. Angew. Math. Phys." }

% Publishers % ================================================= |

 @String{Academic = "Academic Press" }
 @String{ACMPress = "{ACM} Press" }
 @String{AdamHilger = "Adam Hilger" }
 @String{AddisonWesley = "Addison-Wesley" }
 @String{AllynBacon = "Allyn and Bacon" }
 @String{AMS = "American Mathematical Society" }
 @String{Birkhauser = "Birkha{\"u}ser" }
 @String{CambridgePress = "Cambridge University Press" }
 @String{Chelsea = "Chelsea" }
 @String{ClaredonPress = "Claredon Press" }
 @String{DoverPub = "Dover Publications" }
 @String{Eyolles = "Eyolles" }
 @String{HoltRinehartWinston = "Holt, Rinehart and Winston" }
 @String{Interscience = "Interscience" }
 @String{JohnsHopkinsPress = "The Johns Hopkins University Press" }
 @String{JohnWileySons = "John Wiley and Sons" }
 @String{Macmillan = "Macmillan" }
 @String{MathWorks = "The Math Works Inc." }
 @String{McGrawHill = "McGraw-Hill" }
 @String{NatBurStd = "National Bureau of Standards" }
 @String{NorthHolland = "North-Holland" }
 @String{OxfordPress = "Oxford University Press" }  %address Oxford or London?
 @String{PergamonPress = "Pergamon Press" }
 @String{PlenumPress = "Plenum Press" }
 @String{PrenticeHall = "Prentice-Hall" }
 @String{SIAMPub = "{SIAM} Publications" }
 @String{Springer = "Springer-Verlag" }
 @String{TexasPress = "University of Texas Press" }
 @String{VanNostrand = "Van Nostrand" }
 @String{WHFreeman = "W. H. Freeman and Co." }

%Entries
%Various bib entry types.

% incol due to booktitle
@incollection{Mumford:1987:MES:54905.54911,
 author = {Mumford, E.},
 title = {Managerial expert systems and organizational change: some critical research issues},
 booktitle = {Critical issues in information systems research (incoll)},
 year = {1987},
 isbn = {0-471-91281-6},
 pages = {135--155},
 numpages = {21},
 url = {http://portal.acm.org/citation.cfm?id=54905.54911},
 acmid = {54911},
 publisher = {John Wiley \& Sons, Inc.},
 address = {New York, NY, USA},
}

@techreport{Tzamaloukas-01,
  author        = {A. Tzamaloukas and J. J. Garcia-Luna-Aceves},
  title         = {Channel-Hopping Multiple Access},
  number =        {I-CA2301},
  institution =   {Department of Computer Science, University of California},
  address =       {Berkeley, CA},
  year          = {2000}
}

@mastersthesis{ko94,
author = "Jacob Kornerup",
title = "Mapping Powerlists onto Hypercubes",
school = "The University of Texas at Austin",
note = "(In preparation)",
year = "1994"}
%month = "dec",}

@PhdThesis{gerndt:89,
  author =       "Michael Gerndt",
  title =        "Automatic Parallelization for Distributed-Memory
                  Multiprocessing Systems",
  school =       "University of Bonn",
  year =         1989,
  address =      "Bonn, Germany",
  month =        dec
}

@online{CTANacmart,
  author =    {Boris Veytsman},
  title =  {acmart---{C}lass for typesetting publications of {ACM}},
  url =    {http://www.ctan.org/pkg/acmart},
  lastaccessed = {May 27, 2017}
  }

@ARTICLE{braams:babel,
    author = {Braams, Johannes},
    title = {Babel, a Multilingual Style-Option System for Use with LaTeX's Standard Document Styles},
    journal = {TUGboat},
    volume = {12},
    number = {2},
    pages = {291-301},
    month = {June},
    year = {1991},
}

@INPROCEEDINGS{clark:pct,
  AUTHOR = "Malcolm Clark",
  TITLE = "Post Congress Tristesse",
  BOOKTITLE = "TeX90 Conference Proceedings",
  PAGES = "84-89",
  ORGANIZATION = "TeX Users Group",
  MONTH = "March",
  YEAR = {1991}
}

@ARTICLE{herlihy:methodology,
    author = {Herlihy, Maurice},
    title = {A Methodology for Implementing Highly Concurrent Data Objects},
    journal = {ACM Trans. Program. Lang. Syst.},
    volume = {15},
    number = {5},
    pages = {745-770},
    month = {November},
    year = {1993},
    doi = {10.1145/161468.161469},
}

@BOOK{salas:calculus,
  AUTHOR = "S.L. Salas and Einar Hille",
  TITLE = "Calculus: One and Several Variable",
  PUBLISHER = "John Wiley and Sons",
  ADDRESS = "New York",
  YEAR = "1978"
}

@MANUAL{Fear05,
  title =        {Publication quality tables in {\LaTeX}},
  author =       {Simon Fear},
  month =        {April},
  year =         2005,
  note =         {\url{http://www.ctan.org/pkg/booktabs}}
}

@Manual{Amsthm15,
  title =        {Using the amsthm Package},
  organization = {American Mathematical Society},
  month =        {April},
  year =         2015,
  note =         {\url{http://www.ctan.org/pkg/amsthm}}
}

@ArtifactSoftware{R,
    title = {R: A Language and Environment for Statistical Computing},
    author = {{R Core Team}},
    organization = {R Foundation for Statistical Computing},
    address = {Vienna, Austria},
    year = {2019},
    url = {https://www.R-project.org/},
}

@ArtifactDataset{UMassCitations,
 author    =  {Sam Anzaroot and Andrew McCallum},
 title     =  {{UMass} Citation Field Extraction Dataset},
 year      = 2013,
 url       =
    {http://www.iesl.cs.umass.edu/data/data-umasscitationfield},
 lastaccessed = {May 27, 2019}
}

@Eprint{Bornmann2019,
       author = {Bornmann, Lutz and Wray, K. Brad and Haunschild,
                  Robin},
        title = {Citation concept analysis {(CCA)}---A new form of
                  citation analysis revealing the usefulness of
                  concepts for other researchers illustrated by two
                  exemplary case studies including classic books by
                  {Thomas S.~Kuhn} and {Karl R.~Popper}},
     keywords = {Computer Science - Digital Libraries},
         year = 2019,
        month = "May",
          eid = {arXiv:1905.12410},
archivePrefix = {arXiv},
       eprint = {1905.12410},
 primaryClass = {cs.DL},
}

@Eprint{AnzarootPBM14,
  author    = {Sam Anzaroot and
               Alexandre Passos and
               David Belanger and
               Andrew McCallum},
  title     = {Learning Soft Linear Constraints with Application to
                  Citation Field Extraction},
  year      = {2014},
  archivePrefix = {arXiv},
  eprint    = {1403.1349},
}

@inproceedings{Hagerup1993,
title        = {Maintaining Discrete Probability Distributions Optimally},
author       = {Hagerup, Torben and Mehlhorn, Kurt and Munro, J. Ian},
booktitle    = {Proceedings of the 20th International Colloquium on Automata, Languages and Programming},
series       = {Lecture Notes in Computer Science},
volume       = {700},
pages        = {253--264},
year         = {1993},
publisher    = {Springer-Verlag},
address      = {Berlin},
}

@article{li2020switch,
  title={When to switch? Index policies for resource scheduling in emergency response},
  author={Li, Dong and Ding, Li and Connor, Stephen},
  journal={Production and Operations Management},
  volume={29},
  number={2},
  pages={241--262},
  year={2020},
  publisher={Wiley Online Library}
}

@article{argon2008scheduling,
  title={Scheduling impatient jobs in a clearing system with insights on patient triage in mass casualty incidents},
  author={Argon, Nilay Tanik and Ziya, Serhan and Righter, Rhonda},
  journal={Probability in the Engineering and Informational Sciences},
  volume={22},
  number={3},
  pages={301},
  year={2008},
  publisher={Citeseer}
}

@article{jiang2020faster,
  title={Faster dynamic matrix inverse for faster lps},
  author={Jiang, Shunhua and Song, Zhao and Weinstein, Omri and Zhang, Hengjie},
  journal={arXiv preprint arXiv:2004.07470},
  year={2020}
}

@book{puterman2014markov,
  title={Markov Decision Processes: Discrete Stochastic Dynamic Programming},
  author={Puterman, M. L.},
  year={2014},
  publisher={John Wiley \& Sons}
}

@inproceedings{le2008multi,
  title={Multi-UAV dynamic routing with partial observations using restless bandit allocation indices},
  author={Le Ny, J. and Dahleh, M. and Feron, E.},
  booktitle={American Control Conference},
  pages={4220--4225},
  year={2008},
  organization={IEEE}
}

@article{bagheri2015restless,
  title={The restless multi-armed bandit formulation of the cognitive compressive sensing problem},
  author={Bagheri, S. and Scaglione, A.},
  journal={IEEE Trans. Signal Process.},
  volume={63},
  number={5},
  pages={1183--1198},
  year={2015},
  publisher={IEEE}
}

@article{zhao2008myopic,
  title={On myopic sensing for multi-channel opportunistic access: structure, optimality, and performance},
  author={Zhao, Qing and Krishnamachari, Bhaskar and Liu, Keqin},
  journal={IEEE Transactions on Wireless Communications},
  volume={7},
  number={12},
  pages={5431--5440},
  year={2008},
  publisher={IEEE}
}

@inproceedings{ahmad2009multi,
  title={Multi-channel opportunistic access: A case of restless bandits with multiple plays},
  author={Ahmad, Sahand Haji Ali and Liu, Mingyan},
  booktitle={2009 47th Annual Allerton Conference on Communication, Control, and Computing (Allerton)},
  pages={1361--1368},
  year={2009},
  organization={IEEE}
}

@article{xu2019scheduling,
  title={Scheduling Periodic Real-Time Traffic in Lossy Wireless Networks as Restless Multi-Armed Bandit},
  author={Xu, Jun and Guo, Chengcheng},
  journal={IEEE Wireless Communications Letters},
  volume={8},
  number={4},
  pages={1129--1132},
  year={2019},
  publisher={IEEE}
}

@inproceedings{cohen2014restless,
  title={Restless multi-armed bandits under time-varying activation constraints for dynamic spectrum access},
  author={Cohen, Kobi and Zhao, Qing and Scaglione, Anna},
  booktitle={2014 48th Asilomar Conference on Signals, Systems and Computers},
  pages={1575--1578},
  year={2014},
  organization={IEEE}
}

@article{si2010optimal,
  title={Optimal network selection in heterogeneous wireless multimedia networks},
  author={Si, Pengbo and Ji, Hong and Yu, F Richard},
  journal={Wireless Networks},
  volume={16},
  number={5},
  pages={1277--1288},
  year={2010},
  publisher={Springer}
}

@article{modi2019transfer,
  title={Transfer restless multi-armed bandit policy for energy-efficient heterogeneous cellular network},
  author={Modi, Navikkumar and Mary, Philippe and Moy, Christophe},
  journal={EURASIP Journal on Advances in Signal Processing},
  volume={2019},
  number={1},
  pages={46},
  year={2019},
  publisher={Springer}
}

@article{yu2018deadline,
  title={Deadline scheduling as restless bandits},
  author={Yu, Z. and Xu, Y. and Tong, L.},
  journal={IEEE Trans. Automat. Contr.},
  volume={63},
  number={8},
  pages={2343--2358},
  year={2018},
  publisher={IEEE}
}

@inproceedings{he2012scheduling,
  title={Scheduling parallel tasks onto opportunistically available cloud resources},
  author={He, Ting and Chen, Shiyao and Kim, Hyoil and Tong, Lang and Lee, Kang-Won},
  booktitle={2012 IEEE Fifth International Conference on Cloud Computing},
  pages={180--187},
  year={2012},
  organization={IEEE}
}

@inproceedings{yang2018optimal,
  title={Optimal task offloading in fog-enabled networks via index policies},
  author={Yang, F. and Zhu, A. and Zhao, S. and Yang, Y. and Luo, X.},
  booktitle={GlobalSIP},
  pages={688--692},
  year={2018},
  organization={IEEE}
}

@inproceedings{bhattacharya2018restless,
  title={Restless bandits visiting villages: A preliminary study on distributing public health services},
  author={Bhattacharya, Biswarup},
  booktitle={Proceedings of the 1st ACM SIGCAS Conference on Computing and Sustainable Societies},
  pages={1--8},
  year={2018}
}

@article{zayas2019asymptotically,
  title={An asymptotically optimal heuristic for general nonstationary finite-horizon restless multi-armed, multi-action bandits},
  author={Zayas-Caban, Gabriel and Jasin, Stefanus and Wang, Guihua},
  journal={Advances in Applied Probability},
  volume={51},
  number={3},
  pages={745--772},
  year={2019},
  publisher={Cambridge University Press}
}

@article{meshram2020simulation,
  title={Simulation Based Algorithms for {M}arkov Decision Processes and Multi-Action Restless Bandits},
  author={Meshram, Rahul and Kaza, Kesav},
  journal={arXiv preprint arXiv:2007.12933},
  year={2020}
}

@article{thompson1933likelihood,
  title={On the likelihood that one unknown probability exceeds another in view of the evidence of two samples},
  author={Thompson, William R},
  journal={Biometrika},
  volume={25},
  number={3/4},
  pages={285--294},
  year={1933},
  publisher={JSTOR}
}

@article{villar2016indexability,
  title={Indexability and optimal index policies for a class of reinitialising restless bandits},
  author={Villar, Sof{\'\i}a S},
  journal={Probability in the engineering and informational sciences},
  volume={30},
  number={1},
  pages={1--23},
  year={2016},
  publisher={Cambridge University Press}
}

@article{ansell2003whittle,
  title={Whittle's index policy for a multi-class queueing system with convex holding costs},
  author={Ansell, P. S. and Glazebrook, K. D. and Nino-Mora, J. and O'Keeffe, M.},
  journal={Mathematical Methods of Operations Research},
  volume={57},
  number={1},
  pages={21--39},
  year={2003},
  publisher={Springer}
}

@article{hodge2015asymptotic,
  title={On the asymptotic optimality of greedy index heuristics for multi-action restless bandits},
  author={Hodge, D. J. and Glazebrook, K. D.},
  journal={Adv Appl Probab},
  volume={47},
  number={3},
  pages={652--667},
  year={2015},
  publisher={Cambridge University Press}
}

@article{WeberRichard2007CoDp,
issn = {1134-5764},
journal = {TOP},
pages = {211--216},
volume = {15},
publisher = {Springer-Verlag},
number = {2},
year = {2007},
title = {Comments on: Dynamic priority allocation via restless bandit marginal productivity indices},
copyright = {Sociedad de Estadística e Investigación Operativa 2007},
language = {eng},
address = {Berlin/Heidelberg},
author = {Weber, Richard},
keywords = {Game Theory/Mathematical Methods ; Statistics for Business/Economics/Mathematical Finance/Insurance ; Industrial and Production Engineering ; Operations Research/Decision Theory ; Optimization ; Economics / Management Science},
}


@article{verloop2016asymptotically,
  title={Asymptotically optimal priority policies for indexable and nonindexable restless bandits},
  author={Verloop, Ina Maria and others},
  journal={The Annals of Applied Probability},
  volume={26},
  number={4},
  pages={1947--1995},
  year={2016},
  publisher={Institute of Mathematical Statistics}
}


@inproceedings{meuleau1998solving,
  title={Solving very large weakly coupled {M}arkov decision processes},
  author={Meuleau, Nicolas and Hauskrecht, Milos and Kim, Kee-Eung and Peshkin, Leonid and Kaelbling, Leslie Pack and Dean, Thomas L and Boutilier, Craig},
  booktitle={AAAI/IAAI},
  pages={165--172},
  year={1998}
}

@article{gittins1979bandit,
  title={Bandit processes and dynamic allocation indices},
  author={Gittins, John C},
  journal={Journal of the Royal Statistical Society: Series B (Methodological)},
  volume={41},
  number={2},
  pages={148--164},
  year={1979},
  publisher={Wiley Online Library}
}


@inproceedings{killian2019learning,
  title={Learning to prescribe interventions for tuberculosis patients using digital adherence data},
  author={Killian, J. A. and Wilder, B. and Sharma, A. and Choudhary, V. and Dilkina, B. and Tambe, M.},
  booktitle={KDD},
  pages={2430--2438},
  year={2019}
}

@article{cassady2005integrating,
  title={Integrating preventive maintenance planning and production scheduling for a single machine},
  author={Cassady, C Richard and Kutanoglu, Erhan},
  journal={IEEE Transactions on reliability},
  volume={54},
  number={2},
  pages={304--309},
  year={2005},
  publisher={IEEE}
}

@article{nguyen2013capture,
  title={Capture: A new predictive anti-poaching tool for wildlife protection},
  author={Nguyen, Thanh H and Sinha, Arunesh and Gholami, Shahrzad and Plumptre, Andrew and Joppa, Lucas and Tambe, Milind and Driciru, Margaret and Wanyama, Fred and Rwetsiba, Aggrey and Critchlow, Rob},
  year={2013}
}

@book{world2018guideline,
  title={WHO guideline on health policy and system support to optimize community health worker programmes},
  author={World Health Organization and others},
  year={2018},
  publisher={World Health Organization}
}

@article{gafni2020learning,
  title={Learning in restless multi-armed bandits via adaptive arm sequencing rules},
  author={Gafni, T. and Cohen, K.},
  journal={IEEE Trans. Automat. Contr.},
  year={2020},
  publisher={IEEE}
}




@misc{gurobi,  
    author = "Gurobi Optimization, LLC",  
    title = "Gurobi Optimizer Reference Manual",  
    year = 2021,  
    url = "http://www.gurobi.com"
}

@inproceedings{killian2021multiAction,
  title={Beyond ``{T}o Act or Not to Act'': Fast {Lagrangian} Approaches to General Multi-Action Restless Bandits},
  author={Killian, J. A. and Perrault, A. and Tambe, M.},
  booktitle={AAMAS},
  year={2021}
}

@article{adelman2008relaxations,
  title={Relaxations of weakly coupled stochastic dynamic programs},
  author={Adelman, D. and Mersereau, A. J.},
  journal={Oper Res},
  volume={56},
  number={3},
  pages={712--727},
  year={2008},
  publisher={INFORMS}
}

@article{jiang2020faster,
  title={Faster dynamic matrix inverse for faster lps},
  author={Jiang, Shunhua and Song, Zhao and Weinstein, Omri and Zhang, Hengjie},
  journal={arXiv preprint arXiv:2004.07470},
  year={2020}
}


@article{ruiz2020multi,
  title={Multi-machine preventive maintenance scheduling with imperfect interventions: A restless bandit approach},
  author={Ruiz-Hern{\'a}ndez, D. and Pinar-P{\'e}rez, J. M. and Delgado-G{\'o}mez, D.},
  journal={Comput Oper Res},
  volume={119},
  pages={104927},
  year={2020},
  publisher={Elsevier}
}



@article{zhao2008myopic,
  title={On myopic sensing for multi-channel opportunistic access: structure, optimality, and performance},
  author={Zhao, Qing and Krishnamachari, Bhaskar and Liu, Keqin},
  journal={IEEE Transactions on Wireless Communications},
  volume={7},
  number={12},
  pages={5431--5440},
  year={2008},
  publisher={IEEE}
}

@inproceedings{ahmad2009multi,
  title={Multi-channel opportunistic access: A case of restless bandits with multiple plays},
  author={Ahmad, Sahand Haji Ali and Liu, Mingyan},
  booktitle={2009 47th Annual Allerton Conference on Communication, Control, and Computing (Allerton)},
  pages={1361--1368},
  year={2009},
  organization={IEEE}
}

@inproceedings{qian2016restless,
  title={Restless poachers: Handling exploration-exploitation tradeoffs in security domains},
  author={Qian, Y. and Zhang, C. and Krishnamachari, B. and Tambe, M.},
  booktitle={AAMAS},
  pages={123--131},
  year={2016}
}

@article{lee2019optimal,
  title={Optimal screening for hepatocellular carcinoma: A restless bandit model},
  author={Lee, E. and Lavieri, M. S. and Volk, M.},
  journal={Manuf. Serv. Oper. Manag.},
  volume={21},
  number={1},
  pages={198--212},
  year={2019},
  publisher={INFORMS}
}

@inproceedings{mate2020collapsing,
  title={Collapsing Bandits and Their Application to Public Health Interventions},
  author={Mate, A. and Killian, J. A. and Xu, H. and Perrault, A. and Tambe, M.},
  booktitle={NeurIPS},
  year={2020}
}


@article{xu2019scheduling,
  title={Scheduling Periodic Real-Time Traffic in Lossy Wireless Networks as Restless Multi-Armed Bandit},
  author={Xu, Jun and Guo, Chengcheng},
  journal={IEEE Wireless Communications Letters},
  volume={8},
  number={4},
  pages={1129--1132},
  year={2019},
  publisher={IEEE}
}

@inproceedings{cohen2014restless,
  title={Restless multi-armed bandits under time-varying activation constraints for dynamic spectrum access},
  author={Cohen, Kobi and Zhao, Qing and Scaglione, Anna},
  booktitle={2014 48th Asilomar Conference on Signals, Systems and Computers},
  pages={1575--1578},
  year={2014},
  organization={IEEE}
}

@article{si2010optimal,
  title={Optimal network selection in heterogeneous wireless multimedia networks},
  author={Si, Pengbo and Ji, Hong and Yu, F Richard},
  journal={Wireless Networks},
  volume={16},
  number={5},
  pages={1277--1288},
  year={2010},
  publisher={Springer}
}

@article{modi2019transfer,
  title={Transfer restless multi-armed bandit policy for energy-efficient heterogeneous cellular network},
  author={Modi, Navikkumar and Mary, Philippe and Moy, Christophe},
  journal={EURASIP Journal on Advances in Signal Processing},
  volume={2019},
  number={1},
  pages={46},
  year={2019},
  publisher={Springer}
}



@inproceedings{he2012scheduling,
  title={Scheduling parallel tasks onto opportunistically available cloud resources},
  author={He, Ting and Chen, Shiyao and Kim, Hyoil and Tong, Lang and Lee, Kang-Won},
  booktitle={2012 IEEE Fifth International Conference on Cloud Computing},
  pages={180--187},
  year={2012},
  organization={IEEE}
}


@inproceedings{bhattacharya2018restless,
  title={Restless bandits visiting villages: A preliminary study on distributing public health services},
  author={Bhattacharya, Biswarup},
  booktitle={Proceedings of the 1st ACM SIGCAS Conference on Computing and Sustainable Societies},
  pages={1--8},
  year={2018}
}

@article{zayas2019asymptotically,
  title={An asymptotically optimal heuristic for general nonstationary finite-horizon restless multi-armed, multi-action bandits},
  author={Zayas-Caban, Gabriel and Jasin, Stefanus and Wang, Guihua},
  journal={Advances in Applied Probability},
  volume={51},
  number={3},
  pages={745--772},
  year={2019},
  publisher={Cambridge University Press}
}

@article{meshram2020simulation,
  title={Simulation Based Algorithms for {M}arkov Decision Processes and Multi-Action Restless Bandits},
  author={Meshram, Rahul and Kaza, Kesav},
  journal={arXiv preprint arXiv:2007.12933},
  year={2020}
}

@article{thompson1933likelihood,
  title={On the likelihood that one unknown probability exceeds another in view of the evidence of two samples},
  author={Thompson, William R},
  journal={Biometrika},
  volume={25},
  number={3/4},
  pages={285--294},
  year={1933},
  publisher={JSTOR}
}

@article{bertsimas2000restless,
  title={Restless bandits, linear programming relaxations, and a primal-dual index heuristic},
  author={Bertsimas, Dimitris and Ni{\~n}o-Mora, Jos{\'e}},
  journal={Operations Research},
  volume={48},
  number={1},
  pages={80--90},
  year={2000},
  publisher={INFORMS}
}


@article{weber1990index,
  title={On an index policy for restless bandits},
  author={Weber, R. R. and Weiss, G.},
  journal={J Appl Probab},
  volume={27},
  number={3},
  pages={637--648},
  year={1990},
  publisher={Cambridge University Press}
}

@article{villar2016indexability,
  title={Indexability and optimal index policies for a class of reinitialising restless bandits},
  author={Villar, Sof{\'\i}a S},
  journal={Probability in the engineering and informational sciences},
  volume={30},
  number={1},
  pages={1--23},
  year={2016},
  publisher={Cambridge University Press}
}

@article{ansell2003whittle,
  title={Whittle's index policy for a multi-class queueing system with convex holding costs},
  author={Ansell, P. S. and Glazebrook, K. D. and Nino-Mora, J. and O'Keeffe, M.},
  journal={Mathematical Methods of Operations Research},
  volume={57},
  number={1},
  pages={21--39},
  year={2003},
  publisher={Springer}
}

@article{WeberRichard2007CoDp,
issn = {1134-5764},
journal = {TOP},
pages = {211--216},
volume = {15},
publisher = {Springer-Verlag},
number = {2},
year = {2007},
title = {Comments on: Dynamic priority allocation via restless bandit marginal productivity indices},
copyright = {Sociedad de Estadística e Investigación Operativa 2007},
language = {eng},
address = {Berlin/Heidelberg},
author = {Weber, Richard},
keywords = {Game Theory/Mathematical Methods ; Statistics for Business/Economics/Mathematical Finance/Insurance ; Industrial and Production Engineering ; Operations Research/Decision Theory ; Optimization ; Economics / Management Science},
}



@article{verloop2016asymptotically,
  title={Asymptotically optimal priority policies for indexable and nonindexable restless bandits},
  author={Verloop, Ina Maria and others},
  journal={The Annals of Applied Probability},
  volume={26},
  number={4},
  pages={1947--1995},
  year={2016},
  publisher={Institute of Mathematical Statistics}
}


@inproceedings{meuleau1998solving,
  title={Solving very large weakly coupled {M}arkov decision processes},
  author={Meuleau, Nicolas and Hauskrecht, Milos and Kim, Kee-Eung and Peshkin, Leonid and Kaelbling, Leslie Pack and Dean, Thomas L and Boutilier, Craig},
  booktitle={AAAI/IAAI},
  pages={165--172},
  year={1998}
}


@article{gittins1979bandit,
  title={Bandit processes and dynamic allocation indices},
  author={Gittins, John C},
  journal={Journal of the Royal Statistical Society: Series B (Methodological)},
  volume={41},
  number={2},
  pages={148--164},
  year={1979},
  publisher={Wiley Online Library}
}


@article{nino2001restless,
  title={Restless bandits, partial conservation laws and indexability},
  author={Nino-Mora, Jose},
  journal={Advances in Applied Probability},
  pages={76--98},
  year={2001},
  publisher={JSTOR}
}

@misc{pymdptoolbox,
  author  = {Cordwell, Steven A.W. },
  title   = {{P}ython {M}arkov Decision Process (MDP) Toolbox},
  year    = {2015},
  note     = {{PyPi}, \url{https://pypi.org/project/pymdptoolbox/}}
}

@inproceedings{killian2019learning,
  title={Learning to prescribe interventions for tuberculosis patients using digital adherence data},
  author={Killian, Jackson A and Wilder, Bryan and Sharma, Amit and Choudhary, Vinod and Dilkina, Bistra and Tambe, Milind},
  booktitle={Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery \& Data Mining},
  pages={2430--2438},
  year={2019}
}

@article{cassady2005integrating,
  title={Integrating preventive maintenance planning and production scheduling for a single machine},
  author={Cassady, C Richard and Kutanoglu, Erhan},
  journal={IEEE Transactions on reliability},
  volume={54},
  number={2},
  pages={304--309},
  year={2005},
  publisher={IEEE}
}

@article{nguyen2013capture,
  title={Capture: A new predictive anti-poaching tool for wildlife protection},
  author={Nguyen, Thanh H and Sinha, Arunesh and Gholami, Shahrzad and Plumptre, Andrew and Joppa, Lucas and Tambe, Milind and Driciru, Margaret and Wanyama, Fred and Rwetsiba, Aggrey and Critchlow, Rob},
  year={2013}
}

@book{world2018guideline,
  title={WHO guideline on health policy and system support to optimize community health worker programmes},
  author={World Health Organization and others},
  year={2018},
  publisher={World Health Organization}
}


@article{givan2000bounded,
  title={Bounded-parameter {M}arkov decision processes},
  author={Givan, Robert and Leach, Sonia and Dean, Thomas},
  journal={Artificial Intelligence},
  volume={122},
  number={1-2},
  pages={71--109},
  year={2000},
  publisher={Elsevier}
}

@article{ahmed2017sampling,
  title={Sampling based approaches for minimizing regret in uncertain {M}arkov decision processes (MDPs)},
  author={Ahmed, Asrar and Varakantham, Pradeep and Lowalekar, Meghna and Adulyasak, Yossiri and Jaillet, Patrick},
  journal={Journal of Artificial Intelligence Research},
  volume={59},
  pages={229--264},
  year={2017}
}

@inproceedings{kwak2012towards,
  title={Towards robust multi-objective optimization under model uncertainty for energy conservation},
  author={Kwak, Jun-young and Varakantham, Pradeep and Maheswaran, Rajiv and Tambe, Milind and Hayes, Timothy and Wood, Wendy and Becerik-Gerber, Burcin},
  booktitle={AAMAS workshop on agent technologies for energy systems (ATES)},
  year={2012}
}

@inproceedings{wilder2017uncharted,
  title={Uncharted but not Uninfluenced: Influence Maximization with an Uncertain Network.},
  author={Wilder, Bryan and Yadav, Amulya and Immorlica, Nicole and Rice, Eric and Tambe, Milind},
  booktitle={AAMAS},
  volume={17},
  pages={1305--1313},
  year={2017}
}

@inproceedings{Sombabu2020,
  title={Whittle index for AoI-aware scheduling},
  author={Sombabu, B. and Mate, A. and Manjunath, D. and Moharir, S.},
  booktitle={IEEE International Conference on Communication Systems \& Networks (COSMSNETS)},
  pages={},
  year={2020},
  organization={IEEE}
}

@article{Liu2010,
  title={ Indexability of restless bandit problems and optimality of Whittle index for dynamic multichannel access},
  author={Liu, K. and Zhao, Q. },
  journal={IEEE Transactions on Information Theory},
  pages = {5547-5567},
  year={2010}
}

@inproceedings{mcmahan2003planning,
  title={Planning in the presence of cost functions controlled by an adversary},
  author={McMahan, H. B. and Gordon, G. J. and Blum, A.},
  booktitle={ICML},
  pages={536--543},
  year={2003}
}

@inproceedings{wu2018budget,
  title={Budget constrained bidding by model-free reinforcement learning in display advertising},
  author={Wu, Di and Chen, Xiujun and Yang, Xun and Wang, Hao and Tan, Qing and Zhang, Xiaoxun and Xu, Jian and Gai, Kun},
  booktitle={Proceedings of the 27th ACM International Conference on Information and Knowledge Management},
  pages={1443--1451},
  year={2018}
}

@article{haarnoja2018soft,
  title={Soft actor-critic algorithms and applications},
  author={Haarnoja, Tuomas and Zhou, Aurick and Hartikainen, Kristian and Tucker, George and Ha, Sehoon and Tan, Jie and Kumar, Vikash and Zhu, Henry and Gupta, Abhishek and Abbeel, Pieter and others},
  journal={arXiv preprint arXiv:1812.05905},
  year={2018}
}

@article{lowe2017multi,
  title={Multi-agent actor-critic for mixed cooperative-competitive environments},
  author={Lowe, R. and Wu, Y. and Tamar, A. and Harb, J. and Abbeel, P. and Mordatch, I.},
  journal={arXiv preprint arXiv:1706.02275},
  year={2017}
}

@article{whittle1988restless,
  title={Restless bandits: Activity allocation in a changing world},
  author={Whittle, P.},
  journal={J Appl Probab},
  pages={287--298},
  year={1988},
  publisher={JSTOR}
}


% Journals

% First the Full Name is given, then the abbreviation used in the AMS Math
% Reviews, with an indication if it could not be found there.
% Note the 2nd overwrites the 1st, so swap them if you want the full name.

@article{lakshminarayanan2017stability,
  title={A stability criterion for two timescale stochastic approximation schemes},
  author={Lakshminarayanan, Chandrashekar and Bhatnagar, Shalabh},
  journal={Automatica},
  volume={79},
  pages={108--114},
  year={2017},
  publisher={Elsevier}
}

@misc{notes,
  title = {Convex Functions},
  author = {Robert Gardner},
  howpublished = {\url{https://faculty.etsu.edu/gardnerr/5210/Beamer-Proofs/Proofs-6-6-print.pdf}},
  note = {Accessed: 2021-01-15}
}


@inproceedings{papadimitriou1994complexity,
  title={The complexity of optimal queueing network control},
  author={Papadimitriou, C. H. and Tsitsiklis, J. N.},
  booktitle={Structure in Complexity Theory},
  pages={318--322},
  year={1994},
  organization={IEEE}
}

@article{watkins1989learning,
  title={Learning from delayed rewards},
  author={Watkins, Christopher John Cornish Hellaby},
  year={1989},
  publisher={King's College, Cambridge}
}

@article{watkins1992q,
  title={Q-learning},
  author={Watkins, Christopher JCH and Dayan, Peter},
  journal={Machine learning},
  volume={8},
  number={3-4},
  pages={279--292},
  year={1992},
  publisher={Springer}
}

@inproceedings{fu2019towards,
  title={Towards Q-learning the Whittle Index for Restless Bandits},
  author={Fu, J. and Nazarathy, Y. and Moka, S. and Taylor, P. G.},
  booktitle={ANZCC},
  pages={249--254},
  year={2019},
  organization={IEEE}
}

@article{abounadi2001learning,
  title={Learning algorithms for {M}arkov decision processes with average cost},
  author={Abounadi, Jinane and Bertsekas, Dimitrib and Borkar, Vivek S.},
  journal={SIAM Journal on Control and Optimization},
  volume={40},
  number={3},
  pages={681--698},
  year={2001},
  publisher={SIAM}
}

@article{xiong2021learning,
  title={Learning Augmented Index Policy for Optimal Service Placement at the Network Edge},
  author={Xiong, Guojun and Singh, Rahul and Li, Jian},
  journal={arXiv preprint arXiv:2101.03641},
  year={2021}
}

@article{glazebrook2011general,
  title={General notions of indexability for queueing control and asset management},
  author={Glazebrook, K. D. and Hodge, D. J. and Kirkbride, C.},
  journal={Ann Appl Probab},
  volume={21},
  number={3},
  pages={876--907},
  year={2011},
  publisher={JSTOR}
}

@article{Liu2010,
  title={ Indexability of restless bandit problems and optimality of Whittle index for dynamic multichannel access},
  author={Liu, K. and Zhao, Q. },
  journal={IEEE Transactions on Information Theory},
  pages = {5547-5567},
  year={2010}
}

@inproceedings{Sombabu2020,
  title={Whittle index for AoI-aware scheduling},
  author={Sombabu, B. and Mate, A. and Manjunath, D. and Moharir, S.},
  booktitle={IEEE International Conference on Communication Systems \& Networks (COSMSNETS)},
  pages={},
  year={2020},
  organization={IEEE}
}

@article{jacko2016resource,
  title={Resource capacity allocation to stochastic dynamic competitors: knapsack problem for perishable items and index-knapsack heuristic},
  author={Jacko, Peter},
  journal={Annals of Operations Research},
  volume={241},
  number={1},
  pages={83--107},
  year={2016},
  publisher={Springer}
}

 %{AMS}
 @String{AMSTrans = "American Mathematical Society Translations" }
 @String{AMSTrans = "Amer. Math. Soc. Transl." }
 @String{BullAMS = "Bulletin of the American Mathematical Society" }
 @String{BullAMS = "Bull. Amer. Math. Soc." }
 @String{ProcAMS = "Proceedings of the American Mathematical Society" }
 @String{ProcAMS = "Proc. Amer. Math. Soc." }
 @String{TransAMS = "Transactions of the American Mathematical Society" }
 @String{TransAMS = "Trans. Amer. Math. Soc." }

 %ACM
 @String{CACM = "Communications of the {ACM}" }
 @String{CACM = "Commun. {ACM}" }
 @String{CompServ = "Comput. Surveys" }
 @String{JACM = "J. ACM" }
 @String{ACMMathSoft = "{ACM} Transactions on Mathematical Software" }
 @String{ACMMathSoft = "{ACM} Trans. Math. Software" }
 @String{SIGNUM = "{ACM} {SIGNUM} Newsletter" }
 @String{SIGNUM = "{ACM} {SIGNUM} Newslett." }

 @String{AmerSocio = "American Journal of Sociology" }
 @String{AmerStatAssoc = "Journal of the American Statistical Association" }
 @String{AmerStatAssoc = "J. Amer. Statist. Assoc." }
 @String{ApplMathComp = "Applied Mathematics and Computation" }
 @String{ApplMathComp = "Appl. Math. Comput." }
 @String{AmerMathMonthly = "American Mathematical Monthly" }
 @String{AmerMathMonthly = "Amer. Math. Monthly" }
 @String{BIT = "{BIT}" }
 @String{BritStatPsych = "British Journal of Mathematical and Statistical
          Psychology" }
 @String{BritStatPsych = "Brit. J. Math. Statist. Psych." }
 @String{CanMathBull = "Canadian Mathematical Bulletin" }
 @String{CanMathBull = "Canad. Math. Bull." }
 @String{CompApplMath = "Journal of Computational and Applied Mathematics" }
 @String{CompApplMath = "J. Comput. Appl. Math." }
 @String{CompPhys = "Journal of Computational Physics" }
 @String{CompPhys = "J. Comput. Phys." }
 @String{CompStruct = "Computers and Structures" }
 @String{CompStruct = "Comput. \& Structures" }
 @String{CompJour = "The Computer Journal" }
 @String{CompJour = "Comput. J." }
 @String{CompSysSci = "Journal of Computer and System Sciences" }
 @String{CompSysSci = "J. Comput. System Sci." }
 @String{Computing = "Computing" }
 @String{ContempMath = "Contemporary Mathematics" }
 @String{ContempMath = "Contemp. Math." }
 @String{Crelle = "Crelle's Journal" }
 @String{GiornaleMath = "Giornale di Mathematiche" }
 @String{GiornaleMath = "Giorn. Mat." } % didn't find in AMS MR., ibid.

 %IEEE
 @String{Computer = "{IEEE} Computer" }
 @String{IEEETransComp = "{IEEE} Transactions on Computers" }
 @String{IEEETransComp = "{IEEE} Trans. Comput." }
 @String{IEEETransAC = "{IEEE} Transactions on Automatic Control" }
 @String{IEEETransAC = "{IEEE} Trans. Automat. Control" }
 @String{IEEESpec = "{IEEE} Spectrum" } % didn't find in AMS MR
 @String{ProcIEEE = "Proceedings of the {IEEE}" }
 @String{ProcIEEE = "Proc. {IEEE}" } % didn't find in AMS MR
 @String{IEEETransAeroElec = "{IEEE} Transactions on Aerospace and Electronic
     Systems" }
 @String{IEEETransAeroElec = "{IEEE} Trans. Aerospace Electron. Systems" }

 @String{IMANumerAna = "{IMA} Journal of Numerical Analysis" }
 @String{IMANumerAna = "{IMA} J. Numer. Anal." }
 @String{InfProcLet = "Information Processing Letters" }
 @String{InfProcLet = "Inform. Process. Lett." }
 @String{InstMathApp = "Journal of the Institute of Mathematics and
     its Applications" }
 @String{InstMathApp = "J. Inst. Math. Appl." }
 @String{IntControl = "International Journal of Control" }
 @String{IntControl = "Internat. J. Control" }
 @String{IntNumerEng = "International Journal for Numerical Methods in
     Engineering" }
 @String{IntNumerEng = "Internat. J. Numer. Methods Engrg." }
 @String{IntSuper = "International Journal of Supercomputing Applications" }
 @String{IntSuper = "Internat. J. Supercomputing Applic." } % didn't find
%% in AMS MR
 @String{Kibernetika = "Kibernetika" }
 @String{JResNatBurStand = "Journal of Research of the National Bureau
     of Standards" }
 @String{JResNatBurStand = "J. Res. Nat. Bur. Standards" }
 @String{LinAlgApp = "Linear Algebra and its Applications" }
 @String{LinAlgApp = "Linear Algebra Appl." }
 @String{MathAnaAppl = "Journal of Mathematical Analysis and Applications" }
 @String{MathAnaAppl = "J. Math. Anal. Appl." }
 @String{MathAnnalen = "Mathematische Annalen" }
 @String{MathAnnalen = "Math. Ann." }
 @String{MathPhys = "Journal of Mathematical Physics" }
 @String{MathPhys = "J. Math. Phys." }
 @String{MathComp = "Mathematics of Computation" }
 @String{MathComp = "Math. Comp." }
 @String{MathScand = "Mathematica Scandinavica" }
 @String{MathScand = "Math. Scand." }
 @String{TablesAidsComp = "Mathematical Tables and Other Aids to Computation" }
 @String{TablesAidsComp = "Math. Tables Aids Comput." }
 @String{NumerMath = "Numerische Mathematik" }
 @String{NumerMath = "Numer. Math." }
 @String{PacificMath = "Pacific Journal of Mathematics" }
 @String{PacificMath = "Pacific J. Math." }
 @String{ParDistComp = "Journal of Parallel and Distributed Computing" }
 @String{ParDistComp = "J. Parallel and Distrib. Comput." } % didn't find
%% in AMS MR
 @String{ParComputing = "Parallel Computing" }
 @String{ParComputing = "Parallel Comput." }
 @String{PhilMag = "Philosophical Magazine" }
 @String{PhilMag = "Philos. Mag." }
 @String{ProcNAS = "Proceedings of the National Academy of Sciences
                    of the USA" }
 @String{ProcNAS = "Proc. Nat. Acad. Sci. U. S. A." }
 @String{Psychometrika = "Psychometrika" }
 @String{QuartMath = "Quarterly Journal of Mathematics, Oxford, Series (2)" }
 @String{QuartMath = "Quart. J. Math. Oxford Ser. (2)" }
 @String{QuartApplMath = "Quarterly of Applied Mathematics" }
 @String{QuartApplMath = "Quart. Appl. Math." }
 @String{RevueInstStat = "Review of the International Statisical Institute" }
 @String{RevueInstStat = "Rev. Inst. Internat. Statist." }

 %SIAM
 @String{JSIAM = "Journal of the Society for Industrial and Applied
     Mathematics" }
 @String{JSIAM = "J. Soc. Indust. Appl. Math." }
 @String{JSIAMB = "Journal of the Society for Industrial and Applied
     Mathematics, Series B, Numerical Analysis" }
 @String{JSIAMB = "J. Soc. Indust. Appl. Math. Ser. B Numer. Anal." }
 @String{SIAMAlgMeth = "{SIAM} Journal on Algebraic and Discrete Methods" }
 @String{SIAMAlgMeth = "{SIAM} J. Algebraic Discrete Methods" }
 @String{SIAMAppMath = "{SIAM} Journal on Applied Mathematics" }
 @String{SIAMAppMath = "{SIAM} J. Appl. Math." }
 @String{SIAMComp = "{SIAM} Journal on Computing" }
 @String{SIAMComp = "{SIAM} J. Comput." }
 @String{SIAMMatrix = "{SIAM} Journal on Matrix Analysis and Applications" }
 @String{SIAMMatrix = "{SIAM} J. Matrix Anal. Appl." }
 @String{SIAMNumAnal = "{SIAM} Journal on Numerical Analysis" }
 @String{SIAMNumAnal = "{SIAM} J. Numer. Anal." }
 @String{SIAMReview = "{SIAM} Review" }
 @String{SIAMReview = "{SIAM} Rev." }
 @String{SIAMSciStat = "{SIAM} Journal on Scientific and Statistical
     Computing" }
 @String{SIAMSciStat = "{SIAM} J. Sci. Statist. Comput." }

 @String{SoftPracExp = "Software Practice and Experience" }
 @String{SoftPracExp = "Software Prac. Experience" } % didn't find in AMS MR
 @String{StatScience = "Statistical Science" }
 @String{StatScience = "Statist. Sci." }
 @String{Techno = "Technometrics" }
 @String{USSRCompMathPhys = "{USSR} Computational Mathematics and Mathematical
     Physics" }
 @String{USSRCompMathPhys = "{U. S. S. R.} Comput. Math. and Math. Phys." }
 @String{VLSICompSys = "Journal of {VLSI} and Computer Systems" }
 @String{VLSICompSys = "J. {VLSI} Comput. Syst." }
 @String{ZAngewMathMech = "Zeitschrift fur Angewandte Mathematik und
     Mechanik" }
 @String{ZAngewMathMech = "Z. Angew. Math. Mech." }
 @String{ZAngewMathPhys = "Zeitschrift fur Angewandte Mathematik und Physik" }
 @String{ZAngewMathPhys = "Z. Angew. Math. Phys." }

% Publishers % ================================================= |

 @String{Academic = "Academic Press" }
 @String{ACMPress = "{ACM} Press" }
 @String{AdamHilger = "Adam Hilger" }
 @String{AddisonWesley = "Addison-Wesley" }
 @String{AllynBacon = "Allyn and Bacon" }
 @String{AMS = "American Mathematical Society" }
 @String{Birkhauser = "Birkha{\"u}ser" }
 @String{CambridgePress = "Cambridge University Press" }
 @String{Chelsea = "Chelsea" }
 @String{ClaredonPress = "Claredon Press" }
 @String{DoverPub = "Dover Publications" }
 @String{Eyolles = "Eyolles" }
 @String{HoltRinehartWinston = "Holt, Rinehart and Winston" }
 @String{Interscience = "Interscience" }
 @String{JohnsHopkinsPress = "The Johns Hopkins University Press" }
 @String{JohnWileySons = "John Wiley and Sons" }
 @String{Macmillan = "Macmillan" }
 @String{MathWorks = "The Math Works Inc." }
 @String{McGrawHill = "McGraw-Hill" }
 @String{NatBurStd = "National Bureau of Standards" }
 @String{NorthHolland = "North-Holland" }
 @String{OxfordPress = "Oxford University Press" }  %address Oxford or London?
 @String{PergamonPress = "Pergamon Press" }
 @String{PlenumPress = "Plenum Press" }
 @String{PrenticeHall = "Prentice-Hall" }
 @String{SIAMPub = "{SIAM} Publications" }
 @String{Springer = "Springer-Verlag" }
 @String{TexasPress = "University of Texas Press" }
 @String{VanNostrand = "Van Nostrand" }
 @String{WHFreeman = "W. H. Freeman and Co." }

%Entries
%Various bib entry types.

% incol due to booktitle
@incollection{Mumford:1987:MES:54905.54911,
 author = {Mumford, E.},
 title = {Managerial expert systems and organizational change: some critical research issues},
 booktitle = {Critical issues in information systems research (incoll)},
 year = {1987},
 isbn = {0-471-91281-6},
 pages = {135--155},
 numpages = {21},
 url = {http://portal.acm.org/citation.cfm?id=54905.54911},
 acmid = {54911},
 publisher = {John Wiley \& Sons, Inc.},
 address = {New York, NY, USA},
}

@techreport{Tzamaloukas-01,
  author        = {A. Tzamaloukas and J. J. Garcia-Luna-Aceves},
  title         = {Channel-Hopping Multiple Access},
  number =        {I-CA2301},
  institution =   {Department of Computer Science, University of California},
  address =       {Berkeley, CA},
  year          = {2000}
}

@mastersthesis{ko94,
author = "Jacob Kornerup",
title = "Mapping Powerlists onto Hypercubes",
school = "The University of Texas at Austin",
note = "(In preparation)",
year = "1994"}
%month = "dec",}

@PhdThesis{gerndt:89,
  author =       "Michael Gerndt",
  title =        "Automatic Parallelization for Distributed-Memory
                  Multiprocessing Systems",
  school =       "University of Bonn",
  year =         1989,
  address =      "Bonn, Germany",
  month =        dec
}

@online{CTANacmart,
  author =    {Boris Veytsman},
  title =  {acmart---{C}lass for typesetting publications of {ACM}},
  url =    {http://www.ctan.org/pkg/acmart},
  lastaccessed = {May 27, 2017}
  }

@ARTICLE{braams:babel,
    author = {Braams, Johannes},
    title = {Babel, a Multilingual Style-Option System for Use with LaTeX's Standard Document Styles},
    journal = {TUGboat},
    volume = {12},
    number = {2},
    pages = {291-301},
    month = {June},
    year = {1991},
}

@INPROCEEDINGS{clark:pct,
  AUTHOR = "Malcolm Clark",
  TITLE = "Post Congress Tristesse",
  BOOKTITLE = "TeX90 Conference Proceedings",
  PAGES = "84-89",
  ORGANIZATION = "TeX Users Group",
  MONTH = "March",
  YEAR = {1991}
}

@ARTICLE{herlihy:methodology,
    author = {Herlihy, Maurice},
    title = {A Methodology for Implementing Highly Concurrent Data Objects},
    journal = {ACM Trans. Program. Lang. Syst.},
    volume = {15},
    number = {5},
    pages = {745-770},
    month = {November},
    year = {1993},
    doi = {10.1145/161468.161469},
}

@BOOK{salas:calculus,
  AUTHOR = "S.L. Salas and Einar Hille",
  TITLE = "Calculus: One and Several Variable",
  PUBLISHER = "John Wiley and Sons",
  ADDRESS = "New York",
  YEAR = "1978"
}

@MANUAL{Fear05,
  title =        {Publication quality tables in {\LaTeX}},
  author =       {Simon Fear},
  month =        {April},
  year =         2005,
  note =         {\url{http://www.ctan.org/pkg/booktabs}}
}

@Manual{Amsthm15,
  title =        {Using the amsthm Package},
  organization = {American Mathematical Society},
  month =        {April},
  year =         2015,
  note =         {\url{http://www.ctan.org/pkg/amsthm}}
}

@ArtifactSoftware{R,
    title = {R: A Language and Environment for Statistical Computing},
    author = {{R Core Team}},
    organization = {R Foundation for Statistical Computing},
    address = {Vienna, Austria},
    year = {2019},
    url = {https://www.R-project.org/},
}

@ArtifactDataset{UMassCitations,
 author    =  {Sam Anzaroot and Andrew McCallum},
 title     =  {{UMass} Citation Field Extraction Dataset},
 year      = 2013,
 url       =
    {http://www.iesl.cs.umass.edu/data/data-umasscitationfield},
 lastaccessed = {May 27, 2019}
}

@Eprint{Bornmann2019,
       author = {Bornmann, Lutz and Wray, K. Brad and Haunschild,
                  Robin},
        title = {Citation concept analysis {(CCA)}---A new form of
                  citation analysis revealing the usefulness of
                  concepts for other researchers illustrated by two
                  exemplary case studies including classic books by
                  {Thomas S.~Kuhn} and {Karl R.~Popper}},
     keywords = {Computer Science - Digital Libraries},
         year = 2019,
        month = "May",
          eid = {arXiv:1905.12410},
archivePrefix = {arXiv},
       eprint = {1905.12410},
 primaryClass = {cs.DL},
}

@Eprint{AnzarootPBM14,
  author    = {Sam Anzaroot and
               Alexandre Passos and
               David Belanger and
               Andrew McCallum},
  title     = {Learning Soft Linear Constraints with Application to
                  Citation Field Extraction},
  year      = {2014},
  archivePrefix = {arXiv},
  eprint    = {1403.1349},
}

@inproceedings{Hagerup1993,
title        = {Maintaining Discrete Probability Distributions Optimally},
author       = {Hagerup, Torben and Mehlhorn, Kurt and Munro, J. Ian},
booktitle    = {Proceedings of the 20th International Colloquium on Automata, Languages and Programming},
series       = {Lecture Notes in Computer Science},
volume       = {700},
pages        = {253--264},
year         = {1993},
publisher    = {Springer-Verlag},
address      = {Berlin},
}

@article{li2020switch,
  title={When to switch? Index policies for resource scheduling in emergency response},
  author={Li, Dong and Ding, Li and Connor, Stephen},
  journal={Production and Operations Management},
  volume={29},
  number={2},
  pages={241--262},
  year={2020},
  publisher={Wiley Online Library}
}

@article{argon2008scheduling,
  title={Scheduling impatient jobs in a clearing system with insights on patient triage in mass casualty incidents},
  author={Argon, Nilay Tanik and Ziya, Serhan and Righter, Rhonda},
  journal={Probability in the Engineering and Informational Sciences},
  volume={22},
  number={3},
  pages={301},
  year={2008},
  publisher={Citeseer}
}


@phdthesis{hawkins2003langrangian,
  title={A Langrangian decomposition approach to weakly coupled dynamic optimization problems and its applications},
  author={Hawkins, J. T.},
  year={2003},
  school={Massachusetts Institute of Technology}
}

@article{jiang2020faster,
  title={Faster dynamic matrix inverse for faster lps},
  author={Jiang, Shunhua and Song, Zhao and Weinstein, Omri and Zhang, Hengjie},
  journal={arXiv preprint arXiv:2004.07470},
  year={2020}
}


@article{glazebrook2006some,
  title={Some indexable families of restless bandit problems},
  author={Glazebrook, K. D. and Ruiz-Hernandez, D. and Kirkbride, C.},
  journal={J. Appl. Probab},
  volume={38},
  number={3},
  pages={643--672},
  year={2006},
  publisher={Cambridge University Press}
}


@article{zhao2008myopic,
  title={On myopic sensing for multi-channel opportunistic access: structure, optimality, and performance},
  author={Zhao, Qing and Krishnamachari, Bhaskar and Liu, Keqin},
  journal={IEEE Transactions on Wireless Communications},
  volume={7},
  number={12},
  pages={5431--5440},
  year={2008},
  publisher={IEEE}
}

@inproceedings{ahmad2009multi,
  title={Multi-channel opportunistic access: A case of restless bandits with multiple plays},
  author={Ahmad, Sahand Haji Ali and Liu, Mingyan},
  booktitle={2009 47th Annual Allerton Conference on Communication, Control, and Computing (Allerton)},
  pages={1361--1368},
  year={2009},
  organization={IEEE}
}

@article{xu2019scheduling,
  title={Scheduling Periodic Real-Time Traffic in Lossy Wireless Networks as Restless Multi-Armed Bandit},
  author={Xu, Jun and Guo, Chengcheng},
  journal={IEEE Wireless Communications Letters},
  volume={8},
  number={4},
  pages={1129--1132},
  year={2019},
  publisher={IEEE}
}

@inproceedings{cohen2014restless,
  title={Restless multi-armed bandits under time-varying activation constraints for dynamic spectrum access},
  author={Cohen, Kobi and Zhao, Qing and Scaglione, Anna},
  booktitle={2014 48th Asilomar Conference on Signals, Systems and Computers},
  pages={1575--1578},
  year={2014},
  organization={IEEE}
}

@article{si2010optimal,
  title={Optimal network selection in heterogeneous wireless multimedia networks},
  author={Si, Pengbo and Ji, Hong and Yu, F Richard},
  journal={Wireless Networks},
  volume={16},
  number={5},
  pages={1277--1288},
  year={2010},
  publisher={Springer}
}

@article{modi2019transfer,
  title={Transfer restless multi-armed bandit policy for energy-efficient heterogeneous cellular network},
  author={Modi, Navikkumar and Mary, Philippe and Moy, Christophe},
  journal={EURASIP Journal on Advances in Signal Processing},
  volume={2019},
  number={1},
  pages={46},
  year={2019},
  publisher={Springer}
}


@inproceedings{he2012scheduling,
  title={Scheduling parallel tasks onto opportunistically available cloud resources},
  author={He, Ting and Chen, Shiyao and Kim, Hyoil and Tong, Lang and Lee, Kang-Won},
  booktitle={2012 IEEE Fifth International Conference on Cloud Computing},
  pages={180--187},
  year={2012},
  organization={IEEE}
}



@inproceedings{bhattacharya2018restless,
  title={Restless bandits visiting villages: A preliminary study on distributing public health services},
  author={Bhattacharya, Biswarup},
  booktitle={Proceedings of the 1st ACM SIGCAS Conference on Computing and Sustainable Societies},
  pages={1--8},
  year={2018}
}

@article{zayas2019asymptotically,
  title={An asymptotically optimal heuristic for general nonstationary finite-horizon restless multi-armed, multi-action bandits},
  author={Zayas-Caban, Gabriel and Jasin, Stefanus and Wang, Guihua},
  journal={Advances in Applied Probability},
  volume={51},
  number={3},
  pages={745--772},
  year={2019},
  publisher={Cambridge University Press}
}

@article{meshram2020simulation,
  title={Simulation Based Algorithms for {M}arkov Decision Processes and Multi-Action Restless Bandits},
  author={Meshram, Rahul and Kaza, Kesav},
  journal={arXiv preprint arXiv:2007.12933},
  year={2020}
}

@article{thompson1933likelihood,
  title={On the likelihood that one unknown probability exceeds another in view of the evidence of two samples},
  author={Thompson, William R},
  journal={Biometrika},
  volume={25},
  number={3/4},
  pages={285--294},
  year={1933},
  publisher={JSTOR}
}

@article{papadimitriou1999complexity,
  title={The complexity of optimal queuing network control},
  author={Papadimitriou, C. H. and Tsitsiklis, J. N.},
  journal={Math. Oper. Res.},
  volume={24},
  number={2},
  pages={293--305},
  year={1999},
  publisher={INFORMS}
}

@article{villar2016indexability,
  title={Indexability and optimal index policies for a class of reinitialising restless bandits},
  author={Villar, Sof{\'\i}a S},
  journal={Probability in the engineering and informational sciences},
  volume={30},
  number={1},
  pages={1--23},
  year={2016},
  publisher={Cambridge University Press}
}

@article{ansell2003whittle,
  title={Whittle's index policy for a multi-class queueing system with convex holding costs},
  author={Ansell, PS and Glazebrook, Kevin D and Nino-Mora, Jos{\'e} and O'Keeffe, M},
  journal={Mathematical Methods of Operations Research},
  volume={57},
  number={1},
  pages={21--39},
  year={2003},
  publisher={Springer}
}

@article{WeberRichard2007CoDp,
issn = {1134-5764},
journal = {TOP},
pages = {211--216},
volume = {15},
publisher = {Springer-Verlag},
number = {2},
year = {2007},
title = {Comments on: Dynamic priority allocation via restless bandit marginal productivity indices},
copyright = {Sociedad de Estadística e Investigación Operativa 2007},
language = {eng},
address = {Berlin/Heidelberg},
author = {Weber, Richard},
keywords = {Game Theory/Mathematical Methods ; Statistics for Business/Economics/Mathematical Finance/Insurance ; Industrial and Production Engineering ; Operations Research/Decision Theory ; Optimization ; Economics / Management Science},
}

@incollection{mahajan2008multi,
  title={Multi-armed bandit problems},
  author={Mahajan, Aditya and Teneketzis, Demosthenis},
  booktitle={Foundations and applications of sensor management},
  pages={121--151},
  year={2008},
  publisher={Springer}
}

@article{verloop2016asymptotically,
  title={Asymptotically optimal priority policies for indexable and nonindexable restless bandits},
  author={Verloop, Ina Maria and others},
  journal={The Annals of Applied Probability},
  volume={26},
  number={4},
  pages={1947--1995},
  year={2016},
  publisher={Institute of Mathematical Statistics}
}


@inproceedings{meuleau1998solving,
  title={Solving very large weakly coupled {M}arkov decision processes},
  author={Meuleau, Nicolas and Hauskrecht, Milos and Kim, Kee-Eung and Peshkin, Leonid and Kaelbling, Leslie Pack and Dean, Thomas L and Boutilier, Craig},
  booktitle={AAAI/IAAI},
  pages={165--172},
  year={1998}
}


@article{gocgun2012lagrangian,
  title={Lagrangian relaxation and constraint generation for allocation and advanced scheduling},
  author={Gocgun, Y. and Ghate, A.},
  journal={Comput Oper Res},
  volume={39},
  number={10},
  pages={2323--2336},
  year={2012},
  publisher={Elsevier}
}

@article{gittins1979bandit,
  title={Bandit processes and dynamic allocation indices},
  author={Gittins, John C},
  journal={Journal of the Royal Statistical Society: Series B (Methodological)},
  volume={41},
  number={2},
  pages={148--164},
  year={1979},
  publisher={Wiley Online Library}
}


@inproceedings{killian2019learning,
  title={Learning to prescribe interventions for tuberculosis patients using digital adherence data},
  author={Killian, Jackson A and Wilder, Bryan and Sharma, Amit and Choudhary, Vinod and Dilkina, Bistra and Tambe, Milind},
  booktitle={Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery \& Data Mining},
  pages={2430--2438},
  year={2019}
}

@article{cassady2005integrating,
  title={Integrating preventive maintenance planning and production scheduling for a single machine},
  author={Cassady, C Richard and Kutanoglu, Erhan},
  journal={IEEE Transactions on reliability},
  volume={54},
  number={2},
  pages={304--309},
  year={2005},
  publisher={IEEE}
}

@article{nguyen2013capture,
  title={Capture: A new predictive anti-poaching tool for wildlife protection},
  author={Nguyen, Thanh H and Sinha, Arunesh and Gholami, Shahrzad and Plumptre, Andrew and Joppa, Lucas and Tambe, Milind and Driciru, Margaret and Wanyama, Fred and Rwetsiba, Aggrey and Critchlow, Rob},
  year={2013}
}

@book{world2018guideline,
  title={WHO guideline on health policy and system support to optimize community health worker programmes},
  author={World Health Organization and others},
  year={2018},
  publisher={World Health Organization}
}


@misc{SpinningUp2018,
    author = {Achiam, J.},
    title = {{Spinning Up in Deep Reinforcement Learning}},
    year = {2018},
    url = {https://github.com/openai/spinningup}
}

@article{Knight2018,
  doi = {10.21105/joss.00904},
  url = {https://doi.org/10.21105/joss.00904},
  year = {2018},
  publisher = {The Open Journal},
  volume = {3},
  number = {30},
  pages = {904},
  author = {V. Knight and J. Campbell},
  title = {Nashpy: A {P}ython library for the computation of {N}ash equilibria},
  journal = {Journal of Open Source Software}
}

@article{yaesoubi2011generalized,
  title={Generalized {M}arkov models of infectious disease spread: A novel framework for developing dynamic health policies},
  author={Yaesoubi, R. and Cohen, T.},
  journal={Eur. J. Oper. Res.},
  volume={215},
  number={3},
  pages={679--687},
  year={2011},
  publisher={Elsevier}
}

@article{schulman2017proximal,
  title={Proximal policy optimization algorithms},
  author={Schulman, J. and Wolski, F. and Dhariwal, P. and Radford, A. and Klimov, O.},
  journal={arXiv preprint arXiv:1707.06347},
  year={2017}
}

@article{yu2021surprising,
  title={The Surprising Effectiveness of {MAPPO} in Cooperative, Multi-Agent Games},
  author={Yu, C. and Velu, A. and Vinitsky, E. and Wang, Y. and Bayen, A. and Wu, Y.},
  journal={arXiv preprint arXiv:2103.01955},
  year={2021}
}


@incollection{NEURIPS2019_9015,
title = {PyTorch: An Imperative Style, High-Performance Deep Learning Library},
author = {Paszke, A. and Gross, S. and Massa, F. and Lerer, A. and Bradbury, J. and Chanan, G. and others},
booktitle = {NeurIPS},
pages = {8024--8035},
year = {2019},
}

@article{hinch2021openabm,
  title={OpenABM-Covid19—An agent-based model for non-pharmaceutical interventions against COVID-19 including contact tracing},
  author={Hinch, R. and Probert, W. J. M. and Nurtay, A. and Kendall, M. and Wymant, C. and Hall, M. and others},
  journal={ PLoS Comput. Biol.},
  volume={17},
  number={7},
  pages={e1009146},
  year={2021},
  publisher={Public Library of Science San Francisco, CA USA}
}

@article{kerr2021covasim,
  title={Covasim: an agent-based model of COVID-19 dynamics and interventions},
  author={Kerr, C. C. and Stuart, R. M. and Mistry, D. and Abeysuriya, R. G. and Rosenfeld, K. and Hart, G. R. and others},
  journal={ PLoS Comput. Biol.},
  volume={17},
  number={7},
  pages={e1009149},
  year={2021},
  publisher={Public Library of Science San Francisco, CA USA}
}

@article{jung2019thompson,
  title={Thompson sampling in non-episodic restless bandits},
  author={Jung, Y. H. and Abeille, M. and Tewari, A.},
  journal={arXiv preprint arXiv:1910.05654},
  year={2019}
}

@article{wang2020restless,
  title={Restless-UCB, an Efficient and Low-complexity Algorithm for Online Restless Bandits},
  author={Wang, S. and Huang, L. and Lui, J.},
  journal={NeurIPS},
  volume={33},
  pages={11878--11889},
  year={2020}
}

@article{bastani2021efficient,
  title={Efficient and targeted COVID-19 border testing via reinforcement learning},
  author={Bastani, H. and Drakopoulos, K. and Gupta, V. and Vlachogiannis, I. and Hadjicristodoulou, C. and Lagiou, P. and others},
  journal={Nature},
  volume={599},
  number={7883},
  pages={108--113},
  year={2021},
  publisher={Nature Publishing Group}
}

@article{song2019solving,
  title={Solving continual combinatorial selection via deep reinforcement learning},
  author={Song, H. and Jang, H. and Tran, H. H. and Yoon, S. and Son, K. and Yun, D. and others},
  journal={arXiv preprint arXiv:1909.03638},
  year={2019}
}

@article{kool2018attention,
  title={Attention, learn to solve routing problems!},
  author={Kool, W. and Van Hoof, H. and Welling, M.},
  journal={arXiv preprint arXiv:1803.08475},
  year={2018}
}

@article{khalil2017learning,
  title={Learning combinatorial optimization algorithms over graphs},
  author={Khalil, E. and Dai, H. and Zhang, Y. and Dilkina, B. and Song, L.},
  journal={NeurIPS},
  volume={30},
  year={2017}
}