
@incollection{Bengio+chapter2007,
author = {Bengio, Yoshua and LeCun, Yann},
booktitle = {Large Scale Kernel Machines},
publisher = {MIT Press},
title = {Scaling Learning Algorithms Towards {AI}},
year = {2007}
}

@article{Hinton06,
author = {Hinton, Geoffrey E. and Osindero, Simon and Teh, Yee Whye},
journal = {Neural Computation},
pages = {1527--1554},
title = {A Fast Learning Algorithm for Deep Belief Nets},
volume = {18},
year = {2006}
}

@book{goodfellow2016deep,
title={Deep learning},
author={Goodfellow, Ian and Bengio, Yoshua and Courville, Aaron and Bengio, Yoshua},
volume={1},
year={2016},
publisher={MIT Press}
}

% 
@inproceedings{amos2017optnet,
  title={Optnet: Differentiable optimization as a layer in neural networks},
  author={Amos, Brandon and Kolter, J Zico},
  booktitle={International Conference on Machine Learning},
  pages={136--145},
  year={2017},
  organization={PMLR}
}

@article{mattingley2012cvxgen,
  title={CVXGEN: A code generator for embedded convex optimization},
  author={Mattingley, Jacob and Boyd, Stephen},
  journal={Optimization and Engineering},
  volume={13},
  number={1},
  pages={1--27},
  year={2012},
  publisher={Springer}
}

@inproceedings{bhatia2019resource,
  title={Resource constrained deep reinforcement learning},
  author={Bhatia, Abhinav and Varakantham, Pradeep and Kumar, Akshat},
  booktitle={Proceedings of the International Conference on Automated Planning and Scheduling},
  volume={29},
  pages={610--620},
  year={2019}
}

@inproceedings{geibel2006reinforcement,
  title={Reinforcement learning for MDPs with constraints},
  author={Geibel, Peter},
  booktitle={European Conference on Machine Learning},
  pages={646--653},
  year={2006},
  organization={Springer}
}

@article{Zhou_Tomlin_2018, title={Budget-Constrained Multi-Armed Bandits With Multiple Plays}, volume={32}, url={https://ojs.aaai.org/index.php/AAAI/article/view/11629}, abstractNote={ &lt;p&gt; We study the multi-armed bandit problem with multiple plays and a budget constraint for both the stochastic and the adversarial setting. At each round, exactly K out of N possible arms have to be played (with 1 ≤ K &amp;lt;= N). In addition to observing the individual rewards for each arm played, the player also learns a vector of costs which has to be covered with an a-priori defined budget B. The game ends when the sum of current costs associated with the played arms exceeds the remaining budget. Firstly, we analyze this setting for the stochastic case, for which we assume each arm to have an underlying cost and reward distribution with support [c&lt;sub&gt;min&lt;/sub&gt;, 1] and [0, 1], respectively. We derive an Upper Confidence Bound (UCB) algorithm which achieves O(NK&lt;sup&gt;4&lt;/sup&gt; log B) regret. Secondly, for the adversarial case in which the entire sequence of rewards and costs is fixed in advance, we derive an upper bound on the regret of order O(√NB log(N/K)) utilizing an extension of the well-known Exp3 algorithm. We also provide upper bounds that hold with high probability and a lower bound of order Ω((1 – K/N) √NB/K). &lt;/p&gt; }, number={1}, journal={Proceedings of the AAAI Conference on Artificial Intelligence}, author={Zhou, Datong and Tomlin, Claire}, year={2018}, month={Apr.} }

@article{denardo2013multi,
  title={The multi-armed bandit, with constraints},
  author={Denardo, Eric V and Feinberg, Eugene A and Rothblum, Uriel G},
  journal={Annals of Operations Research},
  volume={208},
  number={1},
  pages={37--62},
  year={2013},
  publisher={Springer}
}

@article{deva2021multi,
  title={A Multi-Arm Bandit Approach To Subset Selection Under Constraints},
  author={Deva, Ayush and Abhishek, Kumar and Gujar, Sujit},
  journal={arXiv preprint arXiv:2102.04824},
  year={2021}
}

@inproceedings{achiam2017constrained,
  title={Constrained policy optimization},
  author={Achiam, Joshua and Held, David and Tamar, Aviv and Abbeel, Pieter},
  booktitle={International Conference on Machine Learning},
  pages={22--31},
  year={2017},
  organization={PMLR}
}

@inproceedings{agrawal2016scalable,
  title={Scalable greedy algorithms for task/resource constrained multi-agent stochastic planning.(2016)},
  author={AGRAWAL, Pritee and VARAKANTHAM, Pradeep and YEOH, William},
  booktitle={Proceedings of the 25th International Joint Conference on Artificial Intelligence IJCAI 2016: New York, July 9},
  volume={15},
  pages={10--16},
  year={2016}
}

@phdthesis{de2019resource,
  title={Resource-constrained multi-agent Markov decision processes},
  author={De Nijs, Frits},
  year={2019},
  school={Delft University of Technology}
}

@article{de2017bounding, title={Bounding the Probability of Resource Constraint Violations in Multi-Agent MDPs}, volume={31}, url={https://ojs.aaai.org/index.php/AAAI/article/view/11037}, abstractNote={ &lt;p&gt; Multi-agent planning problems with constraints on global resource consumption occur in several domains. Existing algorithms for solving Multi-agent Markov Decision Processes can compute policies that meet a resource constraint in expectation, but these policies provide no guarantees on the probability that a resource constraint violation will occur. We derive a method to bound constraint violation probabilities using Hoeffding’s inequality. This method is applied to two existing approaches for computing policies satisfying constraints: the Constrained MDP framework and a Column Generation approach. We also introduce an algorithm to adaptively relax the bound up to a given maximum violation tolerance. Experiments on a hard toy problem show that the resulting policies outperform static optimal resource allocations to an arbitrary level. By testing the algorithms on more realistic planning domains from the literature, we demonstrate that the adaptive bound is able to efficiently trade off violation probability with expected value, outperforming state-of-the-art planners. &lt;/p&gt; }, number={1}, journal={Proceedings of the AAAI Conference on Artificial Intelligence}, author={de Nijs, Frits and Walraven, Erwin and de Weerdt, Mathijs and Spaan, Matthijs}, year={2017}, month={Feb.} }

@article{meshram2018whittle,
  title={On the Whittle index for restless multiarmed hidden Markov bandits},
  author={Meshram, Rahul and Manjunath, D and Gopalan, Aditya},
  journal={IEEE Transactions on Automatic Control},
  volume={63},
  number={9},
  pages={3046--3053},
  year={2018},
  publisher={IEEE}
}

@article{whittle1988restless,
  title={Restless bandits: Activity allocation in a changing world},
  author={Whittle, Peter},
  journal={Journal of applied probability},
  pages={287--298},
  year={1988},
  publisher={JSTOR}
}

@article{liu2010indexability,
  title={Indexability of restless bandit problems and optimality of whittle index for dynamic multichannel access},
  author={Liu, Keqin and Zhao, Qing},
  journal={IEEE Transactions on Information Theory},
  volume={56},
  number={11},
  pages={5547--5567},
  year={2010},
  publisher={IEEE}
}

@inproceedings{qian2016restless,
  title={Restless poachers: Handling exploration-exploitation tradeoffs in security domains},
  author={Qian, Yundi and Zhang, Chao and Krishnamachari, Bhaskar and Tambe, Milind},
  booktitle={Proceedings of the 2016 International Conference on Autonomous Agents \& Multiagent Systems},
  pages={123--131},
  year={2016}
}

@article{mate2020collapsing,
  title={Collapsing bandits and their application to public health interventions},
  author={Mate, Aditya and Killian, Jackson A and Xu, Haifeng and Perrault, Andrew and Tambe, Milind},
  journal={arXiv preprint arXiv:2007.04432},
  year={2020}
}

@article{sandholm2020multi,
  title={A Multi-Armed Bandit-based Approach to Mobile Network Provider Selection},
  author={Sandholm, Thomas and Mukherjee, Sayandev},
  journal={arXiv preprint arXiv:2012.04755},
  year={2020}
}

@inproceedings{fu2019towards,
  title={Towards Q-learning the Whittle Index for Restless Bandits},
  author={Fu, Jing and Nazarathy, Yoni and Moka, Sarat and Taylor, Peter G},
  booktitle={2019 Australian \& New Zealand Control Conference (ANZCC)},
  pages={249--254},
  year={2019},
  organization={IEEE}
}

@phdthesis{raghunathan2007physical,
  title={Physical layer aware algorithms and protocols for wireless networks},
  author={Raghunathan, Vivek},
  year={2007},
  school={University of Illinois at Urbana-Champaign}
}

% New added MAB with fairness constraints reference

@article{li2019combinatorial,
  title={Combinatorial sleeping bandits with fairness constraints},
  author={Li, Fengjiao and Liu, Jia and Ji, Bo},
  journal={IEEE Transactions on Network Science and Engineering},
  volume={7},
  number={3},
  pages={1799--1813},
  year={2019},
  publisher={IEEE}
}

@article{patil2020achieving, title={Achieving Fairness in the Stochastic Multi-Armed Bandit Problem}, volume={34}, DOI={10.1609/aaai.v34i04.5986}, number={04}, journal={Proceedings of the AAAI Conference on Artificial Intelligence}, author={Patil, Vishakha and Ghalme, Ganesh and Nair, Vineet and Narahari, Y.}, year={2020}, month={Apr.}, pages={5379-5386} }

@inproceedings{chen2020fair,
  title={Fair contextual multi-armed bandits: Theory and experiments},
  author={Chen, Yifang and Cuellar, Alex and Luo, Haipeng and Modi, Jignesh and Nemlekar, Heramb and Nikolaidis, Stefanos},
  booktitle={Conference on Uncertainty in Artificial Intelligence},
  pages={181--190},
  year={2020},
  organization={PMLR}
}

@article{joseph2016fairness,
  title={Fairness in learning: Classic and contextual bandits},
  author={Joseph, Matthew and Kearns, Michael and Morgenstern, Jamie and Roth, Aaron},
  journal={arXiv preprint arXiv:1605.07139},
  year={2016}
}

@article{katehakis1987multi,
  title={The multi-armed bandit problem: decomposition and computation},
  author={Katehakis, Michael N and Veinott Jr, Arthur F},
  journal={Mathematics of Operations Research},
  volume={12},
  number={2},
  pages={262--268},
  year={1987},
  publisher={INFORMS}
}

@article{abbou2019group,
  title={Group maintenance: A restless bandits approach},
  author={Abbou, Abderrahmane and Makis, Viliam},
  journal={INFORMS Journal on Computing},
  volume={31},
  number={4},
  pages={719--731},
  year={2019},
  publisher={INFORMS}
}

@article{weber1990index,
  title={On an index policy for restless bandits},
  author={Weber, Richard R and Weiss, Gideon},
  journal={Journal of applied probability},
  pages={637--648},
  year={1990},
  publisher={JSTOR}
}

@article{mate2021efficient,
  title={Efficient Algorithms for Finite Horizon and Streaming Restless Multi-Armed Bandit Problems},
  author={Mate, Aditya and Biswas, Arpita and Siebenbrunner, Christoph and Tambe, Milind},
  journal={arXiv preprint arXiv:2103.04730},
  year={2021}
}

@article{hu2017asymptotically,
  title={An asymptotically optimal index policy for finite-horizon restless bandits},
  author={Hu, Weici and Frazier, Peter},
  journal={arXiv preprint arXiv:1707.00205},
  year={2017}
}

@article{gittins1979bandit,
  title={Bandit processes and dynamic allocation indices},
  author={Gittins, John C},
  journal={Journal of the Royal Statistical Society: Series B (Methodological)},
  volume={41},
  number={2},
  pages={148--164},
  year={1979},
  publisher={Wiley Online Library}
}

@article{biswas2021learn,
  title={Learn to Intervene: An Adaptive Learning Policy for Restless Bandits in Application to Preventive Healthcare},
  author={Biswas, Arpita and Aggarwal, Gaurav and Varakantham, Pradeep and Tambe, Milind},
  journal={arXiv preprint arXiv:2105.07965},
  year={2021}
}

@inproceedings{mate2021risk,
  title={Risk-aware interventions in public health: Planning with restless multi-armed bandits},
  author={Mate, Aditya and Perrault, Andrew and Tambe, Milind},
  booktitle={Proceedings of the 20th International Conference on Autonomous Agents and MultiAgent Systems},
  pages={880--888},
  year={2021}
}

@inproceedings{raghunathan2008index,
  title={Index policies for real-time multicast scheduling for wireless broadcast systems},
  author={Raghunathan, Vivek and Borkar, Vivek and Cao, Min and Kumar, P Roshan},
  booktitle={IEEE INFOCOM 2008-The 27th Conference on Computer Communications},
  pages={1570--1578},
  year={2008},
  organization={IEEE}
}

@inproceedings{meshram2017hidden,
  title={A hidden Markov restless multi-armed bandit model for playout recommendation systems},
  author={Meshram, Rahul and Gopalan, Aditya and Manjunath, D},
  booktitle={International Conference on Communication Systems and Networks},
  pages={335--362},
  year={2017},
  organization={Springer}
}

@article{watkins1989learning,
  title={Learning from delayed rewards},
  author={Watkins, Christopher John Cornish Hellaby},
  year={1989},
  publisher={King's College, Cambridge United Kingdom},
}

@article{avrachenkov2020whittle,
  title={Whittle index based Q-learning for restless bandits with average reward},
  author={Avrachenkov, Konstantin and Borkar, Vivek S},
  journal={arXiv preprint arXiv:2004.14427},
  year={2020}
}

@article{kaelbling1998planning,
  title={Planning and acting in partially observable stochastic domains},
  author={Kaelbling, Leslie Pack and Littman, Michael L and Cassandra, Anthony R},
  journal={Artificial intelligence},
  volume={101},
  number={1-2},
  pages={99--134},
  year={1998},
  publisher={Elsevier}
}


@inproceedings{papadimitriou1994,
  title={The complexity of optimal queueing network control},
  author={Papadimitriou, Christos H and Tsitsiklis, John N},
  booktitle={Proceedings of IEEE 9th Annual Conference on Structure in Complexity Theory},
  pages={318--322},
  year={1994},
  organization={IEEE}
}

@inproceedings{li2021claim,
  title={CLAIM: Curriculum learning policy for influence maximization in unknown social networks},
  author={Li, Dexun and Lowalekar, Meghna and Varakantham, Pradeep},
  booktitle={Uncertainty in Artificial Intelligence},
  pages={1455--1465},
  year={2021},
  organization={PMLR}
}

@article{nahum2012q,
  title={Q-learning: a data analysis method for constructing adaptive interventions.},
  author={Nahum-Shani, Inbal and Qian, Min and Almirall, Daniel and Pelham, William E and Gnagy, Beth and Fabiano, Gregory A and Waxmonsky, James G and Yu, Jihnhee and Murphy, Susan A},
  journal={Psychological methods},
  volume={17},
  number={4},
  pages={478},
  year={2012},
  publisher={American Psychological Association}
}

@inproceedings{gopalan2015thompson,
  title={Thompson sampling for learning parameterized markov decision processes},
  author={Gopalan, Aditya and Mannor, Shie},
  booktitle={Conference on Learning Theory},
  pages={861--898},
  year={2015},
  organization={PMLR}
}

@inproceedings{meshram2016optimal,
  title={Optimal recommendation to users that react: Online learning for a class of POMDPs},
  author={Meshram, Rahul and Gopalan, Aditya and Manjunath, D},
  booktitle={2016 IEEE 55th Conference on Decision and Control (CDC)},
  pages={7210--7215},
  year={2016},
  organization={IEEE}
}

@article{thompson1933likelihood,
  title={On the likelihood that one unknown probability exceeds another in view of the evidence of two samples},
  author={Thompson, William R},
  journal={Biometrika},
  volume={25},
  number={3/4},
  pages={285--294},
  year={1933},
  publisher={JSTOR}
}

@article{watkins1992q,
  title={Q-learning},
  author={Watkins, Christopher JCH and Dayan, Peter},
  journal={Machine learning},
  volume={8},
  number={3},
  pages={279--292},
  year={1992},
  publisher={Springer}
}

@inproceedings{agrawal2012analysis,
  title={Analysis of thompson sampling for the multi-armed bandit problem},
  author={Agrawal, Shipra and Goyal, Navin},
  booktitle={Conference on learning theory},
  pages={39--1},
  year={2012},
  organization={JMLR Workshop and Conference Proceedings}
}


@article{lee2019optimal,
  title={Optimal screening for hepatocellular carcinoma: A restless bandit model},
  author={Lee, Elliot and Lavieri, Mariel S and Volk, Michael},
  journal={Manufacturing \& Service Operations Management},
  volume={21},
  number={1},
  pages={198--212},
  year={2019},
  publisher={INFORMS}
}

@article{hirsch1989convergent,
  title={Convergent activation dynamics in continuous time networks},
  author={Hirsch, Morris W},
  journal={Neural networks},
  volume={2},
  number={5},
  pages={331--349},
  year={1989},
  publisher={Elsevier}
}

@article{jaakkola1994convergence,
  title={On the convergence of stochastic iterative dynamic programming algorithms},
  author={Jaakkola, Tommi and Jordan, Michael I and Singh, Satinder P},
  journal={Neural computation},
  volume={6},
  number={6},
  pages={1185--1201},
  year={1994},
  publisher={MIT Press}
}

@article{villar2016indexability,
  title={Indexability and optimal index policies for a class of reinitialising restless bandits},
  author={Villar, Sof{\'\i}a S},
  journal={Probability in the engineering and informational sciences},
  volume={30},
  number={1},
  pages={1--23},
  year={2016},
  publisher={Cambridge University Press}
}

@article{glazebrook2006some,
  title={Some indexable families of restless bandit problems},
  author={Glazebrook, Kevin D and Ruiz-Hernandez, Diego and Kirkbride, Christopher},
  journal={Advances in Applied Probability},
  volume={38},
  number={3},
  pages={643--672},
  year={2006},
  publisher={Cambridge University Press}
}

@article{nino2011computing,
  title={Computing a classic index for finite-horizon bandits},
  author={Nino-Mora, Jos{\'e}},
  journal={INFORMS Journal on Computing},
  volume={23},
  number={2},
  pages={254--267},
  year={2011},
  publisher={INFORMS}
}
