
@inproceedings{zhang2017transfer,
  title={Transfer learning in multi-armed bandit: a causal approach},
  author={Zhang, Junzhe and Bareinboim, Elias},
  booktitle={Proceedings of the 16th Conference on Autonomous Agents and MultiAgent Systems},
  pages={1778--1780},
  year={2017}
}

@inproceedings{forney2017counterfactual,
  title={Counterfactual data-fusion for online reinforcement learners},
  author={Forney, Andrew and Pearl, Judea and Bareinboim, Elias},
  booktitle={International Conference on Machine Learning},
  pages={1156--1164},
  year={2017},
  organization={PMLR}
}

@InProceedings{lu2020regret,
  author       = {Lu, Yangyi and Meisami, Amirhossein and Tewari, Ambuj and Yan, William},
  booktitle    = {Conference on Uncertainty in Artificial Intelligence},
  title        = {Regret analysis of bandit problems with causal background knowledge},
  organization = {PMLR},
  pages        = {141--150},
  groups       = {Bandits},
  year         = {2020},
}

@Book{lattimore2020bandit,
  author    = {Lattimore, Tor and Szepesv{\'a}ri, Csaba},
  title     = {Bandit algorithms},
  publisher = {Cambridge University Press},
  groups    = {Bandits},
  year      = {2020},
}

@InProceedings{yabe2018causal,
  author       = {Yabe, Akihiro and Hatano, Daisuke and Sumita, Hanna and Ito, Shinji and Kakimura, Naonori and Fukunaga, Takuro and Kawarabayashi, Ken-ichi},
  booktitle    = {International Conference on Machine Learning},
  title        = {Causal bandits with propagating inference},
  organization = {PMLR},
  pages        = {5512--5520},
  groups       = {Bandits},
  year         = {2018},
}

@Article{ortega2014generalized,
  author    = {Ortega, Pedro A and Braun, Daniel A},
  title     = {Generalized Thompson sampling for sequential decision-making and causal inference},
  number    = {1},
  pages     = {1--23},
  volume    = {2},
  groups    = {Bandits},
  journal   = {Complex Adaptive Systems Modeling},
  publisher = {Springer},
  year      = {2014},
}

@Article{aglietti2020multi,
  author     = {Aglietti, Virginia and Damoulas, Theodoros and {\'A}lvarez, Mauricio and Gonz{\'a}lez, Javier},
  title      = {Multi-task causal learning with gaussian processes},
  groups     = {CBO-CEO, Bandits},
  journal    = {arXiv preprint arXiv:2009.12821},
  printed    = {printed},
  ranking    = {rank4},
  readstatus = {read},
  timestamp  = {2021-10-26},
  year       = {2020},
}

@InProceedings{sen2017identifying,
  author       = {Sen, Rajat and Shanmugam, Karthikeyan and Dimakis, Alexandros G and Shakkottai, Sanjay},
  booktitle    = {International Conference on Machine Learning},
  title        = {Identifying best interventions through online importance sampling},
  organization = {PMLR},
  pages        = {3057--3066},
  groups       = {Bandits},
  year         = {2017},
}

@InProceedings{aglietti2020causal,
  author       = {Aglietti, Virginia and Lu, Xiaoyu and Paleyes, Andrei and Gonz{\'a}lez, Javier},
  booktitle    = {International Conference on Artificial Intelligence and Statistics},
  title        = {Causal Bayesian Optimization},
  organization = {PMLR},
  pages        = {3155--3164},
  groups       = {CBO-CEO, Bandits},
  printed      = {printed},
  ranking      = {rank4},
  readstatus   = {read},
  timestamp    = {2021-10-16},
  year         = {2020},
}

@InProceedings{sen2017contextual,
  author       = {Sen, Rajat and Shanmugam, Karthikeyan and Kocaoglu, Murat and Dimakis, Alex and Shakkottai, Sanjay},
  booktitle    = {Artificial Intelligence and Statistics},
  title        = {Contextual bandits with latent confounders: An nmf approach},
  organization = {PMLR},
  pages        = {518--527},
  groups       = {Bandits},
  year         = {2017},
}

@Article{feng2022combinatorial,
  author  = {Feng, Shi and Chen, Wei},
  title   = {Combinatorial Causal Bandits},
  groups  = {Bandits},
  journal = {arXiv preprint arXiv:2206.01995},
  year    = {2022},
}

@Article{bilodeau2022adaptively,
  author  = {Bilodeau, Blair and Wang, Linbo and Roy, Daniel M},
  title   = {Adaptively Exploiting d-Separators with Causal Bandits},
  groups  = {Causality},
  journal = {arXiv preprint arXiv:2202.05100},
  year    = {2022},
}

@InProceedings{nair2021budgeted,
  author       = {Nair, Vineet and Patil, Vishakha and Sinha, Gaurav},
  booktitle    = {International Conference on Artificial Intelligence and Statistics},
  title        = {Budgeted and non-budgeted causal bandits},
  organization = {PMLR},
  pages        = {2017--2025},
  groups       = {Bandits},
  year         = {2021},
}

@InProceedings{lee2019structural,
  author    = {Lee, Sanghack and Bareinboim, Elias},
  booktitle = {Proceedings of the AAAI Conference on Artificial Intelligence},
  title     = {Structural causal bandits with non-manipulable variables},
  number    = {01},
  pages     = {4164--4172},
  volume    = {33},
  groups    = {Bandits},
  year      = {2019},
}

@InProceedings{lattimore2016causal,
  author    = {Lattimore, Finnian and Lattimore, Tor and Reid, Mark D},
  booktitle = {Advances in Neural Information Processing Systems},
  title     = {Causal bandits: Learning good interventions via causal inference},
  pages     = {1181--1189},
  groups    = {Bandits},
  year      = {2016},
}

@Article{lu2021bandit,
  author  = {Lu, Yangyi and Xu, Ziping and Tewari, Ambuj},
  title   = {Bandit algorithms for precision medicine},
  groups  = {Bandits},
  journal = {arXiv preprint arXiv:2108.04782},
  year    = {2021},
}

@InProceedings{maiti2022causal,
  author       = {Maiti, Aurghya and Nair, Vineet and Sinha, Gaurav},
  booktitle    = {Uncertainty in Artificial Intelligence},
  title        = {A causal bandit approach to learning good atomic interventions in presence of unobserved confounders},
  organization = {PMLR},
  pages        = {1328--1338},
  groups       = {Bandits},
  year         = {2022},
}

@Article{bareinboim2015bandits,
  author  = {Bareinboim, Elias and Forney, Andrew and Pearl, Judea},
  title   = {Bandits with unobserved confounders: A causal approach},
  pages   = {1342--1350},
  volume  = {28},
  groups  = {Bandits},
  journal = {Advances in Neural Information Processing Systems},
  year    = {2015},
}

@InProceedings{sottocornola2021counterfactual,
  author    = {Sottocornola, Gabriele and Stella, Fabio and Zanker, Markus},
  booktitle = {IEEE/WIC/ACM International Conference on Web Intelligence and Intelligent Agent Technology},
  title     = {Counterfactual Contextual Multi-Armed Bandit to Diagnose Post-Harvest Diseases of Apple},
  pages     = {241--248},
  groups    = {Bandits},
  year      = {2021},
}

@InProceedings{lee2018structural,
  author    = {Lee, Sanghack and Bareinboim, Elias},
  booktitle = {Advances in Neural Information Processing Systems},
  title     = {Structural causal bandits: where to intervene?},
  pages     = {2568--2578},
  groups    = {Bandits},
  year      = {2018},
}

@Article{dhir2021chronological,
  author  = {Dhir, Neil},
  title   = {Chronological Causal Bandits},
  groups  = {Bandits},
  journal = {arXiv preprint arXiv:2112.01819},
  year    = {2021},
}

@Article{song2021hierarchical,
  author  = {Song, Ruiyang and Rini, Stefano and Xu, Kuang},
  title   = {Hierarchical Causal Bandit},
  groups  = {Bandits},
  journal = {arXiv preprint arXiv:2103.04215},
  year    = {2021},
}

@Article{lee2020characterizing,
  author  = {Lee, Sanghack and Bareinboim, Elias},
  title   = {Characterizing optimal mixed policies: Where to intervene and what to observe},
  pages   = {8565--8576},
  volume  = {33},
  groups  = {Bandits},
  journal = {Advances in neural information processing systems},
  year    = {2020},
}

@InProceedings{zennaro2023quantifying,
  author    = {Zennaro, Fabio Massimo and Turrini, Paolo and Damoulas, Theodoros},
  booktitle = {Proceedings of the Thrity-Second International Conference on International Joint Conferences on Artificial Intelligence},
  title     = {Quantifying Consistency and Information Loss for Causal Abstraction Learning},
  year      = {2023},
  groups    = {Alpha-Abstraction},
}

@Book{pearl2009causality,
  author    = {Pearl, Judea},
  publisher = {Cambridge University Press},
  title     = {Causality},
  year      = {2009},
}

@mastersthesis{rischel2020category,
  author     = {Rischel, Eigil Fjeldgren},
  title      = {The Category Theory of Causal Models},
  year       = {2020},
  school     = {University of Copenhagen},
}

@Book{sutton2018reinforcement,
  author    = {Sutton, Richard S and Barto, Andrew G},
  publisher = {MIT Press},
  title     = {Reinforcement learning: An introduction},
  year      = {2018},
}

@InProceedings{tian2002general,
  author     = {Tian, Jin and Pearl, Judea},
  booktitle  = {AAAI},
  title      = {A general identification condition for causal effects},
  year       = {2002},
  pages      = {567--573},
}

@InProceedings{zennaro2023jointly,
  author    = {Fabio Massimo Zennaro and M{\'a}t{\'e} Dr{\'a}vucz and Geanina Apachitei and W. Dhammika Widanage and Theodoros Damoulas},
  booktitle = {2nd Conference on Causal Learning and Reasoning},
  title     = {Jointly Learning Consistent Causal Abstractions Over Multiple Interventional Distributions},
  year      = {2023},
  //url       = {https://openreview.net/forum?id=RNs7aMS6zDq},
}

@InProceedings{beckers2018abstracting,
  author     = {Beckers, Sander and Halpern, Joseph Y},
  booktitle  = {Proceedings of the AAAI Conference on Artificial Intelligence},
  title      = {Abstracting Causal Models},
  year       = {2019},
  number     = {01},
  pages      = {2678--2685},
  volume     = {33},
}

@Article{otsuka2022equivalence,
  author  = {Otsuka, Jun and Saigo, Hayato},
  journal = {arXiv preprint arXiv:2201.06981},
  title   = {On the Equivalence of Causal Models: A Category-Theoretic Approach},
  year    = {2022},
}

@InProceedings{beckers2020approximate,
  author       = {Beckers, Sander and Eberhardt, Frederick and Halpern, Joseph Y},
  booktitle    = {Uncertainty in Artificial Intelligence},
  title        = {Approximate causal abstractions},
  year         = {2020},
  organization = {PMLR},
  pages        = {606--615},
}

@Article{massidda2022causal,
  author  = {Massidda, Riccardo and Geiger, Atticus and Icard, Thomas and Bacciu, Davide},
  journal = {arXiv preprint arXiv:2211.12270},
  title   = {Causal Abstraction with Soft Interventions},
  year    = {2022},
}

@Article{geiger2021causal,
  author  = {Geiger, Atticus and Lu, Hanson and Icard, Thomas and Potts, Christopher},
  journal = {Advances in Neural Information Processing Systems},
  title   = {Causal abstractions of neural networks},
  year    = {2021},
  pages   = {9574--9586},
  volume  = {34},
}

@InProceedings{zennaro2022abstraction,
  author    = {Zennaro, Fabio Massimo},
  booktitle = {UAI 2022 Workshop on Causal Representation Learning},
  title     = {Abstraction between Structural Causal Models: A Review of Definitions and Properties},
  year      = {2022},
}

@Book{peters2017elements,
  author    = {Peters, Jonas and Janzing, Dominik and Sch{\"o}lkopf, Bernhard},
  publisher = {MIT Press},
  title     = {Elements of causal inference: {F}oundations and learning algorithms},
  year      = {2017},
}
@Article{rischel2021compositional,
  author     = {Rischel, Eigil F and Weichwald, Sebastian},
  journal    = {arXiv preprint arXiv:2103.15758},
  title      = {Compositional Abstraction Error and a Category of Causal Models},
  year       = {2021},
}

@article{scholkopf2021toward,
  title={Toward causal representation learning},
  author={Sch{\"o}lkopf, Bernhard and Locatello, Francesco and Bauer, Stefan and Ke, Nan Rosemary and Kalchbrenner, Nal and Goyal, Anirudh and Bengio, Yoshua},
  journal={Proceedings of the IEEE},
  volume={109},
  number={5},
  pages={612--634},
  year={2021},
  publisher={IEEE}
}

@Article{chalupka2017causal,
  author    = {Chalupka, Krzysztof and Eberhardt, Frederick and Perona, Pietro},
  journal   = {Behaviormetrika},
  title     = {Causal feature learning: an overview},
  year      = {2017},
  number    = {1},
  pages     = {137--164},
  volume    = {44},
}

@InProceedings{rubenstein2017causal,
  author       = {Rubenstein, Paul K and Weichwald, Sebastian and Bongers, Stephan and Mooij, Joris M and Janzing, Dominik and Grosse-Wentrup, Moritz and Sch{\"o}lkopf, Bernhard},
  booktitle    = {33rd Conference on Uncertainty in Artificial Intelligence (UAI 2017)},
  title        = {Causal consistency of structural equation models},
  year         = {2017},
  organization = {Curran Associates, Inc.},
  pages        = {808--817},
}

@Article{felekis2023causal,
  author  = {Yorgos Felekis and Fabio Massimo Zennaro and Nicola Branchini and Theodoros Damoulas},
  journal = {arXiv preprint arXiv:2312.08107},
  title   = {Causal Optimal Transport of Abstractions},
  year    = {2023},
  groups  = {Tau-Transformation},
}

@incollection{lazaric2012transfer,
  title={Transfer in reinforcement learning: a framework and a survey},
  author={Lazaric, Alessandro},
  booktitle={Reinforcement Learning: State-of-the-Art},
  pages={143--173},
  year={2012},
  publisher={Springer}
}

@article{van2023optimal,
  title={Optimal learning for structured bandits},
  author={Van Parys, Bart and Golrezaei, Negin},
  journal={Management Science},
  year={2023},
  publisher={INFORMS}
}

@article{lazaric2013sequential,
  title={Sequential transfer in multi-armed bandit with finite set of models},
  author={Lazaric, Alessandro and Brunskill, Emma and others},
  journal={Advances in Neural Information Processing Systems},
  volume={26},
  year={2013}
}

@article{lattimore2014bounded,
  title={Bounded regret for finite-armed structured bandits},
  author={Lattimore, Tor and Munos, R{\'e}mi},
  journal={Advances in Neural Information Processing Systems},
  volume={27},
  year={2014}
}

@article{cappe2013kullback,
  title={Kullback-Leibler upper confidence bounds for optimal sequential allocation},
  author={Capp{\'e}, Olivier and Garivier, Aur{\'e}lien and Maillard, Odalric-Ambrym and Munos, R{\'e}mi and Stoltz, Gilles},
  journal={The Annals of Statistics},
  pages={1516--1541},
  year={2013},
  publisher={JSTOR}
}

@inproceedings{audibert2009minimax,
  title={Minimax Policies for Adversarial and Stochastic Bandits.},
  author={Audibert, Jean-Yves and Bubeck, S{\'e}bastien and others},
  booktitle={COLT},
  volume={7},
  pages={1--122},
  year={2009}
}

@inproceedings{russo2014,
 author = {Russo, Daniel and Van Roy, Benjamin},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {Z. Ghahramani and M. Welling and C. Cortes and N. Lawrence and K.Q. Weinberger},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Learning to Optimize via Information-Directed Sampling},
 //url = {https://proceedings.neurips.cc/paper_files/paper/2014/file/301ad0e3bd5cb1627a2044908a42fdc2-Paper.pdf},
 volume = {27},
 year = {2014}
}


@InProceedings{lattimore17a,
  title = 	 {{The End of Optimism? An Asymptotic Analysis of Finite-Armed Linear Bandits}},
  author = 	 {Lattimore, Tor and Szepesvari, Csaba},
  booktitle = 	 {Proceedings of the 20th International Conference on Artificial Intelligence and Statistics},
  pages = 	 {728--737},
  year = 	 {2017},
  editor = 	 {Singh, Aarti and Zhu, Jerry},
  volume = 	 {54},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {20--22 Apr},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v54/lattimore17a/lattimore17a.pdf},
  //url = 	 {https://proceedings.mlr.press/v54/lattimore17a.html},
abstract = 	 {Stochastic linear bandits are a natural and simple generalisation of finite-armed bandits with numerous practical applications. Current approaches focus on generalising existing techniques for finite-armed bandits, notably the otimism principle and Thompson sampling. Prior analysis has mostly focussed on the worst-case setting. We analyse the asymptotic regret and show matching upper and lower bounds on what is achievable. Surprisingly, our results show that no algorithm based on optimism or Thompson sampling will ever achieve the optimal rate. In fact, they can be arbitrarily far from optimal, even in very simple cases. This is a disturbing result because these techniques are standard tools that are widely used for sequential optimisation, for example,  generalised linear bandits and reinforcement learning. }
}


@InProceedings{magureanu14,
  title = 	 {Lipschitz Bandits: Regret Lower Bound and Optimal Algorithms},
  author = 	 {Magureanu, Stefan and Combes, Richard and Proutiere, Alexandre},
  booktitle = 	 {Proceedings of The 27th Conference on Learning Theory},
  pages = 	 {975--999},
  year = 	 {2014},
  editor = 	 {Balcan, Maria Florina and Feldman, Vitaly and Szepesvári, Csaba},
  volume = 	 {35},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {Barcelona, Spain},
  month = 	 {13--15 Jun},
  publisher =    {PMLR},
  //pdf = 	 {http://proceedings.mlr.press/v35/magureanu14.pdf},
  //url = 	 {https://proceedings.mlr.press/v35/magureanu14.html},
  abstract = 	 {We consider stochastic multi-armed bandit problems where the expected reward is a Lipschitz function of the arm, and where the set of arms is either discrete or continuous. For discrete Lipschitz bandits, we derive asymptotic problem specific lower bounds for the regret satisfied by any algorithm, and propose OSLB and CKL-UCB, two algorithms that efficiently exploit the Lipschitz structure of the problem. In fact, we prove  that OSLB is asymptotically optimal, as its asymptotic regret matches the lower bound. The regret analysis of our algorithms relies on a new concentration inequality for weighted sums of KL divergences between the empirical distributions of rewards and their true distributions. For continuous Lipschitz bandits, we propose to first discretize the action space, and then apply OSLB or CKL-UCB, algorithms that provably exploit the structure efficiently. This approach is shown, through numerical experiments, to significantly outperform existing algorithms that directly deal with the continuous set of arms. Finally the results and algorithms are extended to contextual bandits with similarities.}
}

@article{kleinberg2019,
author = {Kleinberg, Robert and Slivkins, Aleksandrs and Upfal, Eli},
title = {Bandits and Experts in Metric Spaces},
year = {2019},
issue_date = {August 2019},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {66},
number = {4},
issn = {0004-5411},
//url = {https://doi.org/10.1145/3299873},
doi = {10.1145/3299873},
abstract = {In a multi-armed bandit problem, an online algorithm chooses from a set of strategies in a sequence of trials to maximize the total payoff of the chosen strategies. While the performance of bandit algorithms with a small finite strategy set is well understood, bandit problems with large strategy sets are still a topic of active investigation, motivated by practical applications, such as online auctions and web advertisement. The goal of such research is to identify broad and natural classes of strategy sets and payoff functions that enable the design of efficient solutions.In this work, we study a general setting for the multi-armed bandit problem, in which the strategies form a metric space, and the payoff function satisfies a Lipschitz condition with respect to the metric. We refer to this problem as the Lipschitz MAB problem. We present a solution for the multi-armed bandit problem in this setting. That is, for every metric space, we define an isometry invariant that bounds from below the performance of Lipschitz MAB algorithms for this metric space, and we present an algorithm that comes arbitrarily close to meeting this bound. Furthermore, our technique gives even better results for benign payoff functions. We also address the full-feedback (“best expert”) version of the problem, where after every round the payoffs from all arms are revealed.},
journal = {J. ACM},
month = {may},
articleno = {30},
numpages = {77},
keywords = {regret, online learning, metric spaces, covering dimension, Multi-armed bandits, Lipschitz-continuity}
}
@inproceedings{lattimorestructure,
 author = {Lattimore, Tor and Munos, Remi},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {Z. Ghahramani and M. Welling and C. Cortes and N. Lawrence and K.Q. Weinberger},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Bounded Regret for Finite-Armed Structured Bandits},
 //url = {https://proceedings.neurips.cc/paper_files/paper/2014/file/ad972f10e0800b49d76fed33a21f6698-Paper.pdf},
 volume = {27},
 year = {2014}
}

@InProceedings{garivier11a,
  title = 	 {The KL-UCB Algorithm for Bounded Stochastic Bandits and Beyond},
  author = 	 {Garivier, Aurélien and Cappé, Olivier},
  booktitle = 	 {Proceedings of the 24th Annual Conference on Learning Theory},
  pages = 	 {359--376},
  year = 	 {2011},
  editor = 	 {Kakade, Sham M. and von Luxburg, Ulrike},
  volume = 	 {19},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {Budapest, Hungary},
  month = 	 {09--11 Jun},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v19/garivier11a/garivier11a.pdf},
  //url = 	 {https://proceedings.mlr.press/v19/garivier11a.html},
  abstract = 	 {This paper presents a finite-time analysis of the KL-UCB algorithm, an online, horizon-free  index policy for stochastic bandit problems.  We prove two distinct results: first, for arbitrary bounded rewards, the KL-UCB algorithm  satisfies a uniformly better regret bound than UCB and its variants; second, in the special case of  Bernoulli rewards, it reaches the lower bound of Lai and Robbins.  Furthermore, we show that simple adaptations of the KL-UCB algorithm are also optimal for  specific classes of (possibly unbounded) rewards, including those generated from exponential  families of distributions.  A large-scale numerical study comparing KL-UCB with its main competitors (UCB, MOSS,  UCB-Tuned, UCB-V, DMED) shows that KL-UCB is remarkably efficient and stable, including for short time horizons. KL-UCB is also the only method that always performs better  than the basic UCB policy.  Our regret bounds rely on deviations results of independent interest which are stated and proved  in the Appendix. As a by-product, we also obtain an improved regret bound for the standard UCB  algorithm.}
}
@article{rusmevichientong2010linearly,
  title={Linearly parameterized bandits},
  author={Rusmevichientong, Paat and Tsitsiklis, John N},
  journal={Mathematics of Operations Research},
  volume={35},
  number={2},
  pages={395--411},
  year={2010},
  publisher={INFORMS}
}
@inproceedings{valkokernel,
author = {Valko, Michal and Korda, Nathan and Munos, R\'{e}mi and Flaounas, Ilias and Cristianini, Nello},
title = {Finite-time analysis of kernelised contextual bandits},
year = {2013},
publisher = {AUAI Press},
address = {Arlington, Virginia, USA},
abstract = {We tackle the problem of online reward maximisation over a large finite set of actions described by their contexts. We focus on the case when the number of actions is too big to sample all of them even once. However we assume that we have access to the similarities between actions' contexts and that the expected reward is an arbitrary linear function of the contexts' images in the related reproducing kernel Hilbert space (RKHS). We propose KernelUCB, a kernelised UCB algorithm, and give a cumulative regret bound through a frequentist analysis. For contextual bandits, the related algorithm GP-UCB turns out to be a special case of our algorithm, and our finite-time analysis improves the regret bound of GP-UCB for the agnostic case, both in the terms of the kernel-dependent quantity and the RKHS norm of the reward function. Moreover, for the linear kernel, our regret bound matches the lower bound for contextual linear bandits.},
booktitle = {Proceedings of the Twenty-Ninth Conference on Uncertainty in Artificial Intelligence},
pages = {654–663},
numpages = {10},
location = {Bellevue, WA},
series = {UAI'13}
}
@inproceedings{li2019nearly,
  title={Nearly minimax-optimal regret for linearly parameterized bandits},
  author={Li, Yingkai and Wang, Yining and Zhou, Yuan},
  booktitle={Conference on Learning Theory},
  pages={2173--2174},
  year={2019},
  organization={PMLR}
}
@inproceedings{russacnonstat,
 author = {Russac, Yoan and Vernade, Claire and Capp\'{e}, Olivier},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Weighted Linear Bandits for Non-Stationary Environments},
 //url = {https://proceedings.neurips.cc/paper_files/paper/2019/file/263fc48aae39f219b4c71d9d4bb4aed2-Paper.pdf},
 volume = {32},
 year = {2019}
}
@inproceedings{zhao2020simple,
  title={A simple approach for non-stationary linear bandits},
  author={Zhao, Peng and Zhang, Lijun and Jiang, Yuan and Zhou, Zhi-Hua},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={746--755},
  year={2020},
  organization={PMLR}
}

@InProceedings{lucucb,
  title = 	 {Regret Analysis of Bandit Problems with Causal Background Knowledge},
  author =       {Lu, Yangyi and Meisami, Amirhossein and Tewari, Ambuj and Yan, William},
  booktitle = 	 {Proceedings of the 36th Conference on Uncertainty in Artificial Intelligence (UAI)},
  pages = 	 {141--150},
  year = 	 {2020},
  editor = 	 {Peters, Jonas and Sontag, David},
  volume = 	 {124},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {03--06 Aug},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v124/lu20a/lu20a.pdf},
  //url = 	 {https://proceedings.mlr.press/v124/lu20a.html},
  abstract = 	 {We study how to learn optimal interventions sequentially given causal information represented as a causal graph along with associated conditional distributions. Causal modeling is useful in real world problems like online advertisement where complex causal mechanisms underlie the relationship between interventions and outcomes. We propose two algorithms, causal upper confidence bound (C-UCB) and causal Thompson Sampling (C-TS), that enjoy improved cumulative regret bounds compared with algorithms that do not use causal information. We thus resolve an open problem posed by Lattimore et al. (2016). Further, we extend C-UCB and C-TS to the linear bandit setting and propose causal linear UCB (CL-UCB) and causal linear TS (CL-TS) algorithms. These algorithms enjoy a cumulative regret bound that only scales with the feature dimension. Our experiments show the benefit of using causal information. For example, we observe that even with a few hundreds of iterations, the regret of causal algorithms is less than that of standard algorithms by a factor of three. We also show that under certain causal structures, our algorithms scale better than the standard bandit algorithms as the number of interventions increases.}
}
@inproceedings{roydsep,
 author = {Bilodeau, Blair and Wang, Linbo and Roy, Dan},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {S. Koyejo and S. Mohamed and A. Agarwal and D. Belgrave and K. Cho and A. Oh},
 pages = {20381--20392},
 publisher = {Curran Associates, Inc.},
 title = {Adaptively Exploiting d-Separators with Causal Bandits},
 //url = {https://proceedings.neurips.cc/paper_files/paper/2022/file/801ec05b0aae9fcd2ef35c168bd538e0-Paper-Conference.pdf},
 volume = {35},
 year = {2022}
}

@InProceedings{pmlr-v130-nair21a,
  title = 	 { Budgeted and Non-Budgeted Causal Bandits },
  author =       {Nair, Vineet and Patil, Vishakha and Sinha, Gaurav},
  booktitle = 	 {Proceedings of The 24th International Conference on Artificial Intelligence and Statistics},
  pages = 	 {2017--2025},
  year = 	 {2021},
  editor = 	 {Banerjee, Arindam and Fukumizu, Kenji},
  volume = 	 {130},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {13--15 Apr},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v130/nair21a/nair21a.pdf},
  //url = 	 {https://proceedings.mlr.press/v130/nair21a.html},
  abstract = 	 { Learning good interventions in a causal graph can be modelled as a stochastic multi-armed bandit problem with side-information. First, we study this problem when interventions are more expensive than observations and a budget is specified. If there are no backdoor paths from the intervenable nodes to the reward node then we propose an algorithm to minimize simple regret that optimally trades-off observations and interventions based on the cost of intervention. We also propose an algorithm that accounts for the cost of interventions, utilizes causal side-information, and minimizes the expected cumulative regret without exceeding the budget. Our algorithm performs better than standard algorithms that do not take side-information into account. Finally, we study the problem of learning best interventions without budget constraint in general graphs and give an algorithm that achieves constant expected cumulative regret in terms of the instance parameters when the parent distribution of the reward variable for each intervention is known. Our results are experimentally validated and compared to the best-known bounds in the current literature. }
}
@article{sharma2020warm,
  title={Warm starting bandits with side information from confounded data},
  author={Sharma, Nihal and Basu, Soumya and Shanmugam, Karthikeyan and Shakkottai, Sanjay},
  journal={arXiv preprint arXiv:2002.08405},
  year={2020}
}

@inproceedings{combes2017,
 author = {Combes, Richard and Magureanu, Stefan and Proutiere, Alexandre},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {I. Guyon and U. Von Luxburg and S. Bengio and H. Wallach and R. Fergus and S. Vishwanathan and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Minimal Exploration in Structured Stochastic Bandits},
 //url = {https://proceedings.neurips.cc/paper_files/paper/2017/file/e19347e1c3ca0c0b97de5fb3b690855a-Paper.pdf},
 volume = {30},
 year = {2017}
}
﻿@article{Singh2024,
author={Singh, Rahul
and Liu, Fang
and Sun, Yin
and Shroff, Ness},
title={Multi-armed bandits with dependent arms},
journal={Machine Learning},
year={2024},
month={Jan},
day={01},
volume={113},
number={1},
pages={45-71},
abstract={We study a variant of the multi-armed bandit problem (MABP) which we call as MABs with dependent arms. Multiple arms are grouped together to form a cluster, and the reward distributions of arms in the same cluster are known functions of an unknown parameter that is a characteristic of the cluster. Thus, pulling an arm i not only reveals information about its own reward distribution, but also about all arms belonging to the same cluster. This ``correlation'' among the arms complicates the exploration--exploitation trade-off that is encountered in the MABP because the observation dependencies allow us to test simultaneously multiple hypotheses regarding the optimality of an arm. We develop learning algorithms based on the principle of optimism in the face of uncertainty (Lattimore and Szepesv{\'a}ri in Bandit algorithms, Cambridge University Press, 2020), which know the clusters, and hence utilize these additional side observations appropriately while performing exploration--exploitation trade-off. We show that the regret of our algorithms grows as {\$}{\$}O(K{\backslash}log T){\$}{\$}, where K is the number of clusters. In contrast, for an algorithm such as the vanilla UCB that does not utilize these dependencies, the regret scales as {\$}{\$}O(M{\backslash}log T){\$}{\$}, where M is the number of arms. When {\$}{\$}K{\backslash}ll M{\$}{\$}, i.e. there is a lot of dependencies among arms, our proposed algorithm drastically reduces the dependence of regret on the number of arms.},
issn={1573-0565},
doi={10.1007/s10994-023-06457-z},
//url={https://doi.org/10.1007/s10994-023-06457-z}
}

@InProceedings{pmlr-v108-tirinzoni20a,
  title = 	 {A Novel Confidence-Based Algorithm for Structured Bandits},
  author =       {Tirinzoni, Andrea and Lazaric, Alessandro and Restelli, Marcello},
  booktitle = 	 {Proceedings of the Twenty Third International Conference on Artificial Intelligence and Statistics},
  pages = 	 {3175--3185},
  year = 	 {2020},
  editor = 	 {Chiappa, Silvia and Calandra, Roberto},
  volume = 	 {108},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {26--28 Aug},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v108/tirinzoni20a/tirinzoni20a.pdf},
  //url = 	 {https://proceedings.mlr.press/v108/tirinzoni20a.html},
  abstract = 	 {We study finite-armed stochastic bandits where the rewards of each arm might be correlated to those of other arms. We introduce a novel phased algorithm that exploits the given structure to build confidence sets over the parameters of the true bandit problem and rapidly discard all sub-optimal arms. In particular, unlike standard bandit algorithms with no structure, we show that the number of times a suboptimal arm is selected may actually be reduced thanks to the information collected by pulling other arms. Furthermore, we show that, in some structures, the regret of an anytime extension of our algorithm is uniformly bounded over time. For these constant-regret structures, we also derive a matching lower bound. Finally, we demonstrate numerically that our approach better exploits certain structures than existing methods.}
}
@article{gupta2021multi,
  title={Multi-armed bandits with correlated arms},
  author={Gupta, Samarth and Chaudhari, Shreyas and Joshi, Gauri and Ya{\u{g}}an, Osman},
  journal={IEEE Transactions on Information Theory},
  volume={67},
  number={10},
  pages={6711--6732},
  year={2021},
  publisher={IEEE}
}
@inproceedings{neuside,
 author = {Koc\'{a}k, Tom\'{a}\v{s} and Neu, Gergely and Valko, Michal and Munos, Remi},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {Z. Ghahramani and M. Welling and C. Cortes and N. Lawrence and K.Q. Weinberger},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Efficient learning by implicit exploration in bandit problems with side observations},
 //url = {https://proceedings.neurips.cc/paper_files/paper/2014/file/25b2822c2f5a3230abfadd476e8b04c9-Paper.pdf},
 volume = {27},
 year = {2014}
}
@inproceedings{mannorside,
 author = {Mannor, Shie and Shamir, Ohad},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {J. Shawe-Taylor and R. Zemel and P. Bartlett and F. Pereira and K.Q. Weinberger},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {From Bandits to Experts: On the Value of Side-Observations},
 //url = {https://proceedings.neurips.cc/paper_files/paper/2011/file/e1e32e235eee1f970470a3a6658dfdd5-Paper.pdf},
 volume = {24},
 year = {2011}
}
@inproceedings{caronstoch,
author = {Caron, St\'{e}phane and Kveton, Branislav and Lelarge, Marc and Bhagat, Smriti},
title = {Leveraging side observations in stochastic bandits},
year = {2012},
isbn = {9780974903989},
publisher = {AUAI Press},
address = {Arlington, Virginia, USA},
abstract = {This paper considers stochastic bandits with side observations, a model that accounts for both the exploration/exploitation dilemma and relationships between arms. In this setting, after pulling an arm i, the decision maker also observes the rewards for some other actions related to i. We will see that this model is suited to content recommendation in social networks, where users' reactions may be endorsed or not by their friends. We provide efficient algorithms based on upper confidence bounds (UCBs) to leverage this additional information and derive new bounds improving on standard regret guarantees. We also evaluate these policies in the context of movie recommendation in social networks: experiments on real datasets show substantial learning rate speedups ranging from 2.2x to 14x on dense networks.},
booktitle = {Proceedings of the Twenty-Eighth Conference on Uncertainty in Artificial Intelligence},
pages = {142–151},
numpages = {10},
location = {Catalina Island, CA},
series = {UAI'12}
}
@inproceedings{10.1145/2591971.2591989,
author = {Buccapatnam, Swapna and Eryilmaz, Atilla and Shroff, Ness B.},
title = {Stochastic bandits with side observations on networks},
year = {2014},
isbn = {9781450327893},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
//url = {https://doi.org/10.1145/2591971.2591989},
doi = {10.1145/2591971.2591989},
abstract = {We study the stochastic multi-armed bandit (MAB) problem in the presence of side-observations across actions. In our model, choosing an action provides additional side observations for a subset of the remaining actions. One example of this model occurs in the problem of targeting users in online social networks where users respond to their friends's activity, thus providing information about each other's preferences. Our contributions are as follows: 1) We derive an asymptotic (with respect to time) lower bound (as a function of the network structure) on the regret (loss) of any uniformly good policy that achieves the maximum long term average reward. 2) We propose two policies - a randomized policy and a policy based on the well-known upper confidence bound (UCB) policies, both of which explore each action at a rate that is a function of its network position. We show that these policies achieve the asymptotic lower bound on the regret up to a multiplicative factor independent of network structure. The upper bound guarantees on the regret of these policies are better than those of existing policies. Finally, we use numerical examples on a real-world social network to demonstrate the significant benefits obtained by our policies against other existing policies.},
booktitle = {The 2014 ACM International Conference on Measurement and Modeling of Computer Systems},
pages = {289–300},
numpages = {12},
keywords = {multiarmed bandits, side observations, social networks},
location = {Austin, Texas, USA},
series = {SIGMETRICS '14}
}

@article{buccapstoch,
author = {Buccapatnam, Swapna and Eryilmaz, Atilla and Shroff, Ness B.},
title = {Stochastic bandits with side observations on networks},
year = {2014},
issue_date = {June 2014},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {42},
number = {1},
issn = {0163-5999},
//url = {https://doi.org/10.1145/2637364.2591989},
doi = {10.1145/2637364.2591989},
abstract = {We study the stochastic multi-armed bandit (MAB) problem in the presence of side-observations across actions. In our model, choosing an action provides additional side observations for a subset of the remaining actions. One example of this model occurs in the problem of targeting users in online social networks where users respond to their friends's activity, thus providing information about each other's preferences. Our contributions are as follows: 1) We derive an asymptotic (with respect to time) lower bound (as a function of the network structure) on the regret (loss) of any uniformly good policy that achieves the maximum long term average reward. 2) We propose two policies - a randomized policy and a policy based on the well-known upper confidence bound (UCB) policies, both of which explore each action at a rate that is a function of its network position. We show that these policies achieve the asymptotic lower bound on the regret up to a multiplicative factor independent of network structure. The upper bound guarantees on the regret of these policies are better than those of existing policies. Finally, we use numerical examples on a real-world social network to demonstrate the significant benefits obtained by our policies against other existing policies.},
journal = {SIGMETRICS Perform. Eval. Rev.},
month = {jun},
pages = {289–300},
numpages = {12},
keywords = {multiarmed bandits, side observations, social networks}
}
@InProceedings{pmlr-v84-wang18b,
  title = 	 {Regional Multi-Armed Bandits},
  author = 	 {Wang, Zhiyang and Zhou, Ruida and Shen, Cong},
  booktitle = 	 {Proceedings of the Twenty-First International Conference on Artificial Intelligence and Statistics},
  pages = 	 {510--518},
  year = 	 {2018},
  editor = 	 {Storkey, Amos and Perez-Cruz, Fernando},
  volume = 	 {84},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {09--11 Apr},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v84/wang18b/wang18b.pdf},
  //url = 	 {https://proceedings.mlr.press/v84/wang18b.html},
  abstract = 	 {We consider a variant of the classic multi-armed bandit problem where the expected reward of each arm is a function of an unknown parameter. The arms are divided into different groups, each of which has a common parameter. Therefore, when the player selects an arm at each time slot, information of other arms in the same group is also revealed. This regional bandit model naturally bridges the non-informative bandit setting where the player can only learn the chosen arm, and the global bandit model where sampling one arms reveals information of all arms. We propose an efficient algorithm, UCB-g, that solves the regional bandit problem by combining the Upper Confidence Bound (UCB) and greedy principles. Both parameter-dependent and parameter-free regret upper bounds are derived. We also establish a matching lower bound, which proves the order-optimality of UCB-g. Moreover, we propose SW-UCB-g, which is an extension of UCB-g for a non-stationary environment where the parameters slowly vary over time.}
}
@ARTICLE{wang2016,
  author={Wang, Zhiyang and Zhou, Ruida and Shen, Cong},
  journal={IEEE Transactions on Signal Processing}, 
  title={Regional Multi-Armed Bandits With Partial Informativeness}, 
  year={2018},
  volume={66},
  number={21},
  pages={5705-5717},
  keywords={Analytical models;Stochastic processes;Drugs;Computational modeling;Upper bound;Pricing;Random variables;Multi-armed bandits (MAB);Online learning;Regret analysis},
  doi={10.1109/TSP.2018.2870383}}

@InProceedings{pmlr-v180-maiti22a,
  title = 	 {A causal bandit approach to learning good atomic interventions in presence of unobserved confounders},
  author =       {Maiti, Aurghya and Nair, Vineet and Sinha, Gaurav},
  booktitle = 	 {Proceedings of the Thirty-Eighth Conference on Uncertainty in Artificial Intelligence},
  pages = 	 {1328--1338},
  year = 	 {2022},
  editor = 	 {Cussens, James and Zhang, Kun},
  volume = 	 {180},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {01--05 Aug},
  publisher =    {PMLR},
  pdf = 	 {https://proceedings.mlr.press/v180/maiti22a/maiti22a.pdf},
  //url = 	 {https://proceedings.mlr.press/v180/maiti22a.html},
  abstract = 	 {We study the problem of determining the best atomic intervention in a Causal Bayesian Network (CBN) specified only by its causal graph. We model this as a stochastic multi-armed bandit (MAB) problem with side-information, where interventions on CBN correspond to arms of the bandit instance. First, we propose a simple regret minimization algorithm that takes as input a causal graph with observable and unobservable nodes and in $T$ exploration rounds achieves $\tilde{O}(\sqrt{m(\mathcal{C})/T})$ expected simple regret. Here $m(\mathcal{C})$ is a parameter dependent on the input CBN $\mathcal{C}$ and could be much smaller than the number of arms. We also show that this is almost optimal for CBNs whose causal graphs have an $n$-ary tree structure.  Next, we propose a cumulative regret minimization algorithm that takes as input a causal graph with observable nodes and performs better than the optimal MAB algorithms that do not use causal side-information. We experimentally compare both our algorithms with the best known algorithms in the literature.}
}
@article{lu2021causal,
  title={Causal bandits with unknown graph structure},
  author={Lu, Yangyi and Meisami, Amirhossein and Tewari, Ambuj},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={24817--24828},
  year={2021}
}

@article{xia2024neural,
  title={Neural Causal Abstractions},
  author={Xia, Kevin and Bareinboim, Elias},
  journal={arXiv preprint arXiv:2401.02602},
  year={2024}
}

@article{dyer2023interventionally,
  title={Interventionally Consistent Surrogates for Agent-based Simulators},
  author={Dyer, Joel and Bishop, Nicholas and Felekis, Yorgos and Zennaro, Fabio Massimo and Calinescu, Anisoara and Damoulas, Theodoros and Wooldridge, Michael},
  journal={arXiv preprint arXiv:2312.11158},
  year={2023}
}
@inproceedings{anand2023causal,
  title={Causal effect identification in cluster DAGs},
  author={Anand, Tara V and Ribeiro, Adele H and Tian, Jin and Bareinboim, Elias},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={37},
  number={10},
  pages={12172--12179},
  year={2023}
}
@article{rojastransfer,
  author  = {Mateo Rojas-Carulla and Bernhard Sch{{\"o}}lkopf and Richard Turner and Jonas Peters},
  title   = {Invariant Models for Causal Transfer Learning},
  journal = {Journal of Machine Learning Research},
  year    = {2018},
  volume  = {19},
  number  = {36},
  pages   = {1--34},
  url     = {http://jmlr.org/papers/v19/16-432.html}
}
@article{Pearl_Bareinboim_2011, 
    title={Transportability of Causal and Statistical Relations: A Formal Approach}, volume={25}, 
    url={https://ojs.aaai.org/index.php/AAAI/article/view/7861}, 
    DOI={10.1609/aaai.v25i1.7861}, 
    number={1}, 
    journal={Proceedings of the AAAI Conference on Artificial Intelligence}, 
    author={Pearl, Judea and Bareinboim, Elias}, year={2011}, 
    month={Aug.}, 
    pages={247-254} 
}