@inbook{Das_2024,
   title={Linear Contextual Bandits with Hybrid Payoff: Revisited},
   ISBN={9783031703652},
   ISSN={1611-3349},
   url={http://dx.doi.org/10.1007/978-3-031-70365-2_26},
   DOI={10.1007/978-3-031-70365-2_26},
   booktitle={Machine Learning and Knowledge Discovery in Databases. Research Track},
   publisher={Springer Nature Switzerland},
   author={Das, Nirjhar and Sinha, Gaurav},
   year={2024},
   pages={441–455} }

@misc{Faury2022,
      title={Jointly Efficient and Optimal Algorithms for Logistic Bandits}, 
      author={Louis Faury and Marc Abeille and Kwang-Sung Jun and Clément Calauzènes},
      year={2022},
      eprint={2201.01985},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2201.01985}, 
}

@misc{Faury2020,
      title={Improved Optimistic Algorithms for Logistic Bandits}, 
      author={Louis Faury and Marc Abeille and Clément Calauzènes and Olivier Fercoq},
      year={2020},
      eprint={2002.07530},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2002.07530}, 
}


@misc{Chatterji2020,
      title={OSOM: A simultaneously optimal algorithm for multi-armed and linear contextual bandits}, 
      author={Niladri S. Chatterji and Vidya Muthukumar and Peter L. Bartlett},
      year={2020},
      eprint={1905.10040},
      archivePrefix={arXiv},
      primaryClass={stat.ML},
      url={https://arxiv.org/abs/1905.10040}, 
}

@misc{Abeille2021,
      title={Instance-Wise Minimax-Optimal Algorithms for Logistic Bandits}, 
      author={Marc Abeille and Louis Faury and Clément Calauzènes},
      year={2021},
      eprint={2010.12642},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2010.12642}, 
}

@inproceedings{Yadkori2011,
 author = {Abbasi-yadkori, Yasin and P\'{a}l, D\'{a}vid and Szepesv\'{a}ri, Csaba},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {J. Shawe-Taylor and R. Zemel and P. Bartlett and F. Pereira and K.Q. Weinberger},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Improved Algorithms for Linear Stochastic Bandits},
 url = {https://proceedings.neurips.cc/paper_files/paper/2011/file/e1d5be1c7f2f456670de3d53c7b54f4a-Paper.pdf},
 volume = {24},
 year = {2011}
}

@article{Tropp_2011a,
   title={User-Friendly Tail Bounds for Sums of Random Matrices},
   volume={12},
   ISSN={1615-3383},
   url={http://dx.doi.org/10.1007/s10208-011-9099-z},
   DOI={10.1007/s10208-011-9099-z},
   number={4},
   journal={Foundations of Computational Mathematics},
   publisher={Springer Science and Business Media LLC},
   author={Tropp, Joel A.},
   year={2011},
   month=aug, pages={389–434} }

@article{Tropp2011b,
author = {Tropp, Joel},
year = {2011},
month = {01},
pages = {},
title = {Freedman’s Inequality for Matrix Martingales},
volume = {16},
journal = {Electronic Communications in Probability},
doi = {10.1214/ECP.v16-1624}
}

@misc{Ghosh2020,
      title={Problem-Complexity Adaptive Model Selection for Stochastic Linear Bandits}, 
      author={Avishek Ghosh and Abishek Sankararaman and Kannan Ramchandran},
      year={2020},
      eprint={2006.02612},
      archivePrefix={arXiv},
      primaryClass={stat.ML},
      url={https://arxiv.org/abs/2006.02612}, 
}

@misc{Gentile2017,
      title={On Context-Dependent Clustering of Bandits}, 
      author={Claudio Gentile and Shuai Li and Purushottam Kar and Alexandros Karatzoglou and Evans Etrue and Giovanni Zappella},
      year={2017},
      eprint={1608.03544},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/1608.03544}, 
}

@inproceedings{Graepel2010,
author = {Graepel, Thore and Candela, Joaquin Qui\~{n}onero and Borchert, Thomas and Herbrich, Ralf},
title = {Web-scale Bayesian click-through rate prediction for sponsored search advertising in Microsoft's bing search engine},
year = {2010},
isbn = {9781605589077},
publisher = {Omnipress},
address = {Madison, WI, USA},
abstract = {We describe a new Bayesian click-through rate (CTR) prediction algorithm used for Sponsored Search in Microsoft's Bing search engine. The algorithm is based on a probit regression model that maps discrete or real-valued input features to probabilities. It maintains Gaussian beliefs over weights of the model and performs Gaussian online updates derived from approximate message passing. Scalability of the algorithm is ensured through a principled weight pruning procedure and an approximate parallel implementation. We discuss the challenges arising from evaluating and tuning the predictor as part of the complex system of sponsored search where the predictions made by the algorithm decide about future training sample composition. Finally, we show experimental results from the production system and compare to a calibrated Na\"{\i}ve Bayes algorithm.},
booktitle = {Proceedings of the 27th International Conference on International Conference on Machine Learning},
pages = {13–20},
numpages = {8},
location = {Haifa, Israel},
series = {ICML'10}
}

@inproceedings{Dimakopoulou2019,
  title     = {Marginal Posterior Sampling for Slate Bandits},
  author    = {Dimakopoulou, Maria and Vlassis, Nikos and Jebara, Tony},
  booktitle = {Proceedings of the Twenty-Eighth International Joint Conference on
               Artificial Intelligence, {IJCAI-19}},
  publisher = {International Joint Conferences on Artificial Intelligence Organization},
  pages     = {2223--2229},
  year      = {2019},
  month     = {7},
  doi       = {10.24963/ijcai.2019/308},
  url       = {https://doi.org/10.24963/ijcai.2019/308},
}

@inproceedings{Hill_2017, series={KDD ’17},
   title={An Efficient Bandit Algorithm for Realtime Multivariate Optimization},
   url={http://dx.doi.org/10.1145/3097983.3098184},
   DOI={10.1145/3097983.3098184},
   booktitle={Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining},
   publisher={ACM},
   author={Hill, Daniel N. and Nassif, Houssam and Liu, Yi and Iyer, Anand and Vishwanathan, S.V.N.},
   year={2017},
   month=aug, collection={KDD ’17} }

@misc{Sawarni2024,
      title={Generalized Linear Bandits with Limited Adaptivity}, 
      author={Ayush Sawarni and Nirjhar Das and Siddharth Barman and Gaurav Sinha},
      year={2024},
      eprint={2404.06831},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2404.06831}, 
}

@inproceedings{Swaminathan2017,
 author = {Swaminathan, Adith and Krishnamurthy, Akshay and Agarwal, Alekh and Dudik, Miro and Langford, John and Jose, Damien and Zitouni, Imed},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {I. Guyon and U. Von Luxburg and S. Bengio and H. Wallach and R. Fergus and S. Vishwanathan and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Off-policy evaluation for slate recommendation},
 url = {https://proceedings.neurips.cc/paper_files/paper/2017/file/5352696a9ca3397beb79f116f3a33991-Paper.pdf},
 volume = {30},
 year = {2017}
}

@inproceedings{Vlassis2021,
author = {Vlassis, Nikos and Chandrashekar, Ashok and Gil, Fernando Amat and Kallus, Nathan},
title = {Control variates for slate off-policy evaluation},
year = {2024},
isbn = {9781713845393},
publisher = {Curran Associates Inc.},
address = {Red Hook, NY, USA},
booktitle = {Proceedings of the 35th International Conference on Neural Information Processing Systems},
articleno = {280},
numpages = {13},
series = {NIPS '21}
}

@misc{Rhuggenaath2020,
author = {Rhuggenaath, Jason and Akcay, Alp and Zhang, Yingqian and Kaymak, Uzay},
year = {2020},
month = {04},
pages = {},
title = {Algorithms for slate bandits with non-separable reward functions},
doi = {10.48550/arXiv.2004.09957}
}

@inproceedings{Kale2010,
 author = {Kale, Satyen and Reyzin, Lev and Schapire, Robert E},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {J. Lafferty and C. Williams and J. Shawe-Taylor and R. Zemel and A. Culotta},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Non-Stochastic Bandit Slate Problems},
 url = {https://proceedings.neurips.cc/paper_files/paper/2010/file/390e982518a50e280d8e2b535462ec1f-Paper.pdf},
 volume = {23},
 year = {2010}
}

@inproceedings{Kiyohara2024,
author = {Kiyohara, Haruka and Nomura, Masahiro and Saito, Yuta},
title = {Off-Policy Evaluation of Slate Bandit Policies via Optimizing Abstraction},
year = {2024},
isbn = {9798400701719},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3589334.3645343},
doi = {10.1145/3589334.3645343},
abstract = {We study off-policy evaluation (OPE) in the problem of slate contextual bandits where a policy selects multi-dimensional actions known as slates. This problem is widespread in recommender systems, search engines, marketing, to medical applications, however, the typical Inverse Propensity Scoring (IPS) estimator suffers from substantial variance due to large action spaces, making effective OPE a significant challenge. The PseudoInverse (PI) estimator has been introduced to mitigate the variance issue by assuming linearity in the reward function, but this can result in significant bias as this assumption is hard-to-verify from observed data and is often substantially violated. To address the limitations of previous estimators, we develop a novel estimator for OPE of slate bandits, called Latent IPS (LIPS), which defines importance weights in a low-dimensional slate abstraction space where we optimize slate abstractions to minimize the bias and variance of LIPS in a data-driven way. By doing so, LIPS can substantially reduce the variance of IPS without imposing restrictive assumptions on the reward function structure like linearity. Through empirical evaluation, we demonstrate that LIPS substantially outperforms existing estimators, particularly in scenarios with non-linear rewards and large slate spaces.},
booktitle = {Proceedings of the ACM Web Conference 2024},
pages = {3150–3161},
numpages = {12},
keywords = {inverse propensity score, off-policy evaluation, slate bandits},
location = {Singapore, Singapore},
series = {WWW '24}
}


@book{Lattimore2020, 
place={Cambridge}, 
title={Bandit Algorithms}, 
publisher={Cambridge University Press}, 
author={Lattimore, Tor and Szepesvári, Csaba}, 
year={2020}
}


@InProceedings{Agrawal2013,
  title = 	 {Thompson Sampling for Contextual Bandits with Linear Payoffs},
  author = 	 {Agrawal, Shipra and Goyal, Navin},
  booktitle = 	 {Proceedings of the 30th International Conference on Machine Learning},
  pages = 	 {127--135},
  year = 	 {2013},
  editor = 	 {Dasgupta, Sanjoy and McAllester, David},
  volume = 	 {28},
  number =       {3},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {Atlanta, Georgia, USA},
  month = 	 {17--19 Jun},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v28/agrawal13.pdf},
  url = 	 {https://proceedings.mlr.press/v28/agrawal13.html},
  abstract = 	 {Thompson Sampling is one of the oldest heuristics for multi-armed bandit problems. It is a randomized algorithm based on Bayesian ideas, and has recently generated significant interest after several studies demonstrated it to have better empirical performance compared to the state of the art methods. However, many questions regarding its theoretical performance remained open. In this paper, we design and analyze Thompson Sampling algorithm for the stochastic contextual multi-armed bandit problem with linear payoff functions, when the contexts are provided by an adaptive adversary. This is among the most important and widely studied version of the contextual bandits problem. We prove a high probability regret bound of \tildeO(\fracd\sqrtε\sqrtT^1+ε) in time T for any ε∈(0,1), where d is the dimension of each context vector and εis a parameter used by the algorithm. Our results provide the first theoretical guarantees for the contextual version of Thompson Sampling, and are close to the lower bound of Ω(\sqrtdT) for this problem. This essentially solves the COLT open problem of Chapelle and Li [COLT 2012] regarding regret bounds for Thompson Sampling for contextual bandits problem with linear payoff functions.     Our version of Thompson sampling uses Gaussian prior and Gaussian likelihood function. Our novel martingale-based analysis techniques also allow easy extensions to the use of other distributions, satisfying certain general conditions.   }
}

@misc{Lee2024,
      title={A Unified Confidence Sequence for Generalized Linear Models, with Applications to Bandits}, 
      author={Junghyun Lee and Se-Young Yun and Kwang-Sung Jun},
      year={2024},
      eprint={2407.13977},
      archivePrefix={arXiv},
      primaryClass={stat.ML},
      url={https://arxiv.org/abs/2407.13977}, 
}

@misc{Liu2024,
      title={Almost Free: Self-concordance in Natural Exponential Families and an Application to Bandits}, 
      author={Shuai Liu and Alex Ayoub and Flore Sentenac and Xiaoqi Tan and Csaba Szepesvári},
      year={2024},
      eprint={2410.01112},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2410.01112}, 
}

@misc{Papini2021,
      title={Leveraging Good Representations in Linear Contextual Bandits}, 
      author={Matteo Papini and Andrea Tirinzoni and Marcello Restelli and Alessandro Lazaric and Matteo Pirotta},
      year={2021},
      eprint={2104.03781},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2104.03781}, 
}

@misc{Tempera2022,
      title={TEMPERA: Test-Time Prompting via Reinforcement Learning}, 
      author={Tianjun Zhang and Xuezhi Wang and Denny Zhou and Dale Schuurmans and Joseph E. Gonzalez},
      year={2022},
      eprint={2211.11890},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2211.11890}, 
}

@inproceedings{Filippi2010,
 author = {Filippi, Sarah and Cappe, Olivier and Garivier, Aur\'{e}lien and Szepesv\'{a}ri, Csaba},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {J. Lafferty and C. Williams and J. Shawe-Taylor and R. Zemel and A. Culotta},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Parametric Bandits: The Generalized Linear Case},
 url = {https://proceedings.neurips.cc/paper_files/paper/2010/file/c2626d850c80ea07e7511bbae4c76f4b-Paper.pdf},
 volume = {23},
 year = {2010}
}


@InProceedings{Abeille2017,
  title = 	 {{Linear Thompson Sampling Revisited}},
  author = 	 {Abeille, Marc and Lazaric, Alessandro},
  booktitle = 	 {Proceedings of the 20th International Conference on Artificial Intelligence and Statistics},
  pages = 	 {176--184},
  year = 	 {2017},
  editor = 	 {Singh, Aarti and Zhu, Jerry},
  volume = 	 {54},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {20--22 Apr},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v54/abeille17a/abeille17a.pdf},
  url = 	 {https://proceedings.mlr.press/v54/abeille17a.html}
}

@article{Thompson1933,
 ISSN = {00063444},
 URL = {http://www.jstor.org/stable/2332286},
 author = {William R. Thompson},
 journal = {Biometrika},
 number = {3/4},
 pages = {285--294},
 publisher = {[Oxford University Press, Biometrika Trust]},
 title = {On the Likelihood that One Unknown Probability Exceeds Another in View of the Evidence of Two Samples},
 urldate = {2025-02-04},
 volume = {25},
 year = {1933}
}

@article{Russo2018,
author = {Russo, Daniel J. and Van Roy, Benjamin and Kazerouni, Abbas and Osband, Ian and Wen, Zheng},
title = {A Tutorial on Thompson Sampling},
year = {2018},
issue_date = {Jul 2018},
publisher = {Now Publishers Inc.},
address = {Hanover, MA, USA},
volume = {11},
number = {1},
issn = {1935-8237},
url = {https://doi.org/10.1561/2200000070},
doi = {10.1561/2200000070},
journal = {Found. Trends Mach. Learn.},
month = jul,
pages = {1–96},
numpages = {99},
keywords = {Exploration, Bandit learning}
}

@inproceedings{Li2010,
author = {Li, Lihong and Chu, Wei and Langford, John and Schapire, Robert E.},
title = {A contextual-bandit approach to personalized news article recommendation},
year = {2010},
isbn = {9781605587998},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/1772690.1772758},
doi = {10.1145/1772690.1772758},
booktitle = {Proceedings of the 19th International Conference on World Wide Web},
pages = {661–670},
numpages = {10},
keywords = {contextual bandit, exploration/exploitation dilemma, personalization, recommender systems, web service},
location = {Raleigh, North Carolina, USA},
series = {WWW '10}
}

@inproceedings{Zhuang2021,
    title = "A Robustly Optimized {BERT} Pre-training Approach with Post-training",
    author = "Zhuang, Liu  and
      Wayne, Lin  and
      Ya, Shi  and
      Jun, Zhao",
    editor = "Li, Sheng  and
      Sun, Maosong  and
      Liu, Yang  and
      Wu, Hua  and
      Liu, Kang  and
      Che, Wanxiang  and
      He, Shizhu  and
      Rao, Gaoqi",
    booktitle = "Proceedings of the 20th Chinese National Conference on Computational Linguistics",
    month = aug,
    year = "2021",
    address = "Huhhot, China",
    publisher = "Chinese Information Processing Society of China",
    url = "https://aclanthology.org/2021.ccl-1.108/",
    pages = "1218--1227",
    language = "eng",
}

@article{Chen2021,
  title={Automated Creative Optimization for E-Commerce Advertising},
  author={Jin Chen and Ju Xu and Gangwei Jiang and Tiezheng Ge and Zhiqiang Zhang and Defu Lian and Kai Zheng},
  journal={Proceedings of the Web Conference 2021},
  year={2021},
  url={https://api.semanticscholar.org/CorpusID:232076065}
}

@misc{lagree2016,
      title={Multiple-Play Bandits in the Position-Based Model}, 
      author={Paul Lagrée and Claire Vernade and Olivier Cappé},
      year={2016},
      eprint={1606.02448},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/1606.02448}, 
}

@article{Wang2017, title={Efficient Ordered Combinatorial Semi-Bandits for Whole-Page Recommendation}, volume={31}, url={https://ojs.aaai.org/index.php/AAAI/article/view/10939}, DOI={10.1609/aaai.v31i1.10939}, abstractNote={ &lt;p&gt; Multi-Armed Bandit (MAB) framework has been successfully applied in many web applications. However, many complex real-world applications that involve multiple content recommendations cannot fit into the traditional MAB setting. To address this issue, we consider an ordered combinatorial semi-bandit problem where the learner recommends S actions from a base set of&lt;em&gt; K&lt;/em&gt; actions, and displays the results in &lt;em&gt;S&lt;/em&gt; (out of &lt;em&gt;M&lt;/em&gt;) different positions. The aim is to maximize the cumulative reward with respect to the best possible subset and positions in hindsight. By the adaptation of a minimum-cost maximum-flow network, a practical algorithm based on Thompson sampling is derived for the (contextual) combinatorial problem, thus resolving the problem of computational intractability.With its potential to work with whole-page recommendation and any probabilistic models, to illustrate the effectiveness of our method, we focus on Gaussian process optimization and a contextual setting where click-through rate is predicted using logistic regression. We demonstrate the algorithms’ performance on synthetic Gaussian process problems and on large-scale news article recommendation datasets from Yahoo! Front Page Today Module. &lt;/p&gt; }, number={1}, journal={Proceedings of the AAAI Conference on Artificial Intelligence}, author={Wang, Yingfei and Ouyang, Hua and Wang, Chu and Chen, Jianhui and Asamov, Tsvetan and Chang, Yi}, year={2017}, month={Feb.} }


@InProceedings{li2016,
  title = 	 {Contextual Combinatorial Cascading Bandits},
  author = 	 {Li, Shuai and Wang, Baoxiang and Zhang, Shengyu and Chen, Wei},
  booktitle = 	 {Proceedings of The 33rd International Conference on Machine Learning},
  pages = 	 {1245--1253},
  year = 	 {2016},
  editor = 	 {Balcan, Maria Florina and Weinberger, Kilian Q.},
  volume = 	 {48},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {New York, New York, USA},
  month = 	 {20--22 Jun},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v48/lif16.pdf},
  url = 	 {https://proceedings.mlr.press/v48/lif16.html},
  abstract = 	 {We propose the contextual combinatorial cascading bandits, a combinatorial online learning game, where at each time step a learning agent is given a set of contextual information, then selects a list of items, and observes stochastic outcomes of a prefix in the selected items by some stopping criterion. In online recommendation, the stopping criterion might be the first item a user selects; in network routing, the stopping criterion might be the first edge blocked in a path. We consider position discounts in the list order, so that the agent’s reward is discounted depending on the position where the stopping criterion is met. We design a UCB-type algorithm, C^3-UCB, for this problem, prove an n-step regret bound \tildeO(\sqrtn) in the general setting, and give finer analysis for two special cases. Our work generalizes existing studies in several directions, including contextual information, position discounts, and a more general cascading bandit model. Experiments on synthetic and real datasets demonstrate the advantage of involving contextual information and position discounts.}
}

@misc{nussbaum2024nomic,
      title={Nomic Embed: Training a Reproducible Long Context Text Embedder}, 
      author={Zach Nussbaum and John X. Morris and Brandon Duderstadt and Andriy Mulyar},
      year={2024},
      eprint={2402.01613},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}
