\relax 
\providecommand\hyper@newdestlabel[2]{}
\providecommand\HyperFirstAtBeginDocument{\AtBeginDocument}
\HyperFirstAtBeginDocument{\ifx\hyper@anchor\@undefined
\global\let\oldnewlabel\newlabel
\gdef\newlabel#1#2{\newlabelxx{#1}#2}
\gdef\newlabelxx#1#2#3#4#5#6{\oldnewlabel{#1}{{#2}{#3}}}
\AtEndDocument{\ifx\hyper@anchor\@undefined
\let\newlabel\oldnewlabel
\fi}
\fi}
\global\let\hyper@last\relax 
\gdef\HyperFirstAtBeginDocument#1{#1}
\providecommand\HyField@AuxAddToFields[1]{}
\providecommand\HyField@AuxAddToCoFields[2]{}
\providecommand\babel@aux[2]{}
\@nameuse{bbl@beforestart}
\bibstyle{plainnat}
\citation{concreteproblems}
\citation{unsolvedproblems}
\citation{activerl}
\providecommand \oddpage@label [2]{}
\babel@aux{american}{}
\@writefile{toc}{\contentsline {section}{\numberline {1}Introduction}{1}{section.1}\protected@file@percent }
\citation{activerl}
\citation{partialmonitoring}
\citation{activerl,owain}
\@writefile{toc}{\contentsline {section}{\numberline {2}Setting Description}{2}{section.2}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {2.1}Setting Description}{2}{subsection.2.1}\protected@file@percent }
\@writefile{toc}{\contentsline {paragraph}{Standard Bandit Settings.}{2}{paragraph*.1}\protected@file@percent }
\@writefile{toc}{\contentsline {paragraph}{Bandits with Costly Observations.}{2}{paragraph*.2}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {2.2}Background and Related Work}{2}{subsection.2.2}\protected@file@percent }
\@writefile{toc}{\contentsline {paragraph}{Active Learning.}{2}{paragraph*.3}\protected@file@percent }
\@writefile{toc}{\contentsline {paragraph}{Partial Monitoring.}{2}{paragraph*.4}\protected@file@percent }
\@writefile{toc}{\contentsline {paragraph}{Best Arm Identification.}{2}{paragraph*.5}\protected@file@percent }
\@writefile{toc}{\contentsline {paragraph}{Active Reinforcement Learning.}{2}{paragraph*.6}\protected@file@percent }
\citation{azuma,hoeffding}
\citation{ajks}
\@writefile{toc}{\contentsline {section}{\numberline {3}Algorithms}{3}{section.3}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {3.1}Algorithm for Multi-armed BwCRO }{3}{subsection.3.1}\protected@file@percent }
\@writefile{toc}{\contentsline {subsubsection}{\numberline {3.1.1}One-armed Bandit Setting and Algorithm}{3}{subsubsection.3.1.1}\protected@file@percent }
\@writefile{loe}{\contentsline {definition}{\ifthmt@listswap Definition~3.1\else \numberline {3.1}Definition\fi \thmtformatoptarg {Disambiguate}}{3}{definition.3.1}\protected@file@percent }
\newlabel{definition:disambiguate}{{3.1}{3}{Disambiguate}{definition.3.1}{}}
\@writefile{loe}{\contentsline {remark}{\ifthmt@listswap \else \numberline {\let \autodot \@empty }\fi Remark}{3}{thmt@dummyctr.dummy.2}\protected@file@percent }
\newlabel{remark:disambiguate}{{}{3}{}{thmt@dummyctr.dummy.2}{}}
\@writefile{loe}{\contentsline {remark}{\ifthmt@listswap \else \numberline {\let \autodot \@empty }\fi Remark}{3}{thmt@dummyctr.dummy.3}\protected@file@percent }
\newlabel{remark:worthit}{{}{3}{}{thmt@dummyctr.dummy.3}{}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {3.1.2}Multi-armed Bandit Algorithm}{4}{subsubsection.3.1.2}\protected@file@percent }
\@writefile{loa}{\contentsline {algorithm}{\numberline {1}{\ignorespaces Worth-it-Width (WiW) Algorithm\relax }}{4}{algorithm.1}\protected@file@percent }
\providecommand*\caption@xref[2]{\@setref\relax\@undefined{#1}}
\newlabel{alg:wiw}{{1}{4}{Worth-it-Width (WiW) Algorithm\relax }{algorithm.1}{}}
\newlabel{thmt@@simple@data}{{\def \theequation {\@arabic {\c@equation }}\def \theHequation {(restate \theHthmt@dummyctr )3.\@arabic {\c@equation }}\setcounter {equation}{0}}{4}{Multi-armed Bandit Algorithm}{algorithm.1}{}}
\@writefile{loe}{\contentsline {theorem}{\ifthmt@listswap Theorem~1\else \numberline {1}Theorem\fi \thmtformatoptarg {Regret Rate for WiW Algorithm}}{4}{theorem.1}\protected@file@percent }
\newlabel{thmt@@simple}{{1}{4}{Regret Rate for WiW Algorithm}{theorem.1}{}}
\newlabel{theorem:simple}{{1}{4}{Regret Rate for WiW Algorithm}{theorem.1}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.2}General Algorithm for BwCRO }{4}{subsection.3.2}\protected@file@percent }
\@writefile{loa}{\contentsline {algorithm}{\numberline {2}{\ignorespaces Fixed-N Algorithm for Multi-armed Bandits\relax }}{5}{algorithm.2}\protected@file@percent }
\newlabel{alg:fixedn}{{2}{5}{Fixed-N Algorithm for Multi-armed Bandits\relax }{algorithm.2}{}}
\newlabel{thmt@@uniformregret@data}{{\def \theequation {\@arabic {\c@equation }}\def \theHequation {(restate \theHthmt@dummyctr )3.\@arabic {\c@equation }}\setcounter {equation}{0}}{5}{General Algorithm for \abbreviation }{algorithm.2}{}}
\@writefile{loe}{\contentsline {assumption}{\ifthmt@listswap Assumption~3.1\else \numberline {3.1}Assumption\fi \thmtformatoptarg {Uniform Regret Rate}}{5}{assumption.3.1}\protected@file@percent }
\newlabel{thmt@@uniformregret}{{3.1}{5}{Uniform Regret Rate}{assumption.3.1}{}}
\newlabel{assumption:uniformregret}{{3.1}{5}{Uniform Regret Rate}{assumption.3.1}{}}
\@writefile{loe}{\contentsline {theorem}{\ifthmt@listswap Theorem~2\else \numberline {2}Theorem\fi \thmtformatoptarg {Regret Rate for Fixed N Algorithm}}{5}{theorem.2}\protected@file@percent }
\newlabel{thm:fixednregret}{{2}{5}{Regret Rate for Fixed N Algorithm}{theorem.2}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.3}Linear Contextual BwCRO}{5}{subsection.3.3}\protected@file@percent }
\@writefile{toc}{\contentsline {subsubsection}{\numberline {3.3.1}$\Delta $ Max Regret Heuristic}{5}{subsubsection.3.3.1}\protected@file@percent }
\citation{li,dani,yadkori}
\citation{ajks}
\citation{ajks}
\citation{activerl}
\citation{partialmonitoring}
\citation{mabintro}
\newlabel{eqn:deltamaxregret}{{1}{6}{$\Delta $ Max Regret Heuristic}{equation.3.1}{}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {3.3.2}Linear Contextual BwCRO Algorithm}{6}{subsubsection.3.3.2}\protected@file@percent }
\@writefile{loa}{\contentsline {algorithm}{\numberline {3}{\ignorespaces Delta Max Regret for LinUCB Algorithm\relax }}{6}{algorithm.3}\protected@file@percent }
\newlabel{alg:linear}{{3}{6}{Delta Max Regret for LinUCB Algorithm\relax }{algorithm.3}{}}
\@writefile{toc}{\contentsline {section}{\numberline {4}Lower bound}{6}{section.4}\protected@file@percent }
\newlabel{thmt@@lowerboundthm@data}{{\def \theequation {\@arabic {\c@equation }}\def \theHequation {(restate \theHthmt@dummyctr )4.\@arabic {\c@equation }}\setcounter {equation}{1}}{6}{Lower bound}{section.4}{}}
\citation{owain}
\citation{activerl}
\@writefile{loe}{\contentsline {theorem}{\ifthmt@listswap Theorem~3\else \numberline {3}Theorem\fi }{7}{theorem.3}\protected@file@percent }
\newlabel{thmt@@lowerboundthm}{{3}{7}{}{theorem.3}{}}
\newlabel{thm:lowerbound}{{3}{7}{}{theorem.3}{}}
\newlabel{eqn:QTK}{{2}{7}{Lower bound}{equation.4.2}{}}
\@writefile{toc}{\contentsline {paragraph}{How many times do we play $k^*$ in instance $0$?}{7}{paragraph*.8}\protected@file@percent }
\newlabel{eqn:Prleq6TK}{{3}{7}{How many times do we play $k^*$ in instance $0$?}{equation.4.3}{}}
\newlabel{eqn:PrJT}{{4}{7}{How many times do we play $k^*$ in instance $0$?}{equation.4.4}{}}
\@writefile{toc}{\contentsline {paragraph}{What is the regret in instance $k^*$?}{7}{paragraph*.9}\protected@file@percent }
\newlabel{eqn:PrkE}{{5}{7}{What is the regret in instance $k^*$?}{equation.4.5}{}}
\newlabel{eqn:Regret}{{6}{7}{What is the regret in instance $k^*$?}{equation.4.6}{}}
\@writefile{toc}{\contentsline {paragraph}{Conclusion.}{7}{paragraph*.10}\protected@file@percent }
\@writefile{toc}{\contentsline {section}{\numberline {5}Experiments}{7}{section.5}\protected@file@percent }
\citation{activerl}
\citation{ajks}
\@writefile{lof}{\contentsline {figure}{\numberline {1}{\ignorespaces Final average per step regret for varying values of gaps $\Delta $, $c=1$, standard error from 20 trials. Dashed red line is at the predicted worst-case $\Delta = \sqrt  [3]{c/T}$. Left graph has horizon $T=1000$, middle has $T=10000$, and right has $T=100000$.\relax }}{8}{figure.caption.7}\protected@file@percent }
\newlabel{fig:delta}{{1}{8}{Final average per step regret for varying values of gaps $\Delta $, $c=1$, standard error from 20 trials. Dashed red line is at the predicted worst-case $\Delta = \sqrt [3]{c/T}$. Left graph has horizon $T=1000$, middle has $T=10000$, and right has $T=100000$.\relax }{figure.caption.7}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {2}{\ignorespaces Probability of committing to the higher value arm for varying values of gaps $\Delta $, $c=1$, and $T=10000$. Standard error from 20 trials. Dashed red line is at the predicted worst-case $\Delta = \sqrt  [3]{c/T}$.\relax }}{8}{figure.caption.12}\protected@file@percent }
\newlabel{fig:pr}{{2}{8}{Probability of committing to the higher value arm for varying values of gaps $\Delta $, $c=1$, and $T=10000$. Standard error from 20 trials. Dashed red line is at the predicted worst-case $\Delta = \sqrt [3]{c/T}$.\relax }{figure.caption.12}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {5.1}Multi-Armed Bandit Evaluation}{8}{subsection.5.1}\protected@file@percent }
\@writefile{toc}{\contentsline {paragraph}{Baselines.}{8}{paragraph*.11}\protected@file@percent }
\@writefile{toc}{\contentsline {paragraph}{Results.}{8}{paragraph*.13}\protected@file@percent }
\@writefile{lof}{\contentsline {figure}{\numberline {3}{\ignorespaces $c=100, T=10000$. The simpler baseline can achieve similar performance to the Worth-it-Width algorithm, but does considerably worse when the difference between arms $\Delta $ is small.\relax }}{8}{figure.caption.15}\protected@file@percent }
\newlabel{fig:baseline}{{3}{8}{$c=100, T=10000$. The simpler baseline can achieve similar performance to the Worth-it-Width algorithm, but does considerably worse when the difference between arms $\Delta $ is small.\relax }{figure.caption.15}{}}
\@writefile{toc}{\contentsline {paragraph}{Worth-it-Width Ablations.}{8}{paragraph*.14}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {5.2}Linear Contextual Evaluation}{8}{subsection.5.2}\protected@file@percent }
\@writefile{toc}{\contentsline {paragraph}{Baselines.}{8}{paragraph*.16}\protected@file@percent }
\citation{yahoo}
\citation{krause_yahoo}
\bibdata{bibliography}
\bibcite{yadkori}{{1}{2011}{{Abbasi-yadkori et~al.}}{{Abbasi-yadkori, P\'{a}l, and Szepesv\'{a}ri}}}
\@writefile{lof}{\contentsline {figure}{\numberline {4}{\ignorespaces Final average per step regret for varying noises $\sigma ^2$, standard error from 20 trials. Note the logarithmic $y$ scale. The contexts have dimension $d=5$, drawn from $\mathcal  {N}(0, 1)^d$ and rescaled to size $1$. Left has $c=1$ and right has $c=10$.\relax }}{9}{figure.caption.17}\protected@file@percent }
\newlabel{fig:noise}{{4}{9}{Final average per step regret for varying noises $\sigma ^2$, standard error from 20 trials. Note the logarithmic $y$ scale. The contexts have dimension $d=5$, drawn from $\calN (0, 1)^d$ and rescaled to size $1$. Left has $c=1$ and right has $c=10$.\relax }{figure.caption.17}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {5}{\ignorespaces Average per-step reward at each timestep for 1000 timesteps rejection sampled using the Yahoo! Frontpage Dataset. Standard error from 20 trials. Left graph has cost $c=0.01$ and right has $c=0.1$. Non-red dashed lines correspond to using the doubling trick.\relax }}{9}{figure.caption.19}\protected@file@percent }
\newlabel{fig:yahoo}{{5}{9}{Average per-step reward at each timestep for 1000 timesteps rejection sampled using the Yahoo! Frontpage Dataset. Standard error from 20 trials. Left graph has cost $c=0.01$ and right has $c=0.1$. Non-red dashed lines correspond to using the doubling trick.\relax }{figure.caption.19}{}}
\@writefile{toc}{\contentsline {paragraph}{Results.}{9}{paragraph*.18}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {5.3}Evaluation on Real-World Data}{9}{subsection.5.3}\protected@file@percent }
\@writefile{toc}{\contentsline {paragraph}{Baselines.}{9}{paragraph*.20}\protected@file@percent }
\@writefile{toc}{\contentsline {paragraph}{The Doubling Trick.}{9}{paragraph*.21}\protected@file@percent }
\@writefile{toc}{\contentsline {paragraph}{Results.}{9}{paragraph*.22}\protected@file@percent }
\@writefile{toc}{\contentsline {section}{\numberline {6}Conclusions}{9}{section.6}\protected@file@percent }
\bibcite{ajks}{{2}{2023}{{Agarwal et~al.}}{{Agarwal, Jiang, Kakade, and Sun}}}
\bibcite{concreteproblems}{{3}{2016}{{Amodei et~al.}}{{Amodei, Olah, Steinhardt, Christiano, Schulman, and Mané}}}
\bibcite{azuma}{{4}{1967}{{Azuma}}{{}}}
\bibcite{partialmonitoring}{{5}{2014}{{Bart{\'o}k et~al.}}{{Bart{\'o}k, Foster, P{\'a}l, Rakhlin, and Szepesv{\'a}ri}}}
\bibcite{yahoo}{{6}{2009}{{Chu et~al.}}{{Chu, Park, Beaupre, Motgi, Phadke, Chakraborty, and Zachariah}}}
\bibcite{dani}{{7}{2008}{{Dani et~al.}}{{Dani, Hayes, and Kakade}}}
\bibcite{unsolvedproblems}{{8}{2021}{{Hendrycks et~al.}}{{Hendrycks, Carlini, Schulman, and Steinhardt}}}
\bibcite{hoeffding}{{9}{1963}{{Hoeffding}}{{}}}
\bibcite{activerl}{{10}{2016}{{Krueger et~al.}}{{Krueger, Leike, Evans, and Salvatier}}}
\bibcite{li}{{11}{2010}{{Li et~al.}}{{Li, Chu, Langford, and Schapire}}}
\bibcite{owain}{{12}{2018}{{Schulze and Evans}}{{}}}
\bibcite{mabintro}{{13}{2019}{{Slivkins}}{{}}}
\bibcite{krause_yahoo}{{14}{2014}{{Vanchinathan et~al.}}{{Vanchinathan, Nikolic, De~Bona, and Krause}}}
\newlabel{LastPage}{{}{10}{}{page.10}{}}
\xdef\lastpage@lastpage{10}
\xdef\lastpage@lastpageHy{10}
\gdef \@abspage@last{10}
