% \documentclass{uai2025} % for initial submission
\documentclass[accepted]{uai2025} % after acceptance, for a revised
                                    % version; also before submission to
                                    % see how the non-anonymous paper
                                    % would look like
%% There is a class option to choose the math font
% \documentclass[mathfont=ptmx]{uai2025} % ptmx math instead of Computer
                                         % Modern (has noticable issues)
% \documentclass[mathfont=newtx]{uai2023} % newtx fonts (improves upon
                                          % ptmx; less tested, no support)
% NOTE: Only keep *one* line above as appropriate, as it will be replaced
%       automatically for papers to be published. Do not make any other
%       change above this note for an accepted version.

%% Choose your variant of English; be consistent
\usepackage[american]{babel}
% \usepackage[british]{babel}

%% Some suggested packages, as needed:
\usepackage{natbib} % has a nice set of citation styles and commands
    \bibliographystyle{plainnat}
    \renewcommand{\bibsection}{\subsubsection*{References}}
\usepackage{mathtools} % amsmath with fixes and additions
% \usepackage{siunitx} % for proper typesetting of numbers and units
\usepackage{booktabs} % commands to create good-looking tables
\usepackage{tikz} % nice language for creating drawings and diagrams

%% Provided macros
% \smaller: Because the class footnote size is essentially LaTeX's \small,
%           redefining \footnotesize, we provide the original \footnotesize
%           using this macro.
%           (Use only sparingly, e.g., in drawings, as it is quite small.)

%% Self-defined macros
\newcommand{\swap}[3][-]{#3#1#2} % just an example

% \title{Instructions for Authors: Title in Title Case}

% The standard author block has changed for UAI 2023 to provide
% more space for long author lists and allow for complex affiliations
%
% All author information is authomatically removed by the class for the
% anonymous submission version of your paper, so you can already add your
% information below.
%
% AUTHORS' PACKAGES
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsfonts}
\usepackage{amsthm}
\usepackage{nomencl}
\usepackage{subcaption}
\usepackage{svg}
\usepackage{soul}
\usepackage{booktabs}
\usepackage{xcolor}
\usepackage{hyperref}
\usepackage{url}
    
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% AUTHORS' COMMANDS
\newcommand{\red}[1]{\textcolor{red}{#1}}
\newcommand{\startpara}[1]{{\vskip5pt\noindent{\bf #1.}}} % custom paragraph header

% references
\newcommand{\sectref}[1]{Section~\ref{#1}}
\newcommand{\figref}[1]{Figure~\ref{#1}}
\newcommand{\tabref}[1]{Table~\ref{#1}}
\newcommand{\egref}[1]{Example~\ref{#1}}
\newcommand{\eqnref}[1]{Equation~\ref{#1}}
\newcommand{\thmref}[1]{Theorem~\ref{#1}}
\newcommand{\propref}[1]{Proposition~\ref{#1}}
\newcommand{\lemref}[1]{Lemma~\ref{#1}}
\newcommand{\defref}[1]{Definition~\ref{#1}}
\newcommand{\agref}[1]{Algorithm~\ref{#1}}
\newcommand{\apref}[1]{Appendix~\ref{#1}}
\newcommand{\sectsectref}[2]{Sections~\ref{#1} and \ref{#2}}
\newcommand{\figfigref}[2]{Figures~\ref{#1} and \ref{#2}}
\newcommand{\egegref}[2]{Examples~\ref{#1} and \ref{#2}}
\newcommand{\eqneqnref}[2]{Equations~\ref{#1} and \ref{#2}}
\newcommand{\tabtabref}[2]{Tables~\ref{#1} and \ref{#2}}
\newcommand{\defdefref}[2]{Definitions~\ref{#1} and \ref{#2}}
\newcommand{\sectsectsectref}[3]{Sections~\ref{#1}, \ref{#2} and \ref{#3}}
\newcommand{\figfigfigref}[3]{Figures~\ref{#1}, \ref{#2} and \ref{#3}}
\newcommand{\egegegref}[3]{Examples~\ref{#1}, \ref{#2} and \ref{#3}}

% page blocks
\usepackage{ifthen}

% environments
\newtheorem{theorem}{Theorem}
\newtheorem{lemma}{Lemma}

% custom example environment
\newcounter{exampcount}
\setcounter{exampcount}{0}
\newenvironment{examp}
{\refstepcounter{exampcount}
\vskip6pt\noindent
{\it Example \arabic{exampcount}.}}
{\hfill$\blacksquare$\vskip6pt}


% maths
\DeclareMathOperator*{\minimize}{minimize}
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\newcommand{\ra}[1]{\renewcommand{\arraystretch}{#1}}

% letters
\def\Nset{\mathbb{N}}
\def\Nsetinf{\mathbb{N}_{\infty}}
\def\Rset{\mathbb{R}}
\def\Rsetgeq{\mathbb{R}_{\geq 0}}
\def\Rpos{\mathbb{R}_{> 0}}
\def\Qset{\mathbb{Q}}
\def\Pset{\mathbb{P}}
\def\cM{{\mathcal{M}}}
\def\cS{{\mathcal{S}}}
\def\cA{{\mathcal{A}}}
\def\cT{{\mathcal{T}}}
\def\cF{{\mathcal{F}}}
\def\cO{{\mathcal{O}}}
\def\cG{{\mathcal{G}}}

% logic operators
\def\next{{\bigcirc}}
\def\until{{\mathsf{U}}}
\def\always{{\Box}}
\def\eventually{{\Diamond}}
\def\true{{\mathsf{true}}}
\def\false{{\mathsf{false}}}
\def\land{{\wedge}}
\def\lor{{\vee}}

% abbreviations
\def\Dist{{\mathit{Dist}}}
\def\Pr{{\mathit{Pr}}}
\def\exp{{\mathbb{E}}}
\def\Sucq{{{\rm Suc}_q}}
\def\Rn{{R_{\mathsf{nv}}^{\otimes}}}
\def\Rp{{R_{\mathsf{pg}}^{\otimes}}}
\def\Rh{{R_{\mathsf{hd}}^{\otimes}}}
\def\bestprogression{{b^*}}
\def\trajectoryset{\mathcal{T}}
\def\projection{\operatorname{proj}}

\newcommand{\Rap}[1]{{R_{\mathsf{ap},#1}^{\otimes}}}
\newcommand{\Rah}[1]{{R_{\mathsf{ah},#1}^{\otimes}}}
\newcommand{\Vn}[1]{{V_{\mathsf{nv}}^{#1}}}
\newcommand{\Vp}[1]{{V_{\mathsf{pg}}^{#1}}}
\newcommand{\Vh}[1]{{V_{\mathsf{hd}}^{#1}}}
\newcommand{\Vap}[2]{{V_{\mathsf{ap},#1}^{#2}}}
\newcommand{\Vah}[2]{{V_{\mathsf{ah},#1}^{#2}}}

\newcommand{\QFi}{Q_{F_i}}
\newcommand{\QFj}{Q_{F_j}}
\newcommand{\QFk}{Q_{F_k}}
\newcommand{\qFi}{q_{F_i}}
\newcommand{\qFj}{q_{F_j}}
\newcommand{\qFk}{q_{F_k}}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Title

% Your title must be in mixed case, not sentence case.
% That means all verbs (including short verbs like be, is, using,and go),
% nouns, adverbs, adjectives should be capitalized, including both words in hyphenated terms, while
% articles, conjunctions, and prepositions are lower case unless they
% directly follow a colon or long dash

\title{Adaptive Reward Design for Reinforcement Learning}
% Add authors
\author[1]{Minjae Kwon}
\author[1]{Ingy ElSayed-Aly}
\author[1]{Lu Feng}
% Add affiliations after the authors

\affil[1]{%
    The Department of Computer Science\\
    University of Virginia\\
    Charlottesville, VA 22904, USA
}
% \affil[2]{%
%     Second Affiliation\\
%     Address\\
%     …
% }
% \affil[3]{%
%     Another Affiliation\\
%     Address\\
%     …
%   }
% \author{Minjae Kwon, Ingy ElSayed-Aly, and Lu Feng
% \thanks{The authors are with the Department of Computer Science, University of Virginia,  
% Email: \{hbt9su, ie3ne, lf9u\}@virginia.edu}
% }


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

\begin{abstract}
There is a surge of interest in using formal languages such as Linear Temporal Logic (LTL) to precisely and succinctly specify complex tasks and derive reward functions for Reinforcement Learning (RL). However, existing methods often assign sparse rewards (e.g., giving a reward of 1 only if a task is completed and 0 otherwise). By providing feedback solely upon task completion, these methods fail to encourage successful subtask completion. This is particularly problematic in environments with inherent uncertainty, where task completion may be unreliable despite progress on intermediate goals. To address this limitation, we propose a suite of reward functions that incentivize an RL agent to complete a task specified by an LTL formula as much as possible, and develop an adaptive reward shaping approach that dynamically updates reward functions during the learning process. Experimental results on a range of benchmark RL environments demonstrate that the proposed approach generally outperforms baselines, achieving earlier convergence to a better policy with higher expected return and task completion rate.
Code is available at 
\url{https://github.com/safe-autonomy-lab/AdaptiveRewardRL.git}.
\end{abstract}

% Uncomment the following to link to your code, datasets, an extended version or similar.
%
% \begin{links}
%     \link{Code}{https://aaai.org/example/code}
%     \link{Datasets}{https://aaai.org/example/datasets}
%     \link{Extended version}{https://aaai.org/example/extended-version}
% \end{links}

%==========================================================================
\section{Introduction} \label{sec:intro} %1p
\input{1_intro}

%==========================================================================
\subsection{Related Work} \label{sec:related} %0.5p
\input{1a_related}

%==========================================================================
\section{Background} \label{sec:background} %1p
\input{2_background}

%==========================================================================
\section{Problem Formulation} \label{sec:problem} %0.75
\input{3_problem}

%==========================================================================
\section{Approach} \label{sec:approach} %1.5p
\input{4_approach}

%==========================================================================
\section{Experiments} \label{sec:exp} %1.25p
\input{5_exp}

%==========================================================================
\section{Conclusion} \label{sec:conclu} %0.25p
\input{6_conclu}

\section*{Acknowledgments}

This work was supported in part by the U.S. National Science Foundation under Grants CCF-1942836 and CCF-2131511. The opinions, findings, conclusions, or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the sponsoring agencies.

%==========================================================================
% \section*{Acknowledgments}
% This work was supported in part by NSF grant CCF-1942836.

% Use IEEE bibliography style
% \bibliographystyle{IEEEtran}
% Generate bibliography from references.bib
\bibliography{references}


\newboolean{arxiv}
\setboolean{arxiv}{true} % Set to true for arXiv version, false for journal

\newpage

\onecolumn

\title{Supplementary Material}
\maketitle

\appendix
\input{7_proof}
\input{8_ablation}


\end{document}
