%% This BibTeX bibliography file was created using BibDesk.
%% http://bibdesk.sourceforge.net/





%% Saved with string encoding Unicode (UTF-8)
@article{rodbard2009interpretation,
	title={Interpretation of continuous glucose monitoring data: glycemic variability and quality of glycemic control},
	author={Rodbard, David},
	journal={Diabetes technology \& therapeutics},
	volume={11},
	number={S1},
	pages={S--55},
	year={2009},
	publisher={Mary Ann Liebert, Inc. 140 Huguenot Street, 3rd Floor New Rochelle, NY 10801 USA}
}

@inproceedings{kuzborskij2021confident,
  title={Confident off-policy evaluation and selection through self-normalized importance weighting},
  author={Kuzborskij, Ilja and Vernade, Claire and Gyorgy, Andras and Szepesv{\'a}ri, Csaba},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={640--648},
  year={2021},
  organization={PMLR}
}

@article{yang2020offline,
  title={Offline policy selection under uncertainty},
  author={Yang, Mengjiao and Dai, Bo and Nachum, Ofir and Tucker, George and Schuurmans, Dale},
  journal={arXiv preprint arXiv:2012.06919},
  year={2020}
}

@article{mathe2006lepskii,
  title={The Lepskii principle revisited},
  author={Math{\'e}, Peter},
  journal={Inverse problems},
  volume={22},
  number={3},
  pages={L11},
  year={2006},
  publisher={IOP Publishing}
}

@article{yu2020mopo,
  title={Mopo: Model-based offline policy optimization},
  author={Yu, Tianhe and Thomas, Garrett and Yu, Lantao and Ermon, Stefano and Zou, James and Levine, Sergey and Finn, Chelsea and Ma, Tengyu},
  journal={arXiv preprint arXiv:2005.13239},
  year={2020}
}
@article{xie2021bellman,
  title={Bellman-consistent Pessimism for Offline Reinforcement Learning},
  author={Xie, Tengyang and Cheng, Ching-An and Jiang, Nan and Mineiro, Paul and Agarwal, Alekh},
  journal={arXiv preprint arXiv:2106.06926},
  year={2021}
}
@article{zanette2021provable,
  title={Provable Benefits of Actor-Critic Methods for Offline Reinforcement Learning},
  author={Zanette, Andrea and Wainwright, Martin J and Brunskill, Emma},
  journal={arXiv preprint arXiv:2108.08812},
  year={2021}
}
@article{uehara2021pessimistic,
  title={Pessimistic Model-based Offline RL: PAC Bounds and Posterior Sampling under Partial Coverage},
  author={Uehara, Masatoshi and Sun, Wen},
  journal={arXiv preprint arXiv:2107.06226},
  year={2021}
}
@inproceedings{xie2021batch,
  title={Batch value-function approximation with only realizability},
  author={Xie, Tengyang and Jiang, Nan},
  booktitle={International Conference on Machine Learning},
  pages={11404--11413},
  year={2021},
  organization={PMLR}
}
@article{mcleish1974dependent,
  title={Dependent central limit theorems and invariance principles},
  author={McLeish, Donald L},
  journal={the Annals of Probability},
  volume={2},
  number={4},
  pages={620--628},
  year={1974},
  publisher={Institute of Mathematical Statistics}
}

@article{freedman1975tail,
  title={On tail probabilities for martingales},
  author={Freedman, David A},
  journal={the Annals of Probability},
  pages={100--118},
  year={1975},
  publisher={JSTOR}
}

@inproceedings{su2020adaptive,
  title={Adaptive estimator selection for off-policy evaluation},
  author={Su, Yi and Srinath, Pavithra and Krishnamurthy, Akshay},
  booktitle={International Conference on Machine Learning},
  pages={9196--9205},
  year={2020},
  organization={PMLR}
}

@article{uehara2021finite,
  title={Finite sample analysis of minimax offline reinforcement learning: Completeness, fast rates and first-order efficiency},
  author={Uehara, Masatoshi and Imaizumi, Masaaki and Jiang, Nan and Kallus, Nathan and Sun, Wen and Xie, Tengyang},
  journal={arXiv preprint arXiv:2102.02981},
  year={2021}
}

@article{lepski1997optimal,
  title={Optimal pointwise adaptive methods in nonparametric estimation},
  author={Lepski, Oleg V and Spokoiny, Vladimir G},
  journal={The Annals of Statistics},
  pages={2512--2546},
  year={1997},
  publisher={JSTOR}
}

@article{tang2021model,
  title={Model Selection for Offline Reinforcement Learning: Practical Considerations for Healthcare Settings},
  author={Tang, Shengpu and Wiens, Jenna},
  journal={arXiv preprint arXiv:2107.11003},
  year={2021}
}



@article{hu2021fast,
  title={Fast Rates for the Regret of Offline Reinforcement Learning},
  author={Hu, Yichun and Kallus, Nathan and Uehara, Masatoshi},
  journal={arXiv preprint arXiv:2102.00479},
  year={2021}
}

@article{luedtke2016statistical,
  title={Statistical inference for the mean outcome under a possibly non-unique optimal treatment strategy},
  author={Luedtke, Alexander R and Van Der Laan, Mark J},
  journal={Annals of statistics},
  volume={44},
  number={2},
  pages={713},
  year={2016},
  publisher={NIH Public Access}
}

@article{paine2020hyperparameter,
  title={Hyperparameter selection for offline reinforcement learning},
  author={Paine, Tom Le and Paduraru, Cosmin and Michi, Andrea and Gulcehre, Caglar and Zolna, Konrad and Novikov, Alexander and Wang, Ziyu and de Freitas, Nando},
  journal={arXiv preprint arXiv:2007.09055},
  year={2020}
}

@inproceedings{kallus2020statistically,
  title={Statistically efficient off-policy policy gradients},
  author={Kallus, Nathan and Uehara, Masatoshi},
  booktitle={International Conference on Machine Learning},
  pages={5089--5100},
  year={2020},
  organization={PMLR}
}



@article{shi2018high,
	title={High-dimensional A-learning for optimal dynamic treatment regimes},
	author={Shi, Chengchun and Fan, Alin and Song, Rui and Lu, Wenbin},
	journal={Annals of statistics},
	volume={46},
	number={3},
	pages={925},
	year={2018},
	publisher={NIH Public Access}
}

@article{lecun2015deep,
	title={Deep learning},
	author={LeCun, Yann and Bengio, Yoshua and Hinton, Geoffrey},
	journal={nature},
	volume={521},
	number={7553},
	pages={436--444},
	year={2015},
	publisher={Nature Publishing Group}
}

@article{hu2021personalized,
	title={Personalized Policy Learning using Longitudinal Mobile Health Data},
	author={Hu, Xinyu and Qian, Min and Cheng, Bin and Cheung, Ying Kuen},
	journal={Journal of the American Statistical Association},
	volume={116},
	number={533},
	pages={410--420},
	year={2021},
	publisher={Taylor \& Francis}
}

@article{wang2018quantile,
	title={Quantile-optimal treatment regimes},
	author={Wang, Lan and Zhou, Yu and Song, Rui and Sherwood, Ben},
	journal={Journal of the American Statistical Association},
	volume={113},
	number={523},
	pages={1243--1254},
	year={2018},
	publisher={Taylor \& Francis}
}

@inproceedings{riedmiller2005neural,
	title={Neural fitted Q iteration--first experiences with a data efficient neural reinforcement learning method},
	author={Riedmiller, Martin},
	booktitle={European conference on machine learning},
	pages={317--328},
	year={2005},
	organization={Springer}
}

@book{tsiatis2019dynamic,
	title={Dynamic Treatment Regimes: Statistical Methods for Precision Medicine},
	author={Tsiatis, Anastasios A and Davidian, Marie and Holloway, Shannon T and Laber, Eric B},
	year={2019},
	publisher={CRC press}
}

@article{brown1971martingale,
	title={Martingale central limit theorems},
	author={Brown, Bruce M and others},
	journal={The Annals of Mathematical Statistics},
	volume={42},
	number={1},
	pages={59--66},
	year={1971},
	publisher={Institute of Mathematical Statistics}
}

@article{candes2016panning,
  title={Panning for gold: Model-X knockoffs for high-dimensional controlled variable selection},
  author={Candes, Emmanuel and Fan, Yingying and Janson, Lucas and Lv, Jinchi},
  journal={arXiv preprint arXiv:1610.02351},
  year={2016}
}


@article{levine2020offline,
  title={Offline reinforcement learning: Tutorial, review, and perspectives on open problems},
  author={Levine, Sergey and Kumar, Aviral and Tucker, George and Fu, Justin},
  journal={arXiv preprint arXiv:2005.01643},
  year={2020}
}

@article{hubbs2020or,
  title={OR-Gym: A Reinforcement Learning Library for Operations Research Problem},
  author={Hubbs, Christian D and Perez, Hector D and Sarwar, Owais and Sahinidis, Nikolaos V and Grossmann, Ignacio E and Wassick, John M},
  journal={arXiv preprint arXiv:2008.06319},
  year={2020}
}

@article{li2017deep,
  title={Deep reinforcement learning: An overview},
  author={Li, Yuxi},
  journal={arXiv preprint arXiv:1701.07274},
  year={2017}
}


@inproceedings{dabney2018distributional,
  title={Distributional reinforcement learning with quantile regression},
  author={Dabney, Will and Rowland, Mark and Bellemare, Marc and Munos, R{\'e}mi},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={32},
  number={1},
  year={2018}
}

@article{mnih2015human,
  title={Human-level control through deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Rusu, Andrei A and Veness, Joel and Bellemare, Marc G and Graves, Alex and Riedmiller, Martin and Fidjeland, Andreas K and Ostrovski, Georg and others},
  journal={nature},
  volume={518},
  number={7540},
  pages={529--533},
  year={2015},
  publisher={Nature Publishing Group}
}


@article{watkins1992q,
  title={Q-learning},
  author={Watkins, Christopher JCH and Dayan, Peter},
  journal={Machine learning},
  volume={8},
  number={3-4},
  pages={279--292},
  year={1992},
  publisher={Springer}
}


@inproceedings{abbeel2004apprenticeship,
  title={Apprenticeship learning via inverse reinforcement learning},
  author={Abbeel, Pieter and Ng, Andrew Y},
  booktitle={Proceedings of the twenty-first international conference on Machine learning},
  pages={1},
  year={2004}
}


@article{rust1987optimal,
  title={Optimal replacement of GMC bus engines: An empirical model of Harold Zurcher},
  author={Rust, John},
  journal={Econometrica: Journal of the Econometric Society},
  pages={999--1033},
  year={1987},
  publisher={JSTOR}
}

@article{komorowski2018artificial,
  title={The artificial intelligence clinician learns optimal treatment strategies for sepsis in intensive care},
  author={Komorowski, Matthieu and Celi, Leo A and Badawi, Omar and Gordon, Anthony C and Faisal, A Aldo},
  journal={Nature medicine},
  volume={24},
  number={11},
  pages={1716--1720},
  year={2018},
  publisher={Nature Publishing Group}
}

@article{silver2016mastering,
  title={Mastering the game of Go with deep neural networks and tree search},
  author={Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and Van Den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and others},
  journal={nature},
  volume={529},
  number={7587},
  pages={484--489},
  year={2016},
  publisher={Nature Publishing Group}
}


@article{kumar2020conservative,
  title={Conservative q-learning for offline reinforcement learning},
  author={Kumar, Aviral and Zhou, Aurick and Tucker, George and Levine, Sergey},
  journal={arXiv preprint arXiv:2006.04779},
  year={2020}
}

@article{kourogenis2010mixing,
	title={Mixing Conditions, Central Limit Theorems, and Invariance Principles: A Survey of the Literature with Some New Results on Heteroscedastic Sequences},
	author={Kourogenis, Nikolaos and Pittis, Nikitas},
	journal={Econometric Reviews},
	volume={30},
	number={1},
	pages={88--108},
	year={2010},
	publisher={Taylor \& Francis}
}

@article{komunjer2010semiparametric,
	title={Semiparametric efficiency bound in time-series models for conditional quantiles},
	author={Komunjer, Ivana and Vuong, Quang},
	journal={Econometric Theory},
	pages={383--405},
	year={2010},
	publisher={JSTOR}
}
 
@book{casella2002statistical,
	title={Statistical inference},
	author={Casella, George and Berger, Roger L},
	volume={2},
	year={2002},
	publisher={Duxbury Pacific Grove, CA}
}

@article{bhandari2018,
	title={A finite time analysis of temporal difference learning with linear function approximation},
	author={Bhandari, Jalaj and Russo, Daniel and Singal, Raghav},
	journal={arXiv preprint arXiv:1806.02450},
	year={2018}
}

@inproceedings{zou2019,
	title={Finite-sample analysis for SARSA with linear function approximation},
	author={Zou, Shaofeng and Xu, Tengyu and Liang, Yingbin},
	booktitle={Advances in Neural Information Processing Systems},
	pages={8665--8675},
	year={2019}
}

@inproceedings{degris2012off,
	title={Off-policy actor-critic},
	author={Degris, Thomas and White, Martha and Sutton, Richard S},
	booktitle={Proceedings of the 29th International Coference on International Conference on Machine Learning},
	pages={179--186},
	year={2012}
}

@incollection {Dedecker2002,
	AUTHOR = {Dedecker, J\'{e}r\^{o}me and Louhichi, Sana},
	TITLE = {Maximal inequalities and empirical central limit theorems},
	BOOKTITLE = {Empirical process techniques for dependent data},
	PAGES = {137--159},
	PUBLISHER = {Birkh\"{a}user Boston, Boston, MA},
	YEAR = {2002},
}

@article{bradley2005basic,
	title={Basic Properties of Strong Mixing Conditions. A Survey and Some Open Questions},
	author={Bradley, Richard C},
	journal={Probability Surveys},
	volume={2},
	pages={107--144},
	year={2005}
}

@article{chernozhukov2014gaussian,
	title={Gaussian approximation of suprema of empirical processes},
	author={Chernozhukov, Victor and Chetverikov, Denis and Kato, Kengo and others},
	journal={The Annals of Statistics},
	volume={42},
	number={4},
	pages={1564--1597},
	year={2014},
	publisher={Institute of Mathematical Statistics}
}

@article{chen2020statistical,
	title={Statistical Guarantees of Generative Adversarial Networks for Distribution Estimation},
	author={Chen, Minshuo and Liao, Wenjing and Zha, Hongyuan and Zhao, Tuo},
	journal={arXiv preprint arXiv:2002.03938},
	year={2020}
}

@article{wu2019behavior,
	title={Behavior regularized offline reinforcement learning},
	author={Wu, Yifan and Tucker, George and Nachum, Ofir},
	journal={arXiv preprint arXiv:1911.11361},
	year={2019}
}

@article{chernozhukov2017double,
	title={Double/debiased/neyman machine learning of treatment effects},
	author={Chernozhukov, Victor and Chetverikov, Denis and Demirer, Mert and Duflo, Esther and Hansen, Christian and Newey, Whitney},
	journal={American Economic Review},
	volume={107},
	number={5},
	pages={261--65},
	year={2017}
}

@article{shi2020breaking,
	title={Breaking the curse of nonregularity with subagging: inference of the mean outcome under optimal treatment regimes},
	author={Shi, Chengchun and Lu, Wenbin and Song, Rui},
	journal={Journal of Machine Learning Research},
	year={2020},
	volume={accepted}
}



@article{qian2011performance,
	title={Performance guarantees for individualized treatment rules},
	author={Qian, Min and Murphy, Susan A},
	journal={Annals of statistics},
	volume={39},
	number={2},
	pages={1180},
	year={2011},
	publisher={NIH Public Access}
}

@article{audibert2007fast,
	title={Fast learning rates for plug-in classifiers},
	author={Audibert, Jean-Yves and Tsybakov, Alexandre B and others},
	journal={The Annals of statistics},
	volume={35},
	number={2},
	pages={608--633},
	year={2007},
	publisher={Institute of Mathematical Statistics}
}

@article{tsybakov2004optimal,
	title={Optimal aggregation of classifiers in statistical learning},
	author={Tsybakov, Alexander B and others},
	journal={The Annals of Statistics},
	volume={32},
	number={1},
	pages={135--166},
	year={2004},
	publisher={Institute of Mathematical Statistics}
}

@article{liao2020batch,
	title={Batch Policy Learning in Average Reward Markov Decision Processes},
	author={Liao, Peng and Qi, Zhengling and Murphy, Susan},
	journal={arXiv preprint arXiv:2007.11771},
	year={2020}
}


@inproceedings{genevay2016stochastic,
	title={Stochastic optimization for large-scale optimal transport},
	author={Genevay, Aude and Cuturi, Marco and Peyr{\'e}, Gabriel and Bach, Francis},
	booktitle={Advances in neural information processing systems},
	pages={3440--3448},
	year={2016}
}

@inproceedings{Kingma2015,
	author    = {Diederik P. Kingma and
	Jimmy Ba},
	editor    = {Yoshua Bengio and Yann LeCun},
	title     = {Adam: {A} Method for Stochastic Optimization},
	booktitle = {3rd International Conference on Learning Representations, {ICLR} 2015,
	San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings},
	year      = {2015}
}

@inproceedings{cuturi2013sinkhorn,
	title={Sinkhorn distances: Lightspeed computation of optimal transport},
	author={Cuturi, Marco},
	booktitle={Advances in neural information processing systems},
	pages={2292--2300},
	year={2013}
}

@inproceedings{genevay2017learning,
	title={Learning generative models with sinkhorn divergences},
	author={Genevay, Aude and Peyr{\'e}, Gabriel and Cuturi, Marco},
	booktitle={International Conference on Artificial Intelligence and Statistics},
	pages={1608--1617},
	year={2018}
}

@inproceedings{goodfellow2014generative,
	title={Generative adversarial nets},
	author={Goodfellow, Ian and Pouget-Abadie, Jean and Mirza, Mehdi and Xu, Bing and Warde-Farley, David and Ozair, Sherjil and Courville, Aaron and Bengio, Yoshua},
	booktitle={Advances in neural information processing systems},
	pages={2672--2680},
	year={2014}
}

@inproceedings{kumar2019stabilizing,
	title={Stabilizing off-policy q-learning via bootstrapping error reduction},
	author={Kumar, Aviral and Fu, Justin and Soh, Matthew and Tucker, George and Levine, Sergey},
	booktitle={Advances in Neural Information Processing Systems},
	pages={11784--11794},
	year={2019}
}
@comment{BibDesk tempUrl Groups{ <?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <array>  <dict>   <key>tempUrl</key>   <string>file://localhost/Users/Peng/Desktop/tech%20report/reference.bib</string>   <key>group name</key>   <string>reference.bib</string>  </dict> </array> </plist> }}

@article{achiam2017constrained,
	title={Constrained policy optimization},
	author={Achiam, Joshua and Held, David and Tamar, Aviv and Abbeel, Pieter},
	journal={arXiv preprint arXiv:1705.10528},
	year={2017}
}

@inproceedings{schulman2015trust,
	title={Trust region policy optimization},
	author={Schulman, John and Levine, Sergey and Abbeel, Pieter and Jordan, Michael and Moritz, Philipp},
	booktitle={International conference on machine learning},
	pages={1889--1897},
	year={2015}
}

@article{shi2020does,
	title={Does the Markov Decision Process Fit the Data: Testing for the Markov Property in Sequential Decision Making},
	author={Shi, Chengchun and Wan, Runzhe and Song, Rui and Lu, Wenbin and Leng, Ling},
	journal={arXiv preprint arXiv:2002.01751},
	year={2020}
}

@article{laber2014dynamic,
	title={Dynamic treatment regimes: Technical challenges and applications},
	author={Laber, Eric B and Lizotte, Daniel J and Qian, Min and Pelham, William E and Murphy, Susan A},
	journal={Electronic journal of statistics},
	volume={8},
	number={1},
	pages={1225},
	year={2014},
	publisher={NIH Public Access}
}

@article{mitrophanov2005sensitivity,
	title={Sensitivity and convergence of uniformly ergodic Markov chains},
	author={Mitrophanov, A Yu},
	journal={Journal of Applied Probability},
	volume={42},
	number={4},
	pages={1003--1014},
	year={2005},
	publisher={Cambridge University Press}
}

@article{cho2001comparison,
	title={Comparison of perturbation bounds for the stationary distribution of a Markov chain},
	author={Cho, Grace E and Meyer, Carl D},
	journal={Linear Algebra and its Applications},
	volume={335},
	number={1-3},
	pages={137--150},
	year={2001},
	publisher={North-Holland}
}

@article{rudolf2018perturbation,
	title={Perturbation theory for Markov chains via Wasserstein distance},
	author={Rudolf, Daniel and Schweizer, Nikolaus and others},
	journal={Bernoulli},
	volume={24},
	number={4A},
	pages={2610--2639},
	year={2018},
	publisher={Bernoulli Society for Mathematical Statistics and Probability}
}



@article{richardson1995theory,
	title={The theory of the market economy},
	author={Richardson, George B},
	journal={Revue {\'e}conomique},
	pages={1487--1496},
	year={1995},
	publisher={JSTOR}
}



@book{migdalas2013multilevel,
	title={Multilevel optimization: algorithms and applications},
	author={Migdalas, Athanasios and Pardalos, Panos M and V{\"a}rbrand, Peter},
	volume={20},
	year={2013},
	publisher={Springer Science \& Business Media}
}


@article{nie2019learning,
	title={Learning when-to-treat policies},
	author={Nie, Xinkun and Brunskill, Emma and Wager, Stefan},
	journal={arXiv preprint arXiv:1905.09751},
	year={2019}
}

@book{bickel1993efficient,
	title={Efficient and adaptive estimation for semiparametric models},
	author={Bickel, Peter J and Klaassen, Chris AJ and Bickel, Peter J and Ritov, Ya’acov and Klaassen, J and Wellner, Jon A and Ritov, YA'Acov},
	volume={4},
	year={1993},
	publisher={Johns Hopkins University Press Baltimore}
}
@inproceedings{
	zhang2020gendice,
	title={Gen{\{}DICE{\}}: Generalized Offline Estimation of Stationary Values},
	author={Ruiyi Zhang and Bo Dai  and Lihong Li and Dale Schuurmans},
	booktitle={International Conference on Learning Representations},
	year={2020},
	url={https://openreview.net/forum?id=HkxlcnVFwB}
}

@inproceedings{nachum2019dualdice,
	title={Dualdice: Behavior-agnostic estimation of discounted stationary distribution corrections},
	author={Nachum, Ofir and Chow, Yinlam and Dai, Bo and Li, Lihong},
	booktitle={Advances in Neural Information Processing Systems},
	pages={2315--2325},
	year={2019}
}


@inproceedings{fukumizu2009kernel,
	title={Kernel choice and classifiability for RKHS embeddings of probability distributions},
	author={Fukumizu, Kenji and Gretton, Arthur and Lanckriet, Gert R and Sch{\"o}lkopf, Bernhard and Sriperumbudur, Bharath K},
	booktitle={Advances in neural information processing systems},
	pages={1750--1758},
	year={2009}
}
@article{loh2017statistical,
	title={Statistical consistency and asymptotic normality for high-dimensional robust $ M $-estimators},
	author={Loh, Po-Ling and others},
	journal={The Annals of Statistics},
	volume={45},
	number={2},
	pages={866--896},
	year={2017},
	publisher={Institute of Mathematical Statistics}
}
@article{mei2018landscape,
	title={The landscape of empirical risk for nonconvex losses},
	author={Mei, Song and Bai, Yu and Montanari, Andrea and others},
	journal={The Annals of Statistics},
	volume={46},
	number={6A},
	pages={2747--2774},
	year={2018},
	publisher={Institute of Mathematical Statistics}
}
@article{liu1989limited,
	title={On the limited memory BFGS method for large scale optimization},
	author={Liu, Dong C and Nocedal, Jorge},
	journal={Mathematical programming},
	volume={45},
	number={1-3},
	pages={503--528},
	year={1989},
	publisher={Springer}
}
@article{sinha2017review,
	title={A review on bilevel optimization: from classical to evolutionary approaches and applications},
	author={Sinha, Ankur and Malo, Pekka and Deb, Kalyanmoy},
	journal={IEEE Transactions on Evolutionary Computation},
	volume={22},
	number={2},
	pages={276--295},
	year={2017},
	publisher={IEEE}
}

@article{marcolino2018impact,
	title={The impact of mHealth interventions: systematic review of systematic reviews},
	author={Marcolino, Milena Soriano and Oliveira, Jo{\~a}o Antonio Queiroz and D'Agostino, Marcelo and Ribeiro, Antonio Luiz and Alkmim, Maria Beatriz Moreira and Novillo-Ortiz, David},
	journal={JMIR mHealth and uHealth},
	volume={6},
	number={1},
	pages={e23},
	year={2018},
	publisher={JMIR Publications Inc., Toronto, Canada}
}
@article{rinaldi2019cost,
	title={Cost and cost-effectiveness of mHealth interventions for the prevention and control of type 2 diabetes mellitus: a protocol for a systematic review},
	author={Rinaldi, Giulia and Hijazi, Alexa and Haghparast-Bidgoli, Hassan},
	journal={BMJ open},
	volume={9},
	number={4},
	pages={e027490},
	year={2019},
	publisher={British Medical Journal Publishing Group}
}



@book{van2000asymptotic,
	title={Asymptotic statistics},
	author={Van der Vaart, Aad W},
	volume={3},
	year={2000},
	publisher={Cambridge university press}
}

@inproceedings{yang2019sample,
	title={Sample-Optimal Parametric Q-Learning Using Linearly Additive Features},
	author={Yang, Lin and Wang, Mengdi},
	booktitle={International Conference on Machine Learning},
	pages={6995--7004},
	year={2019}
}

@article{uehara2019minimax,
	title={Minimax Weight and Q-Function Learning for Off-Policy Evaluation},
	author={Uehara, Masatoshi and Jiang, Nan},
	journal={arXiv preprint arXiv:1910.12809},
	year={2019}
}


@article{dudik2014doubly,
	title={Doubly robust policy evaluation and optimization},
	author={Dud{\'\i}k, Miroslav and Erhan, Dumitru and Langford, John and Li, Lihong and others},
	journal={Statistical Science},
	volume={29},
	number={4},
	pages={485--511},
	year={2014},
	publisher={Institute of Mathematical Statistics}
}

@inproceedings{yang2020theoretical,
title={A theoretical analysis of deep Q-learning},
author={Fan, Jianqing and Wang, Zhaoran and Xie, Yuchen and Yang, Zhuoran},
booktitle={Learning for Dynamics and Control},
pages={486--489},
year={2020},
organization={PMLR}
}

@article{farrell2021deep,
	title={Deep neural networks for estimation and inference},
	author={Farrell, Max H and Liang, Tengyuan and Misra, Sanjog},
	journal={Econometrica},
	volume={89},
	number={1},
	pages={181--213},
	year={2021},
	publisher={Wiley Online Library}
}

@article{devroye2018total,
	title={The total variation distance between high-dimensional Gaussians},
	author={Devroye, Luc and Mehrabian, Abbas and Reddad, Tommy},
	journal={arXiv preprint arXiv:1810.08693},
	year={2018}
}

@misc{chernozhukov2018double,
	title={Double/debiased machine learning for treatment and structural parameters},
	author={Chernozhukov, Victor and Chetverikov, Denis and Demirer, Mert and Duflo, Esther and Hansen, Christian and Newey, Whitney and Robins, James},
	year={2018},
	publisher={Oxford University Press Oxford, UK}
}

@article{robins1994estimation,
	title={Estimation of regression coefficients when some regressors are not always observed},
	author={Robins, James M and Rotnitzky, Andrea and Zhao, Lue Ping},
	journal={Journal of the American statistical Association},
	volume={89},
	number={427},
	pages={846--866},
	year={1994},
	publisher={Taylor \& Francis}
}

@article{mukherjee2017semiparametric,
	title={Semiparametric efficient empirical higher order influence function estimators},
	author={Mukherjee, Rajarshi and Newey, Whitney K and Robins, James M},
	journal={arXiv preprint arXiv:1705.07577},
	year={2017}
}

@inproceedings{harvey2017nearly,
	title={Nearly-tight VC-dimension bounds for piecewise linear neural networks},
	author={Harvey, Nick and Liaw, Christopher and Mehrabian, Abbas},
	booktitle={Conference on learning theory},
	pages={1064--1068},
	year={2017},
	organization={PMLR}
}

@incollection{robins2008higher,
	title={Higher order influence functions and minimax estimation of nonlinear functionals},
	author={Robins, James and Li, Lingling and Tchetgen, Eric and van der Vaart, Aad and others},
	booktitle={Probability and statistics: essays in honor of David A. Freedman},
	pages={335--421},
	year={2008},
	publisher={Institute of Mathematical Statistics}
}

@article{robins2017minimax,
	title={Minimax estimation of a functional on a structured high-dimensional model},
	author={Robins, James M and Li, Lingling and Mukherjee, Rajarshi and Tchetgen, Eric Tchetgen and van der Vaart, Aad and others},
	journal={The Annals of Statistics},
	volume={45},
	number={5},
	pages={1951--1987},
	year={2017},
	publisher={Institute of Mathematical Statistics}
}

@article{kallus2019efficiently,
	title={Efficiently breaking the curse of horizon: Double reinforcement learning in infinite-horizon processes},
	author={Kallus, Nathan and Uehara, Masatoshi},
	journal={arXiv preprint arXiv:1909.05850},
	year={2019}
}

@article{lagoudakis2003least,
	title={Least-squares policy iteration},
	author={Lagoudakis, Michail G and Parr, Ronald},
	journal={Journal of machine learning research},
	volume={4},
	number={Dec},
	pages={1107--1149},
	year={2003}
}

@article{munos2008finite,
	title={Finite-time bounds for fitted value iteration},
	author={Munos, R{\'e}mi and Szepesv{\'a}ri, Csaba},
	journal={Journal of Machine Learning Research},
	volume={9},
	number={May},
	pages={815--857},
	year={2008}
}


@article{murphy2005generalization,
	title={A generalization error for Q-learning},
	author={Murphy, Susan A},
	journal={Journal of Machine Learning Research},
	volume={6},
	number={Jul},
	pages={1073--1097},
	year={2005}
}


@article{kosorok2019precision,
	title={Precision medicine},
	author={Kosorok, Michael R and Laber, Eric B},
	journal={Annual review of statistics and its application},
	volume={6},
	pages={263--286},
	year={2019},
	publisher={Annual Reviews}
}

@inproceedings{kallus2018balanced,
	title={Balanced policy evaluation and learning},
	author={Kallus, Nathan},
	booktitle={Advances in Neural Information Processing Systems},
	pages={8895--8906},
	year={2018}
}

@book{sutton2018reinforcement,
	title={Reinforcement learning: An introduction},
	author={Sutton, Richard S and Barto, Andrew G},
	year={2018},
	publisher={MIT press}
}

@article{howard1960dynamic,
	title={Dynamic programming and markov processes.},
	author={Howard, Ronald A},
	year={1960},
	publisher={John Wiley}
}

@article{mahadevan1996average,
	title={Average reward reinforcement learning: Foundations, algorithms, and empirical results},
	author={Mahadevan, Sridhar},
	journal={Machine learning},
	volume={22},
	number={1-3},
	pages={159--195},
	year={1996},
	publisher={Springer}
}


@article{manski2004statistical,
	title={Statistical treatment rules for heterogeneous populations},
	author={Manski, Charles F},
	journal={Econometrica},
	volume={72},
	number={4},
	pages={1221--1246},
	year={2004},
	publisher={Wiley Online Library}
}

@article{liao2019off,
	title={Off-Policy Estimation of Long-Term Average Outcomes with Applications to Mobile Health},
	author={Liao, Peng and Klasnja, Predrag and Murphy, Susan},
	journal={arXiv preprint arXiv:1912.13088},
	year={2019}
}


@article{murphy2016batch,
	title={A batch, off-policy, actor-critic algorithm for optimizing the average reward},
	author={Murphy, Susan A and Deng, Yanzhen and Laber, Eric B and Maei, Hamid Reza and Sutton, Richard S and Witkiewitz, Katie},
	journal={arXiv preprint arXiv:1607.05047},
	year={2016}
}

@article{liu2019off,
	title={Off-Policy Policy Gradient with State Distribution Correction},
	author={Liu, Yao and Swaminathan, Adith and Agarwal, Alekh and Brunskill, Emma},
	journal={arXiv preprint arXiv:1904.08473},
	year={2019}
}

@article{ertefaie2018constructing,
	title={Constructing dynamic treatment regimes over indefinite time horizons},
	author={Ertefaie, Ashkan and Strawderman, Robert L},
	journal={Biometrika},
	volume={105},
	number={4},
	pages={963--977},
	year={2018},
	publisher={Oxford University Press}
}

@article{murphy2003optimal,
	title={Optimal dynamic treatment regimes},
	author={Murphy, Susan A},
	journal={Journal of the Royal Statistical Society: Series B (Statistical Methodology)},
	volume={65},
	number={2},
	pages={331--355},
	year={2003},
	publisher={Wiley Online Library}
}

@article{zhao2015new,
	title={New statistical learning methods for estimating optimal dynamic treatment regimes},
	author={Zhao, Ying-Qi and Zeng, Donglin and Laber, Eric B and Kosorok, Michael R},
	journal={Journal of the American Statistical Association},
	volume={110},
	number={510},
	pages={583--598},
	year={2015},
	publisher={Taylor \& Francis}
}

@article{luedtke2016super,
	title={Super-learning of an optimal dynamic treatment rule},
	author={Luedtke, Alexander R and van der Laan, Mark J},
	journal={The international journal of biostatistics},
	volume={12},
	number={1},
	pages={305--332},
	year={2016},
	publisher={De Gruyter}
}

@article{zhou2017residual,
	title={Residual weighted learning for estimating individualized treatment rules},
	author={Zhou, Xin and Mayer-Hamblett, Nicole and Khan, Umer and Kosorok, Michael R},
	journal={Journal of the American Statistical Association},
	volume={112},
	number={517},
	pages={169--187},
	year={2017},
	publisher={Taylor \& Francis}
}

@article{zhao2019efficient,
	title={Efficient augmentation and relaxation learning for individualized treatment rules using observational data.},
	author={Zhao, Ying-Qi and Laber, Eric B and Ning, Yang and Saha, Sumona and Sands, Bruce E},
	journal={Journal of Machine Learning Research},
	volume={20},
	number={48},
	pages={1--23},
	year={2019}
}

@inproceedings{le2019batch,
	title={Batch Policy Learning under Constraints},
	author={Le, Hoang and Voloshin, Cameron and Yue, Yisong},
	booktitle={International Conference on Machine Learning},
	pages={3703--3712},
	year={2019}
}

@article{athey2017efficient,
	title={Efficient policy learning},
	author={Athey, Susan and Wager, Stefan},
	journal={arXiv preprint arXiv:1702.02896},
	year={2017}
}

@phdthesis{van1998learning,
	title={Learning and value function approximation in complex decision processes},
	author={Van Roy, Benjamin},
	year={1998},
	school={Massachusetts Institute of Technology}
}

@article{liao2018just,
	title={Just-in-Time but Not Too Much: Determining Treatment Timing in Mobile Health},
	author={Liao, Peng and Dempsey, Walter and Sarker, Hillol and Hossain, Syed Monowar and al'Absi, Mustafa and Klasnja, Predrag and Murphy, Susan},
	journal={Proceedings of the ACM on interactive, mobile, wearable and ubiquitous technologies},
	volume={2},
	number={4},
	pages={179},
	year={2018},
	publisher={ACM}
}

@article{klasnja2018efficacy,
	title={Efficacy of contextually tailored suggestions for physical activity: a micro-randomized optimization trial of HeartSteps},
	author={Klasnja, Predrag and Smith, Shawna and Seewald, Nicholas J and Lee, Andy and Hall, Kelly and Luers, Brook and Hekler, Eric B and Murphy, Susan A},
	journal={Annals of Behavioral Medicine},
	year={2018}
}
@article{dempsey2017stratified,
	title={The stratified micro-randomized trial design: sample size considerations for testing nested causal effects of time-varying treatments},
	author={Dempsey, Walter and Liao, Peng and Kumar, Santosh and Murphy, Susan A},
	journal={arXiv preprint arXiv:1711.03587},
	year={2017}
}

@article{farahmand2011model,
	title={Model selection in reinforcement learning},
	author={Farahmand, Amir-massoud and Szepesv{\'a}ri, Csaba},
	journal={Machine learning},
	volume={85},
	number={3},
	pages={299--332},
	year={2011},
	publisher={Springer}
}

@article{ormoneit2002kernel,
	title={Kernel-based reinforcement learning in average-cost problems},
	author={Ormoneit, Dirk and Glynn, Peter},
	journal={IEEE Transactions on Automatic Control},
	volume={47},
	number={10},
	pages={1624--1636},
	year={2002},
	publisher={IEEE}
}

@article{meyn1995markov,
	title={Markov chains and Stochastic stability},
	author={Meyn, SP and Tweedie, RL and Hibey, JL},
	journal={IEEE Transactions on Automatic Control},
	volume={40},
	number={5},
	pages={979},
	year={1995},
	publisher={New York, NY: Institute of Electrical and Electronics Engineers, 1963-}
}

@book{puterman1994markov,
	title={Markov Decision Processes: Discrete Stochastic Dynamic Programming},
	author={Puterman, Martin L},
	year={1994},
	publisher={John Wiley \& Sons, Inc.}
}

@inproceedings{tagorti2015rate,
	title={On the Rate of Convergence and Error Bounds for LSTD ($\lambda$)},
	author={Tagorti, Manel and Scherrer, Bruno},
	booktitle={International Conference on Machine Learning},
	pages={1521--1529},
	year={2015}
}


@article{munos2007performance,
	title={Performance bounds in l\_p-norm for approximate value iteration},
	author={Munos, R{\'e}mi},
	journal={SIAM journal on control and optimization},
	volume={46},
	number={2},
	pages={541--561},
	year={2007},
	publisher={SIAM}
}

@book{gyorfi2006distribution,
	title={A distribution-free theory of nonparametric regression},
	author={Gy{\"o}rfi, L{\'a}szl{\'o} and Kohler, Michael and Krzyzak, Adam and Walk, Harro},
	year={2006},
	publisher={Springer Science \& Business Media}
}

@article{zhao2016partially,
	title={A partially linear framework for massive heterogeneous data},
	author={Zhao, Tianqi and Cheng, Guang and Liu, Han},
	journal={Annals of statistics},
	volume={44},
	number={4},
	pages={1400},
	year={2016},
	publisher={NIH Public Access}
}

@article{nedic2003least,
	title={Least squares policy evaluation algorithms with linear function approximation},
	author={Nedi{\'c}, A and Bertsekas, Dimitri P},
	journal={Discrete Event Dynamic Systems},
	volume={13},
	number={1-2},
	pages={79--110},
	year={2003},
	publisher={Springer}
}

@article{lazaric2012finite,
	title={Finite-sample analysis of least-squares policy iteration},
	author={Lazaric, Alessandro and Ghavamzadeh, Mohammad and Munos, R{\'e}mi},
	journal={Journal of Machine Learning Research},
	volume={13},
	number={Oct},
	pages={3041--3074},
	year={2012}
}

@article{farahmand2016regularized,
	title={Regularized policy iteration with nonparametric function spaces},
	author={Farahmand, Amir-massoud and Ghavamzadeh, Mohammad and Szepesv{\'a}ri, Csaba and Mannor, Shie},
	journal={The Journal of Machine Learning Research},
	volume={17},
	number={1},
	pages={4809--4874},
	year={2016},
	publisher={JMLR. org}
}

@inproceedings{liu2018breaking,
	title={Breaking the curse of horizon: Infinite-horizon off-policy estimation},
	author={Liu, Qiang and Li, Lihong and Tang, Ziyang and Zhou, Dengyong},
	booktitle={Advances in Neural Information Processing Systems},
	pages={5356--5366},
	year={2018}
}

@article{voloshin2019empirical,
	title={Empirical Study of Off-Policy Policy Evaluation for Reinforcement Learning},
	author={Voloshin, Cameron and Le, Hoang M and Jiang, Nan and Yue, Yisong},
	journal={arXiv preprint arXiv:1911.06854},
	year={2019}
}

@article{precup2000eligibility,
	title={Eligibility traces for off-policy policy evaluation},
	author={Precup, Doina},
	journal={Computer Science Department Faculty Publication Series},
	pages={80},
	year={2000}
}

@inproceedings{thomas2016data,
	title={Data-efficient off-policy policy evaluation for reinforcement learning},
	author={Thomas, Philip and Brunskill, Emma},
	booktitle={International Conference on Machine Learning},
	pages={2139--2148},
	year={2016}
}


@article{luckett2019estimating,
	title={Estimating dynamic treatment regimes in mobile health using V-learning},
	author={Luckett, Daniel J and Laber, Eric B and Kahkoska, Anna R and Maahs, David M and Mayer-Davis, Elizabeth and Kosorok, Michael R},
	journal={Journal of the American Statistical Association},
	number={just-accepted},
	pages={1--39},
	year={2019},
	publisher={Taylor \& Francis}
}

@book{van2000empirical,
	title={Empirical Processes in M-estimation},
	author={Van de Geer, Sara},
	volume={6},
	year={2000},
	publisher={Cambridge university press}
}

@book{steinwart2008support,
	title={Support vector machines},
	author={Steinwart, Ingo and Christmann, Andreas},
	year={2008},
	publisher={Springer Science \& Business Media}
}

@book{hernandez2012further,
	title={Further topics on discrete-time Markov control processes},
	author={Hern{\'a}ndez-Lerma, On{\'e}simo and Lasserre, Jean B},
	volume={42},
	year={1999},
	publisher={Springer}
}
@article{zhang2012robust,
	title={A robust method for estimating optimal treatment regimes},
	author={Zhang, Baqun and Tsiatis, Anastasios A and Laber, Eric B and Davidian, Marie},
	journal={Biometrics},
	volume={68},
	number={4},
	pages={1010--1018},
	year={2012},
	publisher={Wiley Online Library}
}

@book{tsiatis2007semiparametric,
	title={Semiparametric theory and missing data},
	author={Tsiatis, Anastasios},
	year={2007},
	publisher={Springer Science \& Business Media}
}

@article{zhang2013robust,
	title={Robust estimation of optimal dynamic treatment regimes for sequential treatment decisions},
	author={Zhang, Baqun and Tsiatis, Anastasios A and Laber, Eric B and Davidian, Marie},
	journal={Biometrika},
	volume={100},
	number={3},
	pages={681--694},
	year={2013},
	publisher={Oxford University Press}
}

@article{shi2020statistical,
  title={Statistical inference of the value function for reinforcement learning in infinite horizon settings},
  author={Shi, Chengchun and Zhang, Sheng and Lu, Wenbin and Song, Rui},
  journal={arXiv preprint arXiv:2001.04515},
  year={2020}
}

@inproceedings{ortner2012online,
	title={Online regret bounds for undiscounted continuous reinforcement learning},
	author={Ortner, Ronald and Ryabko, Daniil},
	booktitle={Advances in Neural Information Processing Systems},
	pages={1763--1771},
	year={2012}
}


@article{jiang2015doubly,
	title={Doubly robust off-policy value evaluation for reinforcement learning},
	author={Jiang, Nan and Li, Lihong},
	journal={arXiv preprint arXiv:1511.03722},
	year={2015}
}

@article{yom2017,
	title={Encouraging physical activity in patients with diabetes: intervention using a reinforcement learning system},
	author={Yom-Tov, Elad and Feraru, Guy and Kozdoba, Mark and Mannor, Shie and Tennenholtz, Moshe and Hochberg, Irit},
	journal={Journal of medical Internet research},
	volume={19},
	number={10},
	year={2017},
	publisher={JMIR Publications Inc.}
}

@inproceedings{paredes2014poptherapy,
	title={PopTherapy: coping with stress through pop-culture},
	author={Paredes, Pablo and Gilad-Bachrach, Ran and Czerwinski, Mary and Roseway, Asta and Rowan, Kael and Hernandez, Javier},
	booktitle={Proceedings of the 8th International Conference on Pervasive Computing Technologies for Healthcare},
	pages={109--117},
	year={2014},
	organization={ICST (Institute for Computer Sciences, Social-Informatics and Telecommunications Engineering)}
}

@article{forman2018can,
	title={Can the artificial intelligence technique of reinforcement learning use continuously-monitored digital data to optimize treatment for weight loss?},
	author={Forman, Evan M and Kerrigan, Stephanie G and Butryn, Meghan L and Juarascio, Adrienne S and Manasse, Stephanie M and Onta{\~n}{\'o}n, Santiago and Dallal, Diane H and Crochiere, Rebecca J and Moskow, Danielle},
	journal={Journal of behavioral medicine},
	pages={1--15},
	year={2018},
	publisher={Springer}
}


@inproceedings{rabbi2015mybehavior,
	title={MyBehavior: automatic personalized health feedback from user behaviors and preferences using smartphones},
	author={Rabbi, Mashfiqui and Aung, Min Hane and Zhang, Mi and Choudhury, Tanzeem},
	booktitle={Proceedings of the 2015 ACM International Joint Conference on Pervasive and Ubiquitous Computing},
	pages={707--718},
	year={2015},
	organization={ACM}
}

@inproceedings{zhou2018personalizing,
	title={Personalizing Mobile Fitness Apps using Reinforcement Learning},
	author={Zhou, Mo and Mintz, Yonatan and Fukuoka, Yoshimi and Goldberg, Ken and Flowers, Elena and Kaminsky, Philip and Castillejo, Alejandro and Aswani, Anil},
	booktitle={Companion Proceedings of the 23rd International on Intelligent User Interfaces: 2nd Workshop on Theory-Informed User Modeling for Tailoring and Personalizing Interfaces (HUMANIZE)},
	year={2018}
}

@article{mintz2017non,
	title={Non-Stationary Bandits with Habituation and Recovery Dynamics},
	author={Mintz, Yonatan and Aswani, Anil and Kaminsky, Philip and Flowers, Elena and Fukuoka, Yoshimi},
	journal={arXiv preprint arXiv:1707.08423},
	year={2017}
}

@inproceedings{ghosh2017misspecified,
	title={Misspecified Linear Bandits.},
	author={Ghosh, Avishek and Chowdhury, Sayak Ray and Gopalan, Aditya},
	booktitle={AAAI},
	pages={3761--3767},
	year={2017}
}

@article{dimakopoulou2017estimation,
	title={Estimation Considerations in Contextual Bandits},
	author={Dimakopoulou, Maria and Athey, Susan and Imbens, Guido},
	journal={arXiv preprint arXiv:1711.07077},
	year={2017}
}


@inproceedings{chapelle2011empirical,
	title={An empirical evaluation of thompson sampling},
	author={Chapelle, Olivier and Li, Lihong},
	booktitle={Advances in neural information processing systems},
	pages={2249--2257},
	year={2011}
}

@article{naik2019discounted,
	title={Discounted reinforcement learning is not an optimization problem},
	author={Naik, Abhishek and Shariff, Roshan and Yasui, Niko and Sutton, Richard S},
	journal={arXiv preprint arXiv:1910.02140},
	year={2019}
}

@inproceedings{kakade2001optimizing,
	title={Optimizing average reward using discounted rewards},
	author={Kakade, Sham},
	booktitle={International Conference on Computational Learning Theory},
	pages={605--615},
	year={2001},
	organization={Springer}
}

@article{tsitsiklis2002average,
	title={On average versus discounted reward temporal-difference learning},
	author={Tsitsiklis, John N and Van Roy, Benjamin},
	journal={Machine Learning},
	volume={49},
	number={2-3},
	pages={179--191},
	year={2002},
	publisher={Springer}
}

@article{osband2016posterior,
	title={Why is Posterior Sampling Better than Optimism for Reinforcement Learning?},
	author={Osband, Ian and Van Roy, Benjamin},
	journal={arXiv preprint arXiv:1607.00215},
	year={2016}
}

@article{osband2017optimistic,
	title={On optimistic versus randomized exploration in reinforcement learning},
	author={Osband, Ian and Van Roy, Benjamin},
	journal={arXiv preprint arXiv:1706.04241},
	year={2017}
}

@inproceedings{osband2013more,
	title={(More) efficient reinforcement learning via posterior sampling},
	author={Osband, Ian and Russo, Daniel and Van Roy, Benjamin},
	booktitle={Advances in Neural Information Processing Systems},
	pages={3003--3011},
	year={2013}
}
@inproceedings{fonteneau2013optimistic,
	title={An optimistic posterior sampling strategy for Bayesian reinforcement learning},
	author={Fonteneau, Rapha{\"e}l and Korda, Nathan and Munos, R{\'e}mi},
	booktitle={NIPS 2013 Workshop on Bayesian Optimization (BayesOpt2013)},
	year={2013}
}

@inproceedings{ouyang2017learning,
	title={Learning unknown Markov decision processes: A Thompson sampling approach},
	author={Ouyang, Yi and Gagrani, Mukul and Nayyar, Ashutosh and Jain, Rahul},
	booktitle={Advances in Neural Information Processing Systems},
	pages={1333--1342},
	year={2017}
}

@article{krishnamurthy2018semiparametric,
	title={Semiparametric Contextual Bandits},
	author={Krishnamurthy, Akshay and Wu, Zhiwei Steven and Syrgkanis, Vasilis},
	journal={arXiv preprint arXiv:1803.04204},
	year={2018}
}

@inproceedings{greenewald2017action,
	title={Action centered contextual bandits},
	author={Greenewald, Kristjan and Tewari, Ambuj and Murphy, Susan and Klasnja, Predag},
	booktitle={Advances in neural information processing systems},
	pages={5977--5985},
	year={2017}
}



@article{de200625,
	title={25 years of time series forecasting},
	author={De Gooijer, Jan G and Hyndman, Rob J},
	journal={International journal of forecasting},
	volume={22},
	number={3},
	pages={443--473},
	year={2006},
	publisher={Elsevier}
}


@article{Scott2017b,
	title={Using smartphones to decrease substance use via self-monitoring and recovery support: study protocol for a randomized control trial.},
	author={Scott, C.K. and Dennis, M.L. and Gustafson, D.H.},
	journal={Trials},
	volume = {18},
	number = {1},
	pages={374},
	year={2017},
}

@article{Scott2017,
	title={A pilot study of the feasibility and potential effectiveness of using smartphones to provide recovery support},
	author={Scott, C. K. and Dennis, M. L. and Gustafson, D. and Johnson, K.},
	journal={Drug \& Alcohol Dependence},
	volume={171},
	pages = {e185},
	year={2017},
}

@article{Rathbun2012,
	author = {Rathbun, S. and Song, X. and Neustfiter, B. and Shiffman, S.},
	TITLE = {Survival analysis with time-varying covariates measured at random times by design.},
	JOURNAL = {J R Stat Soc Ser C Appl. Stat.},
	VOLUME = {62},
	NUMBER = {3},
	PAGES = {419--434},
	YEAR = {2012},
}

@article{Dennis2015SA,
	title={A pilot study to examine the feasibility and potential effectiveness of using smartphones to provide recovery support for adolescents},
	author={Dennis, M.L. and Scott, C. K. and Funk, R. R. and Nicholson, L.},
	journal={Substance abuse},
	volume={36},
	number={4},
	pages={486--492},
	year={2015},
}


@article{Rathbun2012,
	AUTHOR = {Rathbun, S. and Song, X. and Neustfiter, B. and Shiffman, S.},
	TITLE = {Survival analysis with time-varying covariates measured at random times by design.},
	JOURNAL = {J R Stat Soc Ser C Appl. Stat.},
	VOLUME = {62},
	NUMBER = {3},
	PAGES = {419--434},
	YEAR = {2012},
}


@article{shiffman2008,
	title={Ecological momentary assessment},
	author={Shiffman, Saul and Stone, Arthur A and Hufford, Michael R},
	journal={Annu. Rev. Clin. Psychol.},
	volume={4},
	pages={1--32},
	year={2008},
	publisher={Annual Reviews}
}

@book{stone2007,
	title={The science of real-time data capture: Self-reports in health research},
	author={Stone, Arthur and Shiffman, Saul and Atienza, Audie and Nebeling, Linda},
	year={2007},
	publisher={Oxford University Press}
}

@article{Pielot:2017:BIP:3139486.3130956,
	author = {Pielot, Martin and Cardoso, Bruno and Katevas, Kleomenis and Serr\`{a}, Joan and Matic, Aleksandar and Oliver, Nuria},
	title = {Beyond Interruptibility: Predicting Opportune Moments to Engage Mobile Phone Users},
	journal = {Proc. ACM Interact. Mob. Wearable Ubiquitous Technol.},
	issue_date = {September 2017},
	volume = {1},
	number = {3},
	month = sep,
	year = {2017},
	issn = {2474-9567},
	pages = {91:1--91:25},
	articleno = {91},
	numpages = {25},
	url = {http://doi.acm.org.ezp-prod1.hul.harvard.edu/10.1145/3130956},
	doi = {10.1145/3130956},
	acmid = {3130956},
	publisher = {ACM},
	address = {New York, NY, USA},
	keywords = {Conversion, Mobile Devices, Proactive Recommendations, Push Notifications},
}

@article{Muller2017,
	author = {M{\:u}ller, A. M. and Blandford, A. and Yardley, L.},
	title = {The Conceptualization of a Just-In-Time Adaptive Intervention (JITAI) for the Reduction of Sedentary Behavior in Older Adults},
	journal = {mHealth},
	volume = {3},
	number = {37},
	year = {2017},
}

@inproceedings{
	Tang2020Doubly,
	title={Doubly Robust Bias Reduction in Infinite Horizon Off-Policy Estimation},
	author={Ziyang Tang and Yihao Feng and Lihong Li and Dengyong Zhou and Qiang Liu},
	booktitle={International Conference on Learning Representations},
	year={2020}
}

@article{kallus2019double,
	title={Double reinforcement learning for efficient off-policy evaluation in markov decision processes},
	author={Kallus, Nathan and Uehara, Masatoshi},
	journal={arXiv preprint arXiv:1908.08526},
	year={2019}
}

@article{dimitrijevic1972habituation,
	title={Habituation: effects of regular and stochastic stimulation},
	author={Dimitrijevi{\'c}, MR and Faganel, J and Gregori{\'c}, M and Nathan, PW and Trontelj, JK},
	journal={Journal of Neurology, Neurosurgery \& Psychiatry},
	volume={35},
	number={2},
	pages={234--242},
	year={1972},
	publisher={BMJ Publishing Group Ltd}
}

@article{nahum2016just,
	title={Just-in-Time Adaptive Interventions (JITAIs) in mobile health: key components and design principles for ongoing health behavior support},
	author={Nahum-Shani, Inbal and Smith, Shawna N and Spring, Bonnie J and Collins, Linda M and Witkiewitz, Katie and Tewari, Ambuj and Murphy, Susan A},
	journal={Annals of Behavioral Medicine},
	pages={1--17},
	year={2016},
	publisher={Springer}
}

@article{Wen2017,
	author = {Wen, C. and Schneider, S. and Stone, A. and Spruijt-Metz, D.},
	title = {Compliance With Mobile Ecological Momentary Assessment Protocols in Children and Adolescents: A Systematic Review and Meta-Analysis},
	journal = {J Med Internet Res},
	year = {2017},
	volume = {19},
	number = {4},
	pages = {e132},
}

@article{Liang1986,
	AUTHOR = {Liang, KY and Zeger, SL},
	TITLE = {Longitudinal data analysis using generalized linear models},
	JOURNAL = {Biometrika},
	YEAR = {1986},
	VOLUME = {73},
	number = {1},
	PAGES = {13--22}
}

@misc{microservice,
	title = {What are microservices?},
	howpublished = {\url{http://microservices.io/}},
	note = {Accessed: 2018-07-30}
}


@misc{Autosense,
	title = {Autosense: A wireless sensor system to quantify personal
	exposures to psychosocial stress and addictive substances in
	natural environments.},
	year = {2017}, 
	howpublished = {\url{http://sites.google.com/site/autosenseproject}},
}

@inproceedings{Autosense:Ertin:2011,
	author = {Ertin, E. and Stohs, N. and Kumar, S. and Raij, A. and al'Absi, M. and Shah, S.},
	title = {AutoSense: Unobtrusively Wearable Sensor Suite for Inferring the Onset, Causality, and Consequences of Stress in the Field},
	booktitle = {Proceedings of the 9th ACM Conference on Embedded Networked Sensor Systems},
	year = {2011},
	pages = {274--287},
	numpages = {14},
	address = {New York, NY, USA},
} 


@article{Sankaran63,
	AUTHOR = {Sankaran , M.},
	TITLE = {Approximations to the non-central chi-squared distribution},
	JOURNAL = {Biometrika},
	YEAR = {1963},
	VOLUME = {50},
	PAGES = {199--204}
}






@article {Zhang2016,
	author = {Zhang, Melvyn W B and Ward, John and Ying, John J B and Pan, Fang and Ho, Roger C M},
	title = {The alcohol tracker application: an initial evaluation of user preferences},
	volume = {2},
	number = {1},
	pages = {8--13},
	year = {2016},
	publisher = {BMJ Specialist Journals},
	abstract = {Background The prevalence of at-risk drinking and alcohol use disorders is increasing. Advances in technology have resulted in numerous smartphone applications for this disorder. However, there are still concerns about the evidence base of previously developed alcohol applications.Objective The following study aims to illustrate how the authors have made use of innovative methodologies to overcome the issues relating to the accuracy of tracking the amount of alcohol one has consumed; it also aims to determine user perceptions about the innovative tracker and various other features of an alcohol self-management application among a group of individuals from the general population of a developed country (Canada).Methodology A native alcohol self-management application was developed. In order to determine user perspectives towards this new innovative application, the authors took advantage and made use of crowdsourcing to acquire user perspectives.Results Our results showed that smartphone ownership is highest among the age group of 35{\textendash}44 years (91\%) and lowest for those aged between 55 and 64 (58\%). Our analysis also showed that 25{\textendash}34-year-olds and 35{\textendash}44-year-olds drink more frequently than the other groups. Results suggest that notification and information were the two most useful functions, with psychotherapy expected to be the least useful. Females indicated that notification service was the most useful function, while males preferred the information component.Conclusions This study has demonstrated how the authors have made use of innovative technologies to overcome the existing concerns pertaining to the utilisation of the blood alcohol concentration levels as a tracker. In addition, the authors have managed to highlight user preferences with regard to an alcohol application.},
	issn = {2055-8074},
	journal = {BMJ Innovations}
}


@article{Patrick2014,
	author = {Patrick L. Dulin and Vivian M. Gonzalez and Kendra Campbell},
	title = {Results of a Pilot Test of a Self-Administered Smartphone-Based Treatment System for Alcohol Use Disorders: Usability and Early Outcomes},
	journal = {Substance Abuse},
	volume = {35},
	number = {2},
	pages = {168-175},
	year  = {2014},
	publisher = {Taylor & Francis},
	
}

}




@ARTICLE{Muhammad2017, 
	author={G. Muhammad and M. Alsulaiman and S. U. Amin and A. Ghoneim and M. F. Alhamid}, 
	journal={IEEE Access}, 
	title={A Facial-Expression Monitoring System for Improved Healthcare in Smart Cities}, 
	year={2017}, 
	volume={5}, 
	number={}, 
	pages={10871-10881}, 
	keywords={Gaussian processes;emotion recognition;face recognition;feature extraction;feature selection;health care;image classification;mixture models;smart cities;statistical analysis;support vector machines;transforms;CS-LBP histograms;Gaussian mixture model;bandlet transform;classifier score;confidence score;facial expression monitoring system;facial expression recognition system;feature selection technique;feature vector;health care;human facial expression;smart cities;subbands extraction;support vector machine;weighted center-symmetric local binary pattern;Databases;Emotion recognition;Medical services;Smart cities;Speech;Speech recognition;Support vector machines;CS-LBP;GMM;SVM;Smart cities;bandlet transform;facial expression;health monitoring}, 
	ISSN={}, 
	month={},}

@article{Pearl2009,
	AUTHOR = {Pearl, J.}, 
	TITLE = {Causal inference in statistics: An overview},
	JOURNAL = {Statistics Surveys},
	VOLUME = {3},
	PAGES = {96--146}, 
	YEAR = {2009},
}

@book{Imbens2015,
	author = {Imbens, G.W. and Rubin, D.B.},
	title = {Causal Inference for Statistics, Social, and Biomedical Sciences: An Introduction},
	year = {2015},
	publisher = {Cambridge University Press},
	address = {New York, NY, USA},
} 

@article{Vanderweele2013,
	AUTHOR = {Vanderweele, T. J. and Hong, G. and Jones, S.M., and Brown, J.L.},
	TITLE = {Mediation and spillover effects in group-randomized
	trials: A case study of the 4rs educational intervention.},
	JOURNAL = {Journal of the American Statistical Association},
	NUMBER = {502},
	VOLUME = {108},
	PAGES = {469--482},
	YEAR = {2013},
}

@article{Hong2006,
	AUTHOR = {Hong, G. and Raudenbush, S. W.},
	TITLE = {Evaluating kindergarten retention policy},
	JOURNAL = {Journal of the American Statistical Association},
	NUMBER = {475},
	VOLUME = {101},
	PAGES = {901--910},
	YEAR = {2006},
}

@article{BushLipari2015,
	AUTHOR = {Bush, D. and Lipari, R.},
	TITLE = {Substance use and and substance use disorder by industry},
	JOURNAL = {National Survey on Drug Use and Health},
	YEAR = {2015},
}


@article{Freeetal2013,
	AUTHOR = {C. Free and G. Phillips and L. Galli and 
	L. Watson and L. Felix and P. Edwards and V. Patel and  A. Haines},
	journal = {PLOS Medicine},
	title = {The Effectiveness of Mobile-Health Technology-Based Health Behaviour Change or Disease Management Interventions for Health Care Consumers: A Systematic Review},
	year = {2013},
	volume = {10},
	pages = {1-45},
	number = {1},
}


@book{Ber01b,
	author       = {D. P. Bertsekas},
	title        = {Dynamic Programming and Optimal Control, Vol. II, 2nd
	Ed.},
	publisher    = {Athena Scientific},
	address      = {Belmont, MA},
	year         = {2001}
}


@article{degris_off-policy_2012,
	Abstract = {This paper presents the first actor-critic algorithm for off-policy reinforcement learning. Our algorithm is online and incremental, and its per-time-step complexity scales linearly with the number of learned weights. Previous work on actor-critic algorithms is limited to the on-policy setting and does not take advantage of the recent advances in off-policy gradient temporal-difference learning. Off-policy techniques, such as Greedy-{GQ}, enable a target policy to be learned while following and obtaining data from another (behavior) policy. For many problems, however, actor-critic methods are more practical than action value methods (like Greedy-{GQ}) because they explicitly represent the policy; consequently, the policy can be stochastic and utilize a large action space. In this paper, we illustrate how to practically combine the generality and learning potential of off-policy learning with the flexibility in action selection given by actor-critic methods. We derive an incremental, linear time and space complexity algorithm that includes eligibility traces, prove convergence under assumptions similar to previous off-policy algorithms, and empirically show better or comparable performance to existing algorithms on standard reinforcement-learning benchmark problems.},
	Author = {Degris, Thomas and White, Martha and Sutton, Richard S.},
	Year = {2012},
	Eprint = {1205.4839},
	Eprinttype = {arxiv},
	File = {arXiv\:1205.4839 PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/G5PKC77C/Degris et al. - 2012 - Off-Policy Actor-Critic.pdf:application/pdf;arXiv.org Snapshot:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/JQHW7467/1205.html:text/html},
	Journaltitle = {{arXiv}:1205.4839 [cs]},
	Keywords = {Computer Science - Learning},
	Title = {Off-Policy Actor-Critic},
	Url = {http://arxiv.org/abs/1205.4839},
	Urldate = {2015-11-10},
	Bdsk-Url-1 = {http://arxiv.org/abs/1205.4839}}

@incollection{geist_revisiting_2010,
	Abstract = {Actor-critics architectures have become popular during the last decade in the field of reinforcement learning because of the introduction of the policy gradient with function approximation theorem. It allows combining rationally actor-critic architectures with value function approximation and therefore addressing large-scale problems. Recent researches led to the replacement of policy gradient by a natural policy gradient, improving the efficiency of the corresponding algorithms. However, a common drawback of these approaches is that they require the manipulation of the so-called advantage function which does not satisfy any Bellman equation. Consequently, derivation of actor-critic algorithms is not straightforward. In this paper, we re-derive theorems in a way that allows reasoning directly with the state-action value function (or Q-function) and thus relying on the Bellman equation again. Consequently, new forms of critics can easily be integrated in the actor-critic framework.},
	Author = {Geist, Matthieu and Pietquin, Olivier},
	Booktitle = {Modeling Decisions for Artificial Intelligence},
	Year = {2010},
	Editor = {Torra, Vicen{\c c} and Narukawa, Yasuo and Daumas, Marc},
	File = {Full Text PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/FBVGEEG9/Geist and Pietquin - 2010 - Revisiting Natural Actor-Critics with Value Functi.pdf:application/pdf;Snapshot:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/FUW2F44E/10.html:text/html},
	Isbn = {978-3-642-16291-6 978-3-642-16292-3},
	Keywords = {Artificial Intelligence (incl. Robotics), Computation by Abstract Devices, Database Management, Data Mining and Knowledge Discovery, Information Storage and Retrieval, Information Systems Applications (incl.Internet)},
	Langid = {english},
	Note = {{DOI}: 10.1007/978-3-642-16292-3\_21},
	Number = {6408},
	Pages = {207--218},
	Publisher = {Springer Berlin Heidelberg},
	Rights = {{\copyright}2010 Springer Berlin Heidelberg},
	Series = {Lecture Notes in Computer Science},
	Title = {Revisiting Natural Actor-Critics with Value Function Approximation},
	Url = {http://link.springer.com/chapter/10.1007/978-3-642-16292-3_21},
	Urldate = {2015-10-28},
	Bdsk-Url-1 = {http://link.springer.com/chapter/10.1007/978-3-642-16292-3_21}}

@article{bhatnagar_natural_2009,
	Author = {Bhatnagar, Shalabh and Sutton, Richard S. and Ghavamzadeh, Mohammad and Lee, Mark},
	Year = {2009},
	Date-Modified = {2015-11-29 23:48:58 +0000},
	Doi = {10.1016/j.automatica.2009.07.008},
	File = {1-s2.0-S0005109809003549-main.pdf:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/RIRVE5KF/1-s2.0-S0005109809003549-main.pdf:application/pdf},
	Issn = {00051098},
	Journal = {Automatica},
	Langid = {english},
	Number = {11},
	Pages = {2471--2482},
	Title = {Natural actor--critic algorithms},
	Url = {http://linkinghub.elsevier.com/retrieve/pii/S0005109809003549},
	Urldate = {2015-10-20},
	Volume = {45},
	Year = {2009},
	Bdsk-Url-1 = {http://linkinghub.elsevier.com/retrieve/pii/S0005109809003549},
	Bdsk-Url-2 = {http://dx.doi.org/10.1016/j.automatica.2009.07.008}}

@article{grondman_survey_2012,
	Abstract = {Policy-gradient-based actor-critic algorithms are amongst the most popular algorithms in the reinforcement learning framework. Their advantage of being able to search for optimal policies using low-variance gradient estimates has made them useful in several real-life applications, such as robotics, power control, and finance. Although general surveys on reinforcement learning techniques already exist, no survey is specifically dedicated to actor-critic algorithms in particular. This paper, therefore, describes the state of the art of actor-critic algorithms, with a focus on methods that can work in an online setting and use function approximation in order to deal with continuous state and action spaces. After starting with a discussion on the concepts of reinforcement learning and the origins of actor-critic algorithms, this paper describes the workings of the natural gradient, which has made its way into many actor-critic algorithms over the past few years. A review of several standard and natural actor-critic algorithms is given, and the paper concludes with an overview of application areas and a discussion on open issues.},
	Author = {Grondman, I. and Busoniu, L. and Lopes, G.A.D. and Babuska, R.},
	Year = {2012},
	Doi = {10.1109/TSMCC.2012.2218595},
	File = {IEEE Xplore Abstract Record:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/5QCW97U5/articleDetails.html:text/html;IEEE Xplore Full Text PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/F7Z46ARH/Grondman et al. - 2012 - A Survey of Actor-Critic Reinforcement Learning S.pdf:application/pdf},
	Issn = {1094-6977},
	Journal = {{IEEE} Transactions on Systems, Man, and Cybernetics, Part C: Applications and Reviews},
	Keywords = {Actor-critic, actor-critic reinforcement learning, Approximation algorithms, Approximation methods, convergence, Equations, Function approximation, gradient methods, learning (artificial intelligence), low-variance gradient estimation, natural gradient, natural policy gradients, optimal policies, Optimization, policy gradient, policy-gradient-based actor-critic algorithms, real-life applications, Reinforcement learning ({RL}), {RL}, standard policy gradients, Standards},
	Number = {6},
	Pages = {1291--1307},
	Shorttitle = {A Survey of Actor-Critic Reinforcement Learning},
	Title = {A Survey of Actor-Critic Reinforcement Learning: Standard and Natural Policy Gradients},
	Volume = {42},
	Bdsk-Url-1 = {http://dx.doi.org/10.1109/TSMCC.2012.2218595}}

@inproceedings{peters_reinforcement_2003,
	Abstract = {Reinforcement learning offers one of the most general frame- work to take traditional robotics towards true autonomy and versatility. However, applying reinforcement learning to high dimensional movement systems like humanoid robots remains an unsolved problem. In this pa- per, we discuss different approaches of reinforcement learning in terms of their applicability in humanoid robotics. Methods can be coarsely clas- sified into three different categories, i.e., greedy methods, {\textbackslash}textasciigravevanilla` policy gradient methods, and natural gradient methods. We discuss that greedy methods are not likely to scale into the domain humanoid robotics as they are problematic when used with function approximation. {\textbackslash}{textasciigraveVanilla}` policy gradient methods on the other hand have been successfully ap- plied on real-world robots including at least one humanoid robot [3]. We demonstrate that these methods can be significantly improved us- ing the natural policy gradient instead of the regular policy gradient. A derivation of the natural policy gradient is provided, proving that the av- erage policy gradient of Kakade [10] is indeed the true natural gradient. A general algorithm for estimating the natural gradient, the Natural Actor-Critic algorithm, is introduced. This algorithm converges to the nearest local minimum of the cost function with respect to the Fisher in- formation metric under suitable conditions. The algorithm outperforms non-natural policy gradients by far in a cart-pole balancing evaluation, and for learning nonlinear dynamic motor primitives for humanoid robot control. It offers a promising route for the development of reinforcement learning for truly high-dimensionally continuous state-action systems.},
	Author = {Peters, J and Vijayakumar, S and Schaal, S},
	Booktitle = {3rd {IEEE}-{RAS} International Conference on Humanoid Robots (Humanoids2003)},
	Year = {2003},
	File = {peters-ICHR2003.pdf:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/HX7WCEGM/peters-ICHR2003.pdf:application/pdf},
	Keywords = {{NAC}-{LSTD}},
	Location = {Karlsruhe, Germany},
	Pages = {1--20},
	Title = {Reinforcement Learning for Humanoid Robotics}}

@article{kakade_natural_2001,
	Abstract = {We provide a natural gradient method that represents the steepest descent direction based on the underlying structure of the parameter space. Although gradient methods cannot make large changes in the values of the parameters, we show that the natural gradient is moving toward choosing a greedy optimal action rather than just a better action. These greedy optimal actions are those that would be chosen under one improvement step of policy iteration with approximate, compatible value functions, as deo/ned by Sutton et al. [9]. We then show drastic performance improvements in simple {MDPs} and in the more challenging {MDP} of Tetris.},
	Author = {Kakade, Sham},
	Date = {2002},
	Date-Modified = {2015-11-29 23:58:21 +0000},
	File = {2073-a-natural-policy-gradient.pdf:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/9I2EN238/2073-a-natural-policy-gradient.pdf:application/pdf;Citeseer - Snapshot:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/DE4X27CS/summary.html:text/html},
	Journal = {Advances in Neural Information Processing Systems},
	Title = {A Natural Policy Gradient},
	Volume = {14},
	Year = {2002}}

@inproceedings{sutton_policy_2000,
	Abstract = {Function approximation is essential to reinforcement learning, but the standard approach of approximating a value function and determining a policy from it has so far proven theoretically intractable. In this paper we explore an alternative approach in which the policy is explicitly represented by its own function approximator, independent of the value function, and is updated according to the gradient of expected reward with respect to the policy parameters. Williams's {REINFORCE} method and actor--critic methods are examples of this approach. Our main new result is to show that the gradient can be written in a form suitable for estimation from experience aided by an approximate action-value or advantage function. Using this result, we prove for the first time that a version of policy iteration with arbitrary differentiable function approximation is convergent to a locally optimal policy.},
	Author = {Sutton, Richard S. and Mcallester, David and Singh, Satinder and Mansour, Yishay},
	Booktitle = {In Advances in Neural Information Processing Systems 12},
	Year = {2000},
	File = {Citeseer - Full Text PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/JZCSP8EU/Sutton et al. - 2000 - Policy gradient methods for reinforcement learning.pdf:application/pdf;Citeseer - Snapshot:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/RVSGCMS7/summary.html:text/html},
	Pages = {1057--1063},
	Publisher = {{MIT} Press},
	Title = {Policy gradient methods for reinforcement learning with function approximation}}

@paper{thomas2015,
	Abstract = {Many reinforcement learning algorithms use trajectories collected from the execution of one or more policies to propose a new policy. Because execution of a bad policy can be costly or dangerous, techniques for evaluating the performance of the new policy without requiring its execution have been of recent interest in industry. Such off-policy evaluation methods, which estimate the performance of a policy using trajectories collected from the execution of other policies, heretofore have not provided confidences regarding the accuracy of their estimates. In this paper we propose an off-policy method for computing a lower confidence bound on the expected return of a policy.},
	Author = {Philip Thomas and Georgios Theocharous and Mohammad Ghavamzadeh},
	Conference = {AAAI Conference on Artificial Intelligence},
	Keywords = {policy evaluation; high-confidence; concentration inequality},
	Title = {High-Confidence Off-Policy Evaluation},
	Url = {http://www.aaai.org/ocs/index.php/AAAI/AAAI15/paper/view/10042},
	Year = {2015},
	Bdsk-Url-1 = {http://www.aaai.org/ocs/index.php/AAAI/AAAI15/paper/view/10042}}

@inproceedings{pirotta13,
	Abstract = {This paper presents a study of the policy improvement step that can be usefully exploited by approximate policy-iteration algorithms. When either the policy evaluation step or the policy improvement step returns an approximated result, the sequence of policies produced by policy iteration may not be monotonically increasing, and oscillations may occur. To address this issue, we consider safe policy improvements, i.e., at each iteration we search for a policy that maximizes a lower bound to the policy improvement w.r.t. the current policy. When no improving policy can be found the algorithm stops. We propose two safe policy-iteration algorithms that differ in the way the next policy is chosen w.r.t. the estimated greedy policy. Besides being theoretically derived and discussed, the proposed algorithms are empirically evaluated and compared with state-of-the-art approaches on some chain-walk domains and on the Blackjack card game.},
	Author = {Matteo Pirotta and Marcello Restelli and Alessio Pecorino and Daniele Calandriello},
	Booktitle = {Proceedings of the 30th International Conference on Machine Learning (ICML-13)},
	Editor = {Sanjoy Dasgupta and David Mcallester},
	Month = may,
	Number = {3},
	Pages = {307-315},
	Publisher = {JMLR Workshop and Conference Proceedings},
	Title = {Safe Policy Iteration},
	Url = {http://jmlr.org/proceedings/papers/v28/pirotta13.pdf},
	Volume = {28},
	Year = {2013},
	Bdsk-Url-1 = {http://jmlr.org/proceedings/papers/v28/pirotta13.pdf}}

@inproceedings{kakade2002approximately,
	title={Approximately optimal approximate reinforcement learning},
	author={Kakade, Sham and Langford, John},
	booktitle={ICML},
	volume={2},
	pages={267--274},
	year={2002}
}

@inproceedings{antos2008fitted,
	title={Fitted Q-iteration in continuous action-space MDPs},
	author={Antos, Andr{\'a}s and Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
	booktitle={Advances in neural information processing systems},
	pages={9--16},
	year={2008}
}

@article{antos_learning_2008,
	Author = {Antos, Andr{\'a}s and Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
	Year = {2008},
	Doi = {10.1007/s10994-007-5038-2},
	File = {art%3A10.1007%2Fs10994-007-5038-2.pdf:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/7V4NA3C6/art%3A10.1007%2Fs10994-007-5038-2.pdf:application/pdf},
	Issn = {0885-6125, 1573-0565},
	Journal = {Machine Learning},
	Langid = {english},
	Number = {1},
	Pages = {89--129},
	Title = {Learning near-optimal policies with Bellman-residual minimization based fitted policy iteration and a single sample path},
	Url = {http://link.springer.com/10.1007/s10994-007-5038-2},
	Urldate = {2015-10-17},
	Volume = {71},
	Bdsk-Url-1 = {http://link.springer.com/10.1007/s10994-007-5038-2},
	Bdsk-Url-2 = {http://dx.doi.org/10.1007/s10994-007-5038-2}}

@incollection{farahmand_regularized_20092,
	Author = {Farahmand, Amir M. and Ghavamzadeh, Mohammad and Mannor, Shie and Szepesv{\'a}ri, Csaba},
	Booktitle = {Advances in Neural Information Processing Systems 21},
	Year = {2009},
	Editor = {Koller, D. and Schuurmans, D. and Bengio, Y. and Bottou, L.},
	File = {NIPS Full Text PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/ID4JEM35/Farahmand et al. - 2009 - Regularized Policy Iteration.pdf:application/pdf;NIPS Snapshort:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/I5CAH3GF/3445-regularized-policy-iteration.html:text/html},
	Keywords = {Error Bound},
	Pages = {441--448},
	Publisher = {Curran Associates, Inc.},
	Title = {Regularized Policy Iteration},
	Url = {http://papers.nips.cc/paper/3445-regularized-policy-iteration.pdf},
	Urldate = {2015-11-06},
	Bdsk-Url-1 = {http://papers.nips.cc/paper/3445-regularized-policy-iteration.pdf}}

@article{lagoudakis_least-squares_2003,
	Author = {Lagoudakis, Michail G. and Parr, Ronald},
	Year = {2003},
	File = {lagoudakis03a.pdf:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/MX6WU7UZ/lagoudakis03a.pdf:application/pdf},
	Journal = { The Journal of Machine Learning Research},
	Keywords = {General Batch, {LSPI}},
	Pages = {1107--1149},
	Title = {Least-Squares Policy Iteration},
	Volume = {4}}

@inproceedings{farahmand_regularized_2009,
	Abstract = {Reinforcement learning with linear and non-linear function approximation has been studied extensively in the last decade. However, as opposed to other fields of machine learning such as supervised learning, the effect of finite sample has not been thoroughly addressed within the reinforcement learning framework. In this paper we propose to use L2 regularization to control the complexity of the value function in reinforcement learning and planning problems. We consider the Regularized Fitted Q-Iteration algorithm and provide generalization bounds that account for small sample sizes. Finally, a realistic visual-servoing problem is used to illustrate the benefits of using the regularization procedure.},
	Author = {Farahmand, A.m. and Ghavamzadeh, M. and Szepesvari, C. and Mannor, S.},
	Booktitle = {American Control Conference, 2009. {ACC} '09.},
	Date = {2009-06},
	Doi = {10.1109/ACC.2009.5160611},
	Eventtitle = {American Control Conference, 2009. {ACC} '09.},
	File = {IEEE Xplore Abstract Record:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/IS2CVF4U/abs_all.html:text/html;IEEE Xplore Full Text PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/F54X4JHN/Farahmand et al. - 2009 - Regularized Fitted Q-Iteration for planning in con.pdf:application/pdf},
	Keywords = {Automatic control, Complex networks, Computational modeling, Computer networks, continuous-space Markovian decision problem, Discrete event simulation, Error correction, Function approximation, iterative methods, learning (artificial intelligence), machine learning, Machine learning algorithms, Markov processes, nonlinear function approximation, nonlinear functions, planning (artificial intelligence), regularized fitted Q-iteration, reinforcement learning, Supervised learning, visual-servoing problem},
	Pages = {725--730},
	Title = {Regularized Fitted Q-Iteration for planning in continuous-space Markovian decision problems},
	Bdsk-Url-1 = {http://dx.doi.org/10.1109/ACC.2009.5160611}}

@incollection{neumann_fitted_2009,
	Author = {Neumann, Gerhard and Peters, Jan R.},
	Booktitle = {Advances in Neural Information Processing Systems 21},
	Year = {2009},
	Editor = {Koller, D. and Schuurmans, D. and Bengio, Y. and Bottou, L.},
	File = {NIPS Full Text PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/6XZTJ33H/Neumann and Peters - 2009 - Fitted Q-iteration by Advantage Weighted Regressio.pdf:application/pdf;NIPS Snapshort:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/QUVP2UCT/3501-fitted-q-iteration-by-advantage-weighted-regression.html:text/html},
	Pages = {1177--1184},
	Publisher = {Curran Associates, Inc.},
	Title = {Fitted Q-iteration by Advantage Weighted Regression},
	Url = {http://papers.nips.cc/paper/3501-fitted-q-iteration-by-advantage-weighted-regression.pdf},
	Urldate = {2015-11-29},
	Bdsk-Url-1 = {http://papers.nips.cc/paper/3501-fitted-q-iteration-by-advantage-weighted-regression.pdf}}

@inproceedings{peters_reinforcement_2007,
	Abstract = {Many robot control problems of practical importance, including operational space control, can be reformulated as immediate reward reinforcement learning problems. However, few of the known optimization or reinforcement learning algorithms can be used in online learning control for robots, as they are either prohibitively slow, do not scale to interesting domains of complex robots, or require trying out policies generated by random search, which are infeasible for a physical system. Using a generalization of the {EM}-base reinforcement learning framework suggested by Dayan \& Hinton, we reduce the problem of learning with immediate rewards to a reward-weighted regression problem with an adaptive, integrated reward transformation for faster convergence. The resulting algorithm is efficient, learns smoothly without dangerous jumps in solution space, and works well in applications of complex high degreeof-freedom robots. 1.},
	Author = {Peters, Jan and Schaal, Stefan},
	Booktitle = {In: Proceedings of the International Conference on Machine Learning ({ICML})},
	Year = {2007},
	File = {Citeseer - Full Text PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/22NPSP9Q/Peters and Schaal - 2007 - Reinforcement learning by reward-weighted regressi.pdf:application/pdf},
	Pages = {745--750},
	Title = {Reinforcement learning by reward-weighted regression for operational space control}}

@inproceedings{riedmiller_neural_2005,
	Abstract = {Abstract. This paper introduces {NFQ}, an algorithm for efficient and effective training of a Q-value function represented by a multi-layer perceptron. Based on the principle of storing and reusing transition experiences, a model-free, neural network based Reinforcement Learning algorithm is proposed. The method is evaluated on three benchmark problems. It is shown empirically, that reasonably few interactions with the plant are needed to generate control policies of high quality. 1},
	Author = {Riedmiller, Martin},
	Booktitle = {In 16th European Conference on Machine Learning},
	Year = {2005},
	File = {Citeseer - Full Text PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/NJ9KH9DK/Riedmiller - 2005 - Neural fitted Q iteration -- first experiences with.pdf:application/pdf;Citeseer - Snapshot:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/9E5284KU/summary.html:text/html},
	Keywords = {{FItted} Q Iteration},
	Pages = {317--328},
	Publisher = {Springer},
	Title = {Neural fitted Q iteration -- first experiences with a data efficient neural reinforcement learning method}}

@article{ernst_tree-based_2005,
	Abstract = {Reinforcement learning aims to determine an optimal control policy from interaction with a system or from observations gathered from a system. In batch mode, it can be achieved by approximating the so-called Q-function based on a set of four-tuples (xt,ut,rt,xt+1) where xt denotes the system state at time t, ut the control action taken, rt the instantaneous reward obtained and xt+1 the successor state of the system, and by determining the control policy from this Q-function. The Q-function approximation may be obtained from the limit of a sequence of (batch mode) supervised learning problems. Within this framework we describe the use of several classical tree-based supervised learning methods ({CART}, Kd-tree, tree bagging) and two newly proposed ensemble algorithms, namely extremely and totally randomized trees. We study their performances on several examples and find that the ensemble methods based on regression trees perform well in extracting relevant information about the optimal control policy from sets of four-tuples. In particular, the totally randomized trees give good results while ensuring the convergence of the sequence, whereas by relaxing the convergence constraint even better accuracy results are provided by the extremely randomized trees.},
	Author = {Ernst, Damien and Geurts, Pierre and Wehenkel, Louis and Littman, L.},
	Year = {2005},
	File = {Citeseer - Full Text PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/JZX28TF9/Ernst et al. - 2005 - Tree-based batch mode reinforcement learning.pdf:application/pdf;Citeseer - Snapshot:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/9KVJS2KK/summary.html:text/html},
	Journal = {Journal of Machine Learning Research},
	Pages = {503--556},
	Title = {Tree-based batch mode reinforcement learning},
	Volume = {6}}

@inproceedings{ormoneit_kernel-based_2003,
	Abstract = {We present a kernel-based approach to reinforcement learning that overcomes the stability problems of temporal-difference learning in continuous state-spaces. First, our algorithm converges to a unique solution of an approximate Bellman's equation regardless of its initialization values. Second, the method is consistent in the sense that the resulting policy converges asymptotically to the optimal policy. Parametric value function estimates such as neural networks do not possess this property. Our kernel-based approach also allows us to show that the limiting distribution of the value function estimate is a Gaussian process. This information is useful in studying the bias-variance tradeo in reinforcement learning. We find that all reinforcement learning approaches to estimating the value function, parametric or non-parametric, are subject to a bias. This bias is typically larger in reinforcement learning than in a comparable regression problem.},
	Author = {Ormoneit, Dirk and Sen, Saunak},
	Booktitle = {Machine Learning},
	Year = {2003},
	File = {Citeseer - Snapshot:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/WB3QHCZU/summary.html:text/html;Omoneit Sen-Kernel-based reinforcement learning.pdf:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/ID3NQIEC/Omoneit Sen-Kernel-based reinforcement learning.pdf:application/pdf},
	Pages = {161--178},
	Title = {Kernel-Based Reinforcement Learning}}

@article{geist_algorithmic_2013,
	Abstract = {Reinforcement learning ({RL}) is a machine learning answer to the optimal control problem. It consists of learning an optimal control policy through interactions with the system to be controlled, the quality of this policy being quantified by the so-called value function. A recurrent subtopic of {RL} concerns computing an approximation of this value function when the system is too large for an exact representation. This survey reviews state-of-the-art methods for (parametric) value function approximation by grouping them into three main categories: bootstrapping, residual, and projected fixed-point approaches. Related algorithms are derived by considering one of the associated cost functions and a specific minimization method, generally a stochastic gradient descent or a recursive least-squares approach.},
	Author = {Geist, M. and Pietquin, O.},
	Year = {2013},
	Doi = {10.1109/TNNLS.2013.2247418},
	File = {IEEE Xplore Abstract Record:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/C3Q8JUFJ/abs_all.html:text/html;RL_VF.pdf:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/MMXHQPSG/RL_VF.pdf:application/pdf},
	Issn = {2162-237X},
	Journal = {{IEEE} Transactions on Neural Networks and Learning Systems},
	Keywords = {Approximation algorithms, bootstrapping approach, Cost function, cost function minimization method, Equations, Function approximation, gradient methods, learning (artificial intelligence), least squares approximations, machine learning, Mathematical model, optimal control, optimal control policy learning, parametric value function approximation, projected fixed-point approach, recursive least-squares approach, reinforcement learning, Reinforcement learning ({RL}), residual approach, {RL} recurrent subtopic, stochastic gradient descent, stochastic processes, Supervised learning, survey, value function approximation},
	Number = {6},
	Pages = {845--867},
	Title = {Algorithmic Survey of Parametric Value Function Approximation},
	Volume = {24},
	Bdsk-Url-1 = {http://dx.doi.org/10.1109/TNNLS.2013.2247418}}

@article{ueno_generalized_2011,
	Author = {Ueno, Tsuyoshi and Maeda, Shin-ichi and Kawanabe, Motoaki and Ishii, Shin},
	Date = {2011},
	Date-Modified = {2015-11-30 00:00:04 +0000},
	File = {bottou-lecun-2004a.pdf:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/C2VEZJSV/bottou-lecun-2004a.pdf:application/pdf;Konda 2002.pdf:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/FPWFARHH/51552606-MIT.pdf:application/pdf;[Malay_Ghosh,_Nitis_Mukhopadhyay,_Pranab_Kumar_Sen(BookZZ.org).pdf:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/W3VGKZTU/[Malay_Ghosh,_Nitis_Mukhopadhyay,_Pranab_Kumar_Sen(BookZZ.org).pdf:application/pdf;On asymptotics of estimating functions..pdf:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/UCRGZ6HP/2-2.pdf:application/pdf;Ueno2011_semipar.pdf:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/5TC7QPGT/Ueno2011_semipar.pdf:application/pdf},
	Issn = {1532-4435},
	Journal = {Journal of Machine Learning Research},
	Pages = {1977--2020},
	Title = {Generalized TD Learning},
	Volume = {12},
	Year = {2011}}

@inproceedings{antos_value-iteration_2007,
	Abstract = {We consider batch reinforcement learning problems in continuous space, expected total discounted-reward Markovian decision problems when the training data is composed of the trajectory of some fixed behaviour policy. The algorithm studied is policy iteration where in successive iterations the action-value functions of the intermediate policies are obtained by means of approximate value iteration. {PAC}-style polynomial bounds are derived on the number of samples needed to guarantee near-optimal performance. The bounds depend on the mixing rate of the trajectory, the smoothness properties of the underlying Markovian decision problem, the approximation power and capacity of the function set used. One of the main novelties of the paper is that new smoothness constraints are introduced thereby significantly extending the scope of previous results.},
	Author = {Antos, A. and Szepesvari, C. and Munos, R.},
	Booktitle = {{IEEE} International Symposium on Approximate Dynamic Programming and Reinforcement Learning, 2007. {ADPRL} 2007},
	Year = {2007},
	Doi = {10.1109/ADPRL.2007.368207},
	Eventtitle = {{IEEE} International Symposium on Approximate Dynamic Programming and Reinforcement Learning, 2007. {ADPRL} 2007},
	File = {IEEE Xplore Abstract Record:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/IKAQMCEJ/abs_all.html:text/html;IEEE Xplore Full Text PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/FWXPSM5X/Antos et al. - 2007 - Value-Iteration Based Fitted Policy Iteration Lea.pdf:application/pdf},
	Keywords = {action-value function, Algorithm design and analysis, approximate value iteration, Automation, batch reinforcement learning, continuous space, continuous systems, Control systems, discounted-reward Markovian decision problem, Dynamic programming, Extraterrestrial measurements, Interleaved codes, iterative methods, Learning, learning (artificial intelligence), Markov processes, policy iteration, Polynomials, single trajectory, State-space methods, Training data},
	Pages = {330--337},
	Shorttitle = {Value-Iteration Based Fitted Policy Iteration},
	Title = {Value-Iteration Based Fitted Policy Iteration: Learning with a Single Trajectory},
	Bdsk-Url-1 = {http://dx.doi.org/10.1109/ADPRL.2007.368207}}

@article{geist_off-policy_2013,
	Abstract = {In the framework of Markov Decision Processes, off-policy learning, that is the problem of learning a linear approximation of the value function of some fixed policy from one trajectory possibly generated by some other policy. We briefly review on-policy learning algorithms of the literature (gradient-based and least-squares-based), adopting a unified algorithmic view. Then, we highlight a systematic approach for adapting them to off-policy learning with eligibility traces. This leads to some known algorithms - off-policy {LSTD}({\textbackslash}lambda), {LSPE}({\textbackslash}lambda), {TD}({\textbackslash}lambda), {TDC}/{GQ}({\textbackslash}lambda) - and suggests new extensions - off-policy {FPKF}({\textbackslash}lambda), {BRM}({\textbackslash}lambda), {gBRM}({\textbackslash}lambda), {GTD}2({\textbackslash}lambda). We describe a comprehensive algorithmic derivation of all algorithms in a recursive and memory-efficent form, discuss their known convergence properties and illustrate their relative empirical behavior on Garnet problems. Our experiments suggest that the most standard algorithms on and off-policy {LSTD}({\textbackslash}lambda)/{LSPE}({\textbackslash}lambda) - and {TD}({\textbackslash}lambda) if the feature space dimension is too large for a least-squares approach - perform the best.},
	Author = {Geist, Matthieu and Scherrer, Bruno},
	Year = {2013},
	Eprint = {1304.3999},
	Eprinttype = {arxiv},
	File = {arXiv\:1304.3999 PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/FZP3HSCT/Geist and Scherrer - 2013 - Off-policy Learning with Eligibility Traces A Sur.pdf:application/pdf;arXiv.org Snapshot:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/RNMSFUUW/1304.html:text/html},
	Journaltitle = {{arXiv}:1304.3999 [cs]},
	Keywords = {off-policy evlauation method},
	Shorttitle = {Off-policy Learning with Eligibility Traces},
	Title = {Off-policy Learning with Eligibility Traces: A Survey},
	Url = {http://arxiv.org/abs/1304.3999},
	Urldate = {2015-10-22},
	Bdsk-Url-1 = {http://arxiv.org/abs/1304.3999}}

@article{yu_least_2010,
	Author = {Yu, Huizhen},
	Date-Modified = {2015-11-29 23:59:30 +0000},
	Doi = {10.1137/100807879},
	File = {Least Squares Temporal Difference Methods\: An Analysis Under General Conditions:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/S3PTAXGU/47417022_Least_Squares_Temporal_Difference_Methods_An_Analysis_Under_General_Conditions.html:text/html},
	Issn = {0363-0129},
	Journal = {{SIAM} Journal on Control and Optimization},
	Number = {6},
	Shorttitle = {Least Squares Temporal Difference Methods},
	Title = {Least Squares Temporal Difference Methods: An Analysis Under General Conditions},
	Volume = {50},
	Year = {2010},
	Bdsk-Url-1 = {http://dx.doi.org/10.1137/100807879}}

@article{matthieu_geist_statistically_2010,
	Author = {Matthieu Geist, Olivier Pietquin},
	Year = {2010},
	Doi = {10.1109/ICUMT.2010.5676598},
	File = {Statistically Linearized Least-Squares Temporal Differences (PDF Download Available):/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/GHG5XSFK/224208356_Statistically_Linearized_Least-Squares_Temporal_Differences.html:text/html},
	Issn = {2157-0221},
	Journal = {2010 International Congress on Ultra Modern Telecommunications and Control Systems and Workshops, {ICUMT} 2010},
	Pages = {450 -- 457},
	Title = {Statistically Linearized Least-Squares Temporal Differences},
	Bdsk-Url-1 = {http://dx.doi.org/10.1109/ICUMT.2010.5676598}}

@article{xu_kernel-based_2007,
	Abstract = {In this paper, we present a kernel-based least squares policy iteration ({KLSPI}) algorithm for reinforcement learning ({RL}) in large or continuous state spaces, which can be used to realize adaptive feedback control of uncertain dynamic systems. By using {KLSPI}, near-optimal control policies can be obtained without much a priori knowledge on dynamic models of control plants. In {KLSPI}, Mercer kernels are used in the policy evaluation of a policy iteration process, where a new kernel-based least squares temporal-difference algorithm called {KLSTD}-Q is proposed for efficient policy evaluation. To keep the sparsity and improve the generalization ability of {KLSTD}-Q solutions, a kernel sparsification procedure based on approximate linear dependency ({ALD}) is performed. Compared to the previous works on approximate {RL} methods, {KLSPI} makes two progresses to eliminate the main difficulties of existing results. One is the better convergence and (near) optimality guarantee by using the {KLSTD}-Q algorithm for policy evaluation with high precision. The other is the automatic feature selection using the {ALD}-based kernel sparsification. Therefore, the {KLSPI} algorithm provides a general {RL} method with generalization performance and convergence guarantee for large-scale Markov decision problems ({MDPs}). Experimental results on a typical {RL} task for a stochastic chain problem demonstrate that {KLSPI} can consistently achieve better learning efficiency and policy quality than the previous least squares policy iteration ({LSPI}) algorithm. Furthermore, the {KLSPI} method was also evaluated on two nonlinear feedback control problems, including a ship heading control problem and the swing up control of a double-link underactuated pendulum called acrobot. Simulation results illustrate that the proposed method can optimize controller performance using little a priori information of uncertain dynamic systems. It is also demonstrated that {KLSPI} can be applied to online learning control by incorporating a- - n initial controller to ensure online performance.},
	Author = {Xu, Xin and Hu, Dewen and Lu, Xicheng},
	Year = {2007},
	Doi = {10.1109/TNN.2007.899161},
	File = {IEEE Xplore Abstract Record:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/W9M36PBS/abs_all.html:text/html;IEEE Xplore Full Text PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/2BBVH5QB/Xu et al. - 2007 - Kernel-Based Least Squares Policy Iteration for Re.pdf:application/pdf},
	Issn = {1045-9227},
	Journal = {{IEEE} Transactions on Neural Networks},
	Keywords = {acrobot, adaptive control, adaptive feedback control, Algorithms, Approximate dynamic programming, approximate linear dependency, Artificial Intelligence, Biomimetics, Computer Simulation, continuous state spaces, control plants, convergence, Decision Support Techniques, double-link underactuated pendulum, Error Bound, feedback, Feedback control, generalization ability, iterative methods, Kernel, kernel-based least squares policy iteration, kernel-based least squares temporal-difference algorithm, kernel methods, kernel sparsification procedure, large-scale Markov decision problems, Learning, learning (artificial intelligence), least squares, Least-Squares Analysis, Least squares approximation, least squares approximations, Least squares methods, least squares policy iteration algorithm, Linear approximation, Markov Chains, Markov decision problems ({MDPs}), Markov processes, Mercer kernels, Models, Theoretical, near-optimal control policy, nonlinear control systems, nonlinear feedback control, online learning control, optimal control, policy evaluation, Programmable control, reinforcement learning, Reinforcement learning ({RL}), Reinforcement (Psychology), ship heading control problem, State-space methods, stochastic chain problem, swing up control, uncertain dynamic systems, uncertain systems},
	Number = {4},
	Pages = {973--992},
	Title = {Kernel-Based Least Squares Policy Iteration for Reinforcement Learning},
	Volume = {18},
	Bdsk-Url-1 = {http://dx.doi.org/10.1109/TNN.2007.899161}}

@article{chen_efficient_2013,
	Author = {Chen, Shenglei and Chen, Geng and Gu, Ruijun},
	Year = {2013},
	Doi = {10.1016/j.knosys.2013.02.010},
	Issn = {09507051},
	Journal = {Knowledge-Based Systems},
	Langid = {english},
	Pages = {94--99},
	Title = {An efficient L2-norm regularized least-squares temporal difference learning algorithm},
	Url = {http://linkinghub.elsevier.com/retrieve/pii/S0950705113000695},
	Urldate = {2015-11-29},
	Volume = {45},
	Bdsk-Url-1 = {http://linkinghub.elsevier.com/retrieve/pii/S0950705113000695},
	Bdsk-Url-2 = {http://dx.doi.org/10.1016/j.knosys.2013.02.010}}

@incollection{hoffman_regularized_2011,
	Abstract = {The construction of a suitable set of features to approximate value functions is a central problem in reinforcement learning ({RL}). A popular approach to this problem is to use high-dimensional feature spaces together with least-squares temporal difference learning ({LSTD}). Although this combination allows for very accurate approximations, it often exhibits poor prediction performance because of overfitting when the number of samples is small compared to the number of features in the approximation space. In the linear regression setting, regularization is commonly used to overcome this problem. In this paper, we review some regularized approaches to policy evaluation and we introduce a novel scheme (L 21) which uses ℓ2 regularization in the projection operator and an ℓ1 penalty in the fixed-point step. We show that such formulation reduces to a standard Lasso problem. As a result, any off-the-shelf solver can be used to compute its solution and standardization techniques can be applied to the data. We report experimental results showing that L 21 is effective in avoiding overfitting and that it compares favorably to existing ℓ1 regularized methods.},
	Author = {Hoffman, Matthew W. and Lazaric, Alessandro and Ghavamzadeh, Mohammad and Munos, R{\'e}mi},
	Booktitle = {Recent Advances in Reinforcement Learning},
	Year = {2011},
	Editor = {Sanner, Scott and Hutter, Marcus},
	File = {Full Text PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/5IU5IN29/Hoffman et al. - 2011 - Regularized Least Squares Temporal Difference Lear.pdf:application/pdf;Snapshot:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/QQWR5VSV/10.html:text/html},
	Isbn = {978-3-642-29945-2 978-3-642-29946-9},
	Keywords = {Algorithm Analysis and Problem Complexity, Artificial Intelligence (incl. Robotics), Computation by Abstract Devices, Database Management, Information Systems Applications (incl. Internet), Probability and Statistics in Computer Science},
	Langid = {english},
	Note = {{DOI}: 10.1007/978-3-642-29946-9\_13},
	Number = {7188},
	Pages = {102--114},
	Publisher = {Springer Berlin Heidelberg},
	Rights = {{\copyright}2012 Springer-Verlag Berlin Heidelberg},
	Series = {Lecture Notes in Computer Science},
	Title = {Regularized Least Squares Temporal Difference Learning with Nested ℓ2 and ℓ1 Penalization},
	Url = {http://link.springer.com/chapter/10.1007/978-3-642-29946-9_13},
	Urldate = {2015-11-06},
	Bdsk-Url-1 = {http://link.springer.com/chapter/10.1007/978-3-642-29946-9_13}}

@book{kolter_regularization_2009,
	Abstract = {We consider the task of reinforcement learning with linear value function approximation. Temporal difference algorithms, and in particular the Least-Squares Temporal Difference ({LSTD}) algorithm, provide a method for learning the parameters of the value function, but when the number of features is large this algorithm can over-fit to the data and is computationally expensive. In this paper, we propose a regularization framework for the {LSTD} algorithm that overcomes these difficulties. In particular, we focus on the case of l1 regularization, which is robust to irrelevant features and also serves as a method for feature selection. Although the l1 regularized {LSTD} solution cannot be expressed as a convex optimization problem, we present an algorithm similar to the Least Angle Regression ({LARS}) algorithm that can efficiently compute the optimal solution. Finally, we demonstrate the performance of the algorithm experimentally.},
	Author = {Kolter, J. Zico and Ng, Andrew Y.},
	Year = {2009},
	File = {Citeseer - Full Text PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/2TXGJDFM/Kolter and Ng - 2009 - Regularization and feature selection in least-squa.pdf:application/pdf;Citeseer - Snapshot:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/D5AG9CTD/summary.html:text/html},
	Title = {Regularization and feature selection in least-squares temporal difference learning}}

@inproceedings{bradtke_linear_1996,
	Abstract = {Abstract. We introduce two new temporal difference ({TD}) algorithms based on the theory of linear leastsquares function approximation. We define an algorithm we call Least-Squares {TD} ({LS} {TD}) for which we prove probability-one convergence when it is used with a function approximator linear in the adjustable parameters. We then define a recursive version of this algorithm, Recursive Least-Squares {TD} ({RLS} {TD}). Although these new {TD} algorithms require more computation per time-step than do Sutton's {TD}(A) algorithms, they are more efficient in a statistical sense because they extract more information from training experiences. We describe a simulation experiment showing the substantial improvement in learning rate achieved by {RLS} {TD} in an example Markov prediction problem. To quantify this improvement, we introduce the {TD} error variance of a Markov chain, arc,, and experimentally conclude that the convergence rate of a {TD} algorithm depends linearly on {\textasciitilde}ro. In addition to converging more rapidly, {LS} {TD} and {RLS} {TD} do not have control parameters, such as a learning rate parameter, thus eliminating the possibility of achieving poor performance by an unlucky choice of parameters.},
	Author = {Bradtke, Steven J. and Barto, Andrew G. and Kaelbling, Pack},
	Booktitle = {Machine Learning},
	Year = {1996},
	File = {Citeseer - Full Text PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/ZS74USRG/Bradtke et al. - 1996 - Linear least-squares algorithms for temporal diffe.pdf:application/pdf;Citeseer - Snapshot:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/H27FAQ8J/summary.html:text/html},
	Keywords = {{LSTD}},
	Pages = {22--33},
	Title = {Linear least-squares algorithms for temporal difference learning}}

@book{sutton1998introduction,
	title={Introduction to reinforcement learning},
	author={Sutton, Richard S and Barto, Andrew G and others},
	volume={135},
	year={1998},
	publisher={MIT press Cambridge}
}

@inproceedings{jiang2016doubly,
	title={Doubly robust off-policy value evaluation for reinforcement learning},
	author={Jiang, Nan and Li, Lihong},
	booktitle={International Conference on Machine Learning},
	pages={652--661},
	year={2016}
}

@book{chakraborty2013statistical,
	title={Statistical methods for dynamic treatment regimes},
	author={Chakraborty, Bibhas and Moodie, EE},
	year={2013},
	publisher={Springer}
}

@article{murphy2001marginal,
	title={Marginal mean models for dynamic regimes},
	author={Murphy, Susan A and van der Laan, Mark J and Robins, James M and Conduct Problems Prevention Research Group},
	journal={Journal of the American Statistical Association},
	volume={96},
	number={456},
	pages={1410--1423},
	year={2001},
	publisher={Taylor \& Francis}
}


@book{bertsekas1996neuro,
	title={Neuro-dynamic programming},
	author={Bertsekas, Dimitri P and Tsitsiklis, John N},
	volume={5},
	year={1996},
	publisher={Athena Scientific Belmont, MA}
}
@inproceedings{boyan_technical_2002,
	Abstract = {Abstract. {TD}(λ) is a popular family of algorithms for approximate policy evaluation in large {MDPs}. {TD}(λ) works by incrementally updating the value function after each observed transition. It has two major drawbacks: it may make inefficient use of data, and it requires the user to manually tune a stepsize schedule for good performance. For the case of linear value function approximations and λ = 0, the Least-Squares {TD} ({LSTD}) algorithm of Bradtke and Barto (1996, Machine learning, 22:1--3, 33--57) eliminates all stepsize parameters and improves data efficiency. This paper updates Bradtke and Barto's work in three significant ways. First, it presents a simpler derivation of the {LSTD} algorithm. Second, it generalizes from λ = 0 to arbitrary values of λ; at the extreme of λ = 1, the resulting new algorithm is shown to be a practical, incremental formulation of supervised linear regression. Third, it presents a novel and intuitive interpretation of {LSTD} as a model-based reinforcement learning technique.},
	Author = {Boyan, Justin A.},
	Booktitle = {Machine Learning},
	Year = {2002},
	File = {Citeseer - Full Text PDF:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/4XWRX4V7/Boyan - 2002 - Technical update Least-squares temporal differenc.pdf:application/pdf;Citeseer - Snapshot:/Users/Peng/Library/Application Support/Zotero/Profiles/5iwakgz5.default/zotero/storage/WMXKSJXV/summary.html:text/html},
	Pages = {233--246},
	Shorttitle = {Technical update},
	Title = {Technical update: Least-squares temporal difference learning}}

@inproceedings{ueno2008semiparametric,
	title={A semiparametric statistical approach to model-free policy evaluation},
	author={Ueno, Tsuyoshi and Kawanabe, Motoaki and Mori, Takeshi and Maeda, Shin-ichi and Ishii, Shin},
	booktitle={Proceedings of the 25th international conference on Machine learning},
	pages={1072--1079},
	year={2008},
	organization={ACM}
}

@article{Wang2012,
	Author = {Wang, Lu and Rotnitzky, Andrea and Lin, Xihong and Millikan, Randall E. and Thall, Peter F.},
	Journal = {Journal of the American Statistical Association},
	Number = {498},
	Pages = {493-508},
	Title = {Evaluation of Viable Dynamic Treatment Regimes in a Sequentially Randomized Trial of Advanced Prostate Cancer},
	Volume = {107},
	Year = {2012}}

@inproceedings{Robins2004,
	Address = {New York},
	Author = {Robins, J. M.},
	Booktitle = {Proceedings of the Second Seattle Symposium on Biostatistics},
	Editor = {Lin, D. Y. and Heagerty, P.},
	Pages = {189-326},
	Publisher = {Springer},
	Title = {Optimal Structural Nested Models for Optimal Sequential Decisions},
	Year = {2004}}

@article{king2015technologies,
	title={Technologies to measure and modify physical activity and eating environments},
	author={King, Abby C and Glanz, Karen and Patrick, Kevin},
	journal={American journal of preventive medicine},
	volume={48},
	number={5},
	pages={630--638},
	year={2015},
	publisher={Elsevier}
}

@article{mckay2018evaluating,
	title={Evaluating mobile phone applications for health behaviour change: a systematic review},
	author={McKay, Fiona H and Cheng, Christina and Wright, Annemarie and Shill, Jane and Stephens, Hugh and Uccellini, Mary},
	journal={Journal of telemedicine and telecare},
	volume={24},
	number={1},
	pages={22--30},
	year={2018},
	publisher={SAGE Publications Sage UK: London, England}
}

@article{witkiewitz2014development,
	title={Development and evaluation of a mobile intervention for heavy drinking and smoking among college students.},
	author={Witkiewitz, Katie and Desai, Sruti A and Bowen, Sarah and Leigh, Barbara C and Kirouac, Megan and Larimer, Mary E},
	journal={Psychology of Addictive Behaviors},
	volume={28},
	number={3},
	pages={639},
	year={2014},
	publisher={American Psychological Association}
}

@article{patrick2009text,
	title={A text message--based intervention for weight loss: randomized controlled trial},
	author={Patrick, Kevin and Raab, Fred and Adams, Marc A and Dillon, Lindsay and Zabinski, Marian and Rock, Cheryl L and Griswold, William G and Norman, Gregory J},
	journal={Journal of medical Internet research},
	volume={11},
	number={1},
	year={2009},
	publisher={JMIR Publications Inc.}
}

@article{kristjansdottir2013smartphone,
	title={A smartphone-based intervention with diaries and therapist-feedback to reduce catastrophizing and increase functioning in women with chronic widespread pain: randomized controlled trial},
	author={Kristj{\'a}nsd{\'o}ttir, {\'O}l{\"o}f Birna and Fors, Egil A and Eide, Erlend and Finset, Arnstein and Stensrud, Tonje Lauritzen and van Dulmen, Sandra and Wigers, Sigrid H{\o}rven and Eide, Hilde},
	journal={Journal of medical Internet research},
	volume={15},
	number={1},
	year={2013},
	publisher={JMIR Publications Inc.}
}

@article{anastasiadou2018systematic,
	title={A systematic review of mHealth interventions for the support of eating disorders},
	author={Anastasiadou, Dimitra and Folkvord, Frans and Lupia{\~n}ez-Villanueva, Francisco},
	journal={European Eating Disorders Review},
	volume={26},
	number={5},
	pages={394--416},
	year={2018},
	publisher={Wiley Online Library}
}

@article{King2013,
	Author = {King, Abby C. and Castro, Cynthia M. and Buman, Matthew P. and Hekler, Eric B. and Urizar, Guido G., Jr and Ahn, David K.},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1007/s12160-013-9501-y},
	Date-Added = {2015-01-24 00:24:02 +0000},
	Date-Modified = {2015-01-24 00:24:44 +0000},
	Doi = {10.1007/s12160-013-9501-y},
	Issn = {0883-6612},
	Journal = {Annals of Behavioral Medicine},
	Keywords = {Physical activity; Dietary change; Multiple health behaviors; Sequential; Simultaneous; Stress},
	Language = {English},
	Number = {2},
	Pages = {157-168},
	Publisher = {Springer US},
	Tempurl = {http://dx.doi.org/10.1007/s12160-013-9501-y},
	Title = {Behavioral Impacts of Sequentially versus Simultaneously Delivered Dietary Plus Physical Activity Interventions: the CALM Trial},
	Volume = {46},
	Year = {2013},
	Bdsk-Url-1 = {http://dx.doi.org/10.1007/s12160-013-9501-y}}

@article{Dallery,
	Author = {Dallery, J and Cassidy, RN and Raiff, BR},
	Date-Added = {2015-09-05 08:34:59 +0000},
	Date-Modified = {2015-09-05 08:43:36 +0000},
	Journal = {Journal of Medical Internet Research},
	Pages = {e22},
	Title = {Single-Case Experimental Designs to Evaluate Novel Technology-Based Health Interventions},
	Volume = {15},
	Year = {2013}}

@article{ShadishSullivan,
	Author = {Shadish, WilliamR. and Sullivan, KristynnJ.},
	Date-Added = {2015-09-05 08:29:44 +0000},
	Date-Modified = {2015-09-05 08:29:59 +0000},
	Doi = {10.3758/s13428-011-0111-y},
	Journal = {Behavior Research Methods},
	Keywords = {Single-case design; Characteristics; Outcome studies},
	Language = {English},
	Number = {4},
	Pages = {971-980},
	Publisher = {Springer-Verlag},
	Title = {Characteristics of single-case designs used to assess intervention effects in 2008},
	Url = {http://dx.doi.org/10.3758/s13428-011-0111-y},
	Volume = {43},
	Year = {2011},
	Bdsk-Url-1 = {http://dx.doi.org/10.3758/s13428-011-0111-y}}

@article{DalleryRaiff,
	Author = {Dallery, Jesse and Raiff, BethanyR},
	Date-Added = {2015-09-05 08:23:43 +0000},
	Date-Modified = {2015-09-05 08:24:17 +0000},
	Doi = {10.1007/s13142-014-0258-z},
	Issn = {1869-6716},
	Journal = {Translational Behavioral Medicine},
	Keywords = {Generality; Optimization; Research designs; Single-case designs},
	Language = {English},
	Number = {3},
	Pages = {290-303},
	Publisher = {Springer US},
	Title = {Optimizing behavioral health interventions with single-case designs: from development to dissemination},
	Url = {http://dx.doi.org/10.1007/s13142-014-0258-z},
	Volume = {4},
	Year = {2014},
	Bdsk-Url-1 = {http://dx.doi.org/10.1007/s13142-014-0258-z}}


@article{Wang2012,
	Author = {Wang, Lu and Rotnitzky, Andrea and Lin, Xihong and Millikan, Randall E. and Thall, Peter F.},
	Journal = {Journal of the American Statistical Association},
	Number = {498},
	Pages = {493-508},
	Title = {Evaluation of Viable Dynamic Treatment Regimes in a Sequentially Randomized Trial of Advanced Prostate Cancer},
	Volume = {107},
	Year = {2012}}

@article{Robins2004,
	Address = {New York},
	Author = {Robins, J. M.},
	Journal = {In Proceedings of the Second Seattle Symposium on Biostatistics},
	Editor = {Lin, D. Y. and Heagerty, P.},
	Pages = {189-326},
	Publisher = {Springer},
	Title = {Optimal Structural Nested Models for Optimal Sequential Decisions},
	Volume = {179},
	Year = {2004}}

@article{King2013,
	Author = {King, Abby C. and Castro, Cynthia M. and Buman, Matthew P. and Hekler, Eric B. and Urizar, Guido G., Jr and Ahn, David K.},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1007/s12160-013-9501-y},
	Date-Added = {2015-01-24 00:24:02 +0000},
	Date-Modified = {2015-01-24 00:24:44 +0000},
	Doi = {10.1007/s12160-013-9501-y},
	Issn = {0883-6612},
	Journal = {Annals of Behavioral Medicine},
	Keywords = {Physical activity; Dietary change; Multiple health behaviors; Sequential; Simultaneous; Stress},
	Language = {English},
	Number = {2},
	Pages = {157-168},
	Publisher = {Springer US},
	Tempurl = {http://dx.doi.org/10.1007/s12160-013-9501-y},
	Title = {Behavioral Impacts of Sequentially versus Simultaneously Delivered Dietary Plus Physical Activity Interventions: the CALM Trial},
	Volume = {46},
	Year = {2013},
	Bdsk-Url-1 = {http://dx.doi.org/10.1007/s12160-013-9501-y}}

@article{GEE,
	Abstract = {This paper proposes an extension of generalized linear models to the analysis of longitudinal data. We introduce a class of estimating equations that give consistent estimates of the regression parameters and of their variance under mild assumptions about the time dependence. The estimating equations are derived without specifying the joint distribution of a subject's observations yet they reduce to the score equations for niultivariate Gaussian outcomes. Asymptotic theory is presented for the general class of estimators. Specific cases in which we assume independence, m-dependence and exchangeable correlation structures from each subject are discussed. Efficiency of the pioposecl estimators in two simple situations is considered. The approach is closely related to quasi-likelihood.},
	Author = {Liang, Kung-Yee and Zeger, Scott L.},
	Bdsk-Tempurl-1 = {http://biomet.oxfordjournals.org/content/73/1/13.abstract},
	Bdsk-Tempurl-2 = {http://dx.doi.org/10.1093/biomet/73.1.13},
	Doi = {10.1093/biomet/73.1.13},
	Eprint = {http://biomet.oxfordjournals.org/content/73/1/13.full.pdf+html},
	Journal = {Biometrika},
	Number = {1},
	Pages = {13-22},
	Tempurl = {http://biomet.oxfordjournals.org/content/73/1/13.abstract},
	Title = {Longitudinal data analysis using generalized linear models},
	Volume = {73},
	Year = {1986},
	Bdsk-Url-1 = {http://dx.doi.org/10.1093/biomet/73.1.13}}

@article{Tu2004,
	Abstract = {Existing methods for power and sample size estimation for longitudinal and other clustered study designs have limited applications. In this paper, we review and extend existing approaches to improve these limitations. In particular, we focus on power analysis for the two most popular approaches for clustered data analysis, the generalized estimating equations and the linear mixed-effects models. By basing the derivation of the power function on the asymptotic distribution of the model estimates, the proposed approach provides estimates of power that are consistent with the methods of inference for data analysis. The proposed methodology is illustrated with numerous examples that are motivated by real study designs. Copyright {\copyright} 2004 John Wiley & Sons, Ltd.},
	Author = {Tu, X. M. and Kowalski, J. and Zhang, J. and Lynch, K. G. and Crits-Christoph, P.},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1002/sim.1869},
	Date-Added = {2015-01-22 18:28:21 +0000},
	Date-Modified = {2015-01-22 18:28:30 +0000},
	Doi = {10.1002/sim.1869},
	Issn = {1097-0258},
	Journal = {Statistics in Medicine},
	Keywords = {epidemiological study, GEE, HIV, linear mixed-effects models, intraclass correlation, psychosocial and survey research},
	Number = {18},
	Pages = {2799--2815},
	Publisher = {John Wiley & Sons, Ltd.},
	Tempurl = {http://dx.doi.org/10.1002/sim.1869},
	Title = {Power analyses for longitudinal trials and other clustered designs},
	Volume = {23},
	Year = {2004},
	Bdsk-Url-1 = {http://dx.doi.org/10.1002/sim.1869}}

@article{Hotelling,
	Author = {Hotelling, Harold},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1214/aoms/1177732979},
	Doi = {10.1214/aoms/1177732979},
	Fjournal = {The Annals of Mathematical Statistics},
	Journal = {Ann. Math. Statist.},
	Month = {08},
	Number = {3},
	Pages = {360--378},
	Publisher = {The Institute of Mathematical Statistics},
	Tempurl = {http://dx.doi.org/10.1214/aoms/1177732979},
	Title = {The Generalization of Student's Ratio},
	Volume = {2},
	Year = {1931},
	Bdsk-Url-1 = {http://dx.doi.org/10.1214/aoms/1177732979}}

@article{LiRedden,
	Abstract = {The sandwich estimator in generalized estimating equations (GEE) approach underestimates the true variance in small samples and consequently results in inflated type I error rates in hypothesis testing. This fact limits the application of the GEE in cluster-randomized trials (CRTs) with few clusters. Under various CRT scenarios with correlated binary outcomes, we evaluate the small sample properties of the GEE Wald tests using bias-corrected sandwich estimators. Our results suggest that the GEE Wald z-test should be avoided in the analyses of CRTs with few clusters even when bias-corrected sandwich estimators are used. With t-distribution approximation, the Kauermann and Carroll (KC)-correction can keep the test size to nominal levels even when the number of clusters is as low as 10 and is robust to the moderate variation of the cluster sizes. However, in cases with large variations in cluster sizes, the Fay and Graubard (FG)-correction should be used instead. Furthermore, we derive a formula to calculate the power and minimum total number of clusters one needs using the t-test and KC-correction for the CRTs with binary outcomes. The power levels as predicted by the proposed formula agree well with the empirical powers from the simulations. The proposed methods are illustrated using real CRT data. We conclude that with appropriate control of type I error rates under small sample sizes, we recommend the use of GEE approach in CRTs with binary outcomes because of fewer assumptions and robustness to the misspecification of the covariance structure. Copyright {\copyright} 2014 John Wiley & Sons, Ltd.},
	Author = {Li, Peng and Redden, David T.},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1002/sim.6344},
	Date-Added = {2015-01-21 19:07:43 +0000},
	Date-Modified = {2015-01-21 19:07:52 +0000},
	Doi = {10.1002/sim.6344},
	Issn = {1097-0258},
	Journal = {Statistics in Medicine},
	Keywords = {generalized estimating equations (GEE), correlated data, type I error rates, power, sample size},
	Number = {2},
	Pages = {281--296},
	Tempurl = {http://dx.doi.org/10.1002/sim.6344},
	Title = {Small sample performance of bias-corrected sandwich estimators for cluster-randomized trials with binary outcomes},
	Volume = {34},
	Year = {2015},
	Bdsk-Url-1 = {http://dx.doi.org/10.1002/sim.6344}}

@article{Jawbone,
	Author = {Buman, M.A. and Hekler, E.B. and Floegel, T. and Florez Pregonero, A.,Gutierrez and M. and Riley, K.},
	Date-Added = {2015-01-18 04:26:39 +0000},
	Date-Modified = {2015-03-31 20:23:08 +0000},
	Journal = {In Proceedings of the American Medical Society for Sports Medicine.},
	Title = {Step validation of the Jawbone UP band in normal, overweight, and obese adults.},
	Year = {2014}}

@article{Rubin1978,
	Author = {Rubin, Donald B.},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1214/aos/1176344064},
	Date-Added = {2015-01-18 03:55:06 +0000},
	Date-Modified = {2015-01-18 03:55:16 +0000},
	Doi = {10.1214/aos/1176344064},
	Fjournal = {The Annals of Statistics},
	Journal = {Ann. Statist.},
	Month = {01},
	Number = {1},
	Pages = {34--58},
	Publisher = {The Institute of Mathematical Statistics},
	Tempurl = {http://dx.doi.org/10.1214/aos/1176344064},
	Title = {Bayesian Inference for Causal Effects: The Role of Randomization},
	Volume = {6},
	Year = {1978},
	Bdsk-Url-1 = {http://dx.doi.org/10.1214/aos/1176344064}}

@article{Robins1986,
	Abstract = {In observational cohort mortality studies with prolonged periods of exposure to the agent under study, it is not uncommon for risk factors for death to be determinants of subsequent exposure. For instance, in occupational mortality studies date of termination of employment is both a determinant of future exposure (since terminated individuals receive no further exposure) and an independent risk factor for death (since disabled individuals tend to leave employment). When current risk factor status determines subsequent exposure and is determined by previous exposure, standard analyses that estimate age-specific mortality rates as a function of cumulative exposure may underestimate the true effect of exposure on mortality whether or not one adjusts for the risk factor in the analysis. This observation raises the question, which if any population parameters can be given a causal interpretation in observational mortality studies? In answer, we offer a graphical approach to the identification and computation of causal parameters in mortality studies with sustained exposure periods. This approach is shown to be equivalent to an approach in which the observational study is identified with a hypothetical double-blind randomized trial in which data on each subject's assigned treatment protocol has been erased from the data file. Causal inferences can then be made by comparing mortality as a function of treatment protocol, since, in a double-blind randomized trial missing data on treatment protocol, the association of mortality with treatment protocol can still be estimated. We reanalyze the mortality experience of a cohort of arsenic-exposed copper smelter workers with our method and compare our results with those obtained using standard methods. We find an adverse effect of arsenic exposure on all-cause and lung cancer mortality which standard methods fail to detect. },
	Author = {J.M. Robins},
	Bdsk-Tempurl-1 = {http://www.sciencedirect.com/science/article/pii/0270025586900886},
	Bdsk-Tempurl-2 = {http://dx.doi.org/10.1016/0270-0255(86)90088-6},
	Date-Added = {2015-01-18 03:50:00 +0000},
	Date-Modified = {2015-01-18 03:58:37 +0000},
	Doi = {http://dx.doi.org/10.1016/0270-0255(86)90088-6},
	Issn = {0270-0255},
	Journal = {Mathematical Modelling},
	Number = {9--12},
	Pages = {1393 - 1512},
	Tempurl = {http://www.sciencedirect.com/science/article/pii/0270025586900886},
	Title = {A new approach to causal inference in mortality studies with a sustained exposure period---application to control of the healthy worker survivor effect},
	Volume = {7},
	Year = {1986},
	Bdsk-File-1 = { }}

@article{Robins1987,
	Author = {J.M. Robins},
	Bdsk-Tempurl-1 = {http://www.sciencedirect.com/science/article/pii/0898122187902380},
	Bdsk-Tempurl-2 = {http://dx.doi.org/10.1016/0898-1221(87)90238-0},
	Date-Added = {2015-01-18 03:47:59 +0000},
	Date-Modified = {2015-01-18 03:58:53 +0000},
	Doi = {http://dx.doi.org/10.1016/0898-1221(87)90238-0},
	Issn = {0898-1221},
	Journal = {Computers and Mathematics with Applications},
	Number = {9--12},
	Pages = {923 - 945},
	Tempurl = {http://www.sciencedirect.com/science/article/pii/0898122187902380},
	Title = {Addendum to ``a new approach to causal inference in mortality studies with a sustained exposure period---application to control of the healthy worker survivor effect''},
	Volume = {14},
	Year = {1987},
	Bdsk-Url-1 = {http://dx.doi.org/10.1016/0898-1221(87)90238-0}}

@book{Cohen1988,
	Author = {Jocob Cohen},
	Date-Added = {2015-01-18 02:43:34 +0000},
	Date-Modified = {2015-01-18 02:48:53 +0000},
	Edition = {2nd},
	Month = {July 1},
	Publisher = {Routledge},
	Title = {Statistical Power Analysis for the Behavioral Sciences(2nd)},
	Year = {1988}}

@article{ManclandDeRouen2001,
	Abstract = {Summary.  In this paper, we propose an alternative covariance estimator to the robust covariance estimator of generalized estimating equations (GEE). Hypothesis tests using the robust covariance estimator can have inflated size when the number of independent clusters is small. Resampling methods, such as the jackknife and bootstrap, have been suggested for covariance estimation when the number of clusters is small. A drawback of the resampling methods when the response is binary is that the methods can break down when the number of subjects is small due to zero or near-zero cell counts caused by resampling. We propose a bias-corrected covariance estimator that avoids this problem. In a small simulation study, we compare the bias-corrected covariance estimator to the robust and jackknife covariance estimators for binary responses for situations involving 10--40 subjects with equal and unequal cluster sizes of 16--64 observations. The bias-corrected covariance estimator gave tests with sizes close to the nominal level even when the number of subjects was 10 and cluster sizes were unequal, whereas the robust and jackknife covariance estimators gave tests with sizes that could be 2--3 times the nominal level. The methods are illustrated using data from a randomized clinical trial on treatment for bone loss in subjects with periodontal disease.},
	Author = {Mancl, Lloyd A. and DeRouen, Timothy A.},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1111/j.0006-341X.2001.00126.x},
	Date-Added = {2015-01-18 02:34:23 +0000},
	Date-Modified = {2015-01-18 02:34:35 +0000},
	Doi = {10.1111/j.0006-341X.2001.00126.x},
	Issn = {1541-0420},
	Journal = {Biometrics},
	Keywords = {Correlated binary data, Logistic regression, Periodontal disease, Simulation study},
	Number = {1},
	Pages = {126--134},
	Publisher = {Blackwell Publishing Ltd},
	Tempurl = {http://dx.doi.org/10.1111/j.0006-341X.2001.00126.x},
	Title = {A Covariance Estimator for GEE with Improved Small-Sample Properties},
	Volume = {57},
	Year = {2001},
	Bdsk-Url-1 = {http://dx.doi.org/10.1111/j.0006-341X.2001.00126.x}}

@book{Box1978,
	Author = {Box, George E and P.Hunter, J. Stuart and Hunter, William Gordon},
	Date-Added = {2015-01-18 02:21:03 +0000},
	Date-Modified = {2015-03-10 22:25:39 +0000},
	Publisher = {Wiley series in probability and mathematical statistics},
	Title = {Statistics for experimenters : an introduction to design, data analysis, and model building},
	Year = {1978}}

@article{Chakraborty2009,
	Abstract = {Multicomponent interventions composed of behavioral, delivery, or implementation factors in addition to medications are becoming increasingly common in health sciences. A natural experimental approach to developing and refining such multicomponent interventions is to start with a large number of potential components and screen out the least active ones. Factorial designs can be used efficiently in this endeavor. We address common criticisms and misconceptions regarding the use of factorial designs in these screening studies. We also provide an operationalization of screening studies. As an example, we consider the use of a screening study in the development of a multicomponent smoking cessation intervention. Simulation results are provided to support the discussions. Copyright {\copyright} 2009 John Wiley & Sons, Ltd.},
	Author = {Chakraborty, Bibhas and Collins, Linda M. and Strecher, Victor J. and Murphy, Susan A.},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1002/sim.3643},
	Date-Added = {2015-01-18 02:15:52 +0000},
	Date-Modified = {2015-01-18 02:16:35 +0000},
	Doi = {10.1002/sim.3643},
	Issn = {1097-0258},
	Journal = {Statistics in Medicine},
	Keywords = {multicomponent intervention, experimental design, fractional factorial design, screening, follow-up studies},
	Number = {21},
	Pages = {2687--2708},
	Publisher = {John Wiley & Sons, Ltd.},
	Tempurl = {http://dx.doi.org/10.1002/sim.3643},
	Title = {Developing multicomponent interventions using fractional factorial designs},
	Volume = {28},
	Year = {2009},
	Bdsk-Url-1 = {http://dx.doi.org/10.1002/sim.3643}}

@article{Kumar2013,
	Address = {Los Alamitos, CA, USA},
	Author = {Santosh Kumar and Wendy Nilsen and Misha Pavel and Mani Srivastava},
	Bdsk-Tempurl-1 = {http://doi.ieeecomputersociety.org/10.1109/MC.2012.392},
	Date-Added = {2015-01-17 22:56:01 +0000},
	Date-Modified = {2015-01-17 22:56:01 +0000},
	Doi = {http://doi.ieeecomputersociety.org/10.1109/MC.2012.392},
	Issn = {0018-9162},
	Journal = {Computer},
	Number = {1},
	Pages = {28-35},
	Publisher = {IEEE Computer Society},
	Title = {Mobile Health: Revolutionizing Healthcare Through Transdisciplinary Research},
	Volume = {46},
	Year = {2013},
	Bdsk-Url-1 = {http://doi.ieeecomputersociety.org/10.1109/MC.2012.392}}

@article{MetzandNilsen2014,
	Abstract = {Improvements in health behavior theory will be central to creating successful interventions that encourage and support behavior change and maintenance. The authors discuss dynamic, multimethod, conceptually driven, and data-rich approaches for the development of testable computational models of health-related behaviors in real time.},
	Author = {Spruijt-Metz, D. and Nilsen, W.},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1109/MPRV.2014.46},
	Date-Added = {2015-01-17 22:56:01 +0000},
	Date-Modified = {2015-01-17 22:56:01 +0000},
	Doi = {10.1109/MPRV.2014.46},
	Issn = {1536-1268},
	Journal = {Pervasive Computing, IEEE},
	Keywords = {behavioural sciences computing;health care;just-in-time;behavior change;behavior maintenance;conceptually driven approach;data-rich approach;dynamic behavior models;health behavior theory;health-related behaviors;just-in-time adaptive interventions;multimethod approach;testable computational models;Adaptation models;Computational modeling;Context awareness;Context modeling;Data models;Deformable models;Maintenance engineering;Real-time systems;JITAI;adaptive interventions;behavior theory;healthcare;just-in-time;pervasive computing},
	Month = {July},
	Number = {3},
	Pages = {13-17},
	Title = {Dynamic Models of Behavior for Just-in-Time Adaptive Interventions},
	Volume = {13},
	Year = {2014},
	Bdsk-Url-1 = {http://dx.doi.org/10.1109/MPRV.2014.46}}

@article{Muessig2013,
	Abstract = {Background: Mobile phone applications (apps) provide a new platform for delivering tailored human immunodeficiency virus (HIV) and sexually transmitted disease (STD) prevention and care. Objective: To identify and evaluate currently available mobile phone apps related to the prevention and care of HIV and other STDs. Methods: We searched the Apple iTunes and Android Google Play stores for HIV/STD-related apps, excluding apps that exclusively targeted industry, providers, and researchers. Each eligible app was downloaded, tested, and assessed for user ratings and functionality as well as 6 broad content areas of HIV prevention and care: HIV/STD disease knowledge, risk reduction/safer sex, condom promotion, HIV/STD testing information, resources for HIV-positive persons, and focus on key populations. Results: Search queries up to May 2012 identified 1937 apps. Of these, 55 unique apps met the inclusion criteria (12 for Android, 29 for iPhone, and 14 for both platforms). Among these apps, 71\% provided disease information about HIV/STDs, 36\% provided HIV/STD testing information or resources, 29\% included information about condom use or assistance locating condoms, and 24\% promoted safer sex. Only 6 apps (11\%) covered all 4 of these prevention areas. Eight apps (15\%) provided tools or resources specifically for HIV/STD positive persons. Ten apps included information for a range of sexual orientations, 9 apps appeared to be designed for racially/ethnically diverse audiences, and 15 apps featured interactive components. Apps were infrequently downloaded (median 100-500 downloads) and not highly rated (average customer rating 3.7 out of 5 stars). Conclusions: Most available HIV/STD apps have failed to attract user attention and positive reviews. Public health practitioners should work with app developers to incorporate elements of evidence-based interventions for risk reduction and improve app inclusiveness and interactivity. },
	Author = {Muessig, E. Kathryn and Pike, C. Emily and LeGrand, Sara and Hightow-Weidman, B. Lisa},
	Bdsk-Tempurl-1 = {http://www.ncbi.nlm.nih.gov/pubmed/23291245},
	Bdsk-Tempurl-2 = {http://dx.doi.org/10.2196/jmir.2301},
	Date-Added = {2015-01-17 22:56:01 +0000},
	Date-Modified = {2015-01-17 22:56:01 +0000},
	Day = {04},
	Doi = {10.2196/jmir.2301},
	Journal = {J Med Internet Res},
	Keywords = {HIV; technology; mobile phone applications},
	Month = {Jan},
	Number = {1},
	Pages = {e1},
	Tempurl = {http://www.ncbi.nlm.nih.gov/pubmed/23291245},
	Title = {Mobile Phone Applications for the Care and Prevention of HIV and Other Sexually Transmitted Diseases: A Review},
	Volume = {15},
	Year = {2013},
	Bdsk-Url-1 = {http://dx.doi.org/10.2196/jmir.2301}}

@article{Nilsen2012,
	Abstract = { Mobile health (mHealth) technologies have the potential to greatly impact health research, health care, and health outcomes, but the exponential growth of the technology has outpaced the science. This article outlines two initiatives designed to enhance the science of mHealth. The mHealth Evidence Workshop used an expert panel to identify optimal methodological approaches for mHealth research. The NIH mHealth Training Institutes address the silos among the many academic and technology areas in mHealth research and is an effort to build the interdisciplinary research capacity of the field. Both address the growing need for high quality mobile health research both in the United States and internationally. mHealth requires a solid, interdisciplinary scientific approach that pairs the rapid change associated with technological progress with a rigorous evaluation approach. The mHealth Evidence Workshop and the NIH mHealth Training Institutes were both designed to address and further develop this scientific approach to mHealth. },
	Author = {Nilsen, Wendy and Kumar, Santosh and Shar, Albert and Varoquiers, Carrie and Wiley, Tisha and Riley, William T. and Pavel, Misha and Atienza, Audie A.},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1080/10810730.2012.677394},
	Date-Added = {2015-01-17 22:56:01 +0000},
	Date-Modified = {2015-01-17 22:56:01 +0000},
	Doi = {10.1080/10810730.2012.677394},
	Eprint = {http://dx.doi.org/10.1080/10810730.2012.677394},
	Journal = {Journal of Health Communication},
	Number = {sup1},
	Pages = {5-10},
	Tempurl = {http://dx.doi.org/10.1080/10810730.2012.677394},
	Title = {Advancing the Science of mHealth},
	Volume = {17},
	Year = {2012},
	Bdsk-Url-1 = {http://dx.doi.org/10.1080/10810730.2012.677394}}

@article{Free2013,
	Abstract = {<p>Caroline Free and colleagues systematically review a fast-moving field, that of the effectiveness of mobile technology interventions delivered to healthcare consumers, and conclude that high-quality, adequately powered trials of optimized interventions are required to evaluate effects on objective outcomes.</p>},
	Author = {Free, Caroline AND Phillips, Gemma AND Galli, Leandro AND Watson, Louise AND Felix, Lambert AND Edwards, Phil AND Patel, Vikram AND Haines, Andy},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1371%2Fjournal.pmed.1001362},
	Bdsk-Tempurl-2 = {http://dx.doi.org/10.1371/journal.pmed.1001362},
	Date-Added = {2015-01-17 22:56:01 +0000},
	Date-Modified = {2015-01-17 22:56:01 +0000},
	Doi = {10.1371/journal.pmed.1001362},
	Journal = {PLoS Med},
	Month = {01},
	Number = {1},
	Pages = {e1001362},
	Publisher = {Public Library of Science},
	Tempurl = {http://dx.doi.org/10.1371%2Fjournal.pmed.1001362},
	Title = {The Effectiveness of Mobile-Health Technology-Based Health Behaviour Change or Disease Management Interventions for Health Care Consumers: A Systematic Review},
	Volume = {10},
	Year = {2013},
	Bdsk-Url-1 = {http://dx.doi.org/10.1371/journal.pmed.1001362}}

@article{Quanbeck2014,
	Abstract = {BACKGROUND:Healthcare reform in the United States is encouraging Federally Qualified Health Centers and other primary-care practices to integrate treatment for addiction and other behavioral health conditions into their practices. The potential of mobile health technologies to manage addiction and comorbidities such as HIV in these settings is substantial but largely untested. This paper describes a protocol to evaluate the implementation of an E-Health integrated communication technology delivered via mobile phones, called Seva, into primary-care settings. Seva is an evidence-based system of addiction treatment and recovery support for patients and real-time caseload monitoring for clinicians.METHODS/DESIGN:Our implementation strategy uses three models of organizational change: the Program Planning Model to promote acceptance and sustainability, the NIATx quality improvement model to create a welcoming environment for change, and Rogers's diffusion of innovations research, which facilitates adaptations of innovations to maximize their adoption potential. We will implement Seva and conduct an intensive, mixed-methods assessment at three diverse Federally Qualified Healthcare Centers in the United States. Our non-concurrent multiple-baseline design includes three periods -- pretest (ending in four months of implementation preparation), active Seva implementation, and maintenance -- with implementation staggered at six-month intervals across sites. The first site will serve as a pilot clinic. We will track the timing of intervention elements and assess study outcomes within each dimension of the Reach, Effectiveness, Adoption, Implementation, and Maintenance framework, including effects on clinicians, patients, and practices. Our mixed-methods approach will include quantitative (e.g., interrupted time-series analysis of treatment attendance, with clinics as the unit of analysis) and qualitative (e.g., staff interviews regarding adaptations to implementation protocol) methods, and assessment of implementation costs.DISCUSSION:If implementation is successful, the field will have a proven technology that helps Federally Qualified Health Centers and affiliated organizations provide addiction treatment and recovery support, as well as a proven strategy for implementing the technology. Seva also has the potential to improve core elements of addiction treatment, such as referral and treatment processes. A mobile technology for addiction treatment and accompanying implementation model could provide a cost-effective means to improve the lives of patients with drug and alcohol problems.TRIAL REGISTRATION:ClinicalTrials.gov (NCT01963234).},
	Author = {Quanbeck, Andrew and Gustafson, David and Marsch, Lisa and McTavish, Fiona and Brown, Randall and Mares, Marie-Louise and Johnson, Roberta and Glass, Joseph and Atwood, Amy and McDowell, Helene},
	Bdsk-Tempurl-1 = {http://www.implementationscience.com/content/9/1/65},
	Bdsk-Tempurl-2 = {http://dx.doi.org/10.1186/1748-5908-9-65},
	Date-Added = {2015-01-17 22:56:01 +0000},
	Date-Modified = {2015-01-17 22:56:01 +0000},
	Doi = {10.1186/1748-5908-9-65},
	Issn = {1748-5908},
	Journal = {Implementation Science},
	Number = {1},
	Pages = {65},
	Pubmedid = {24884976},
	Tempurl = {http://www.implementationscience.com/content/9/1/65},
	Title = {Integrating addiction treatment into primary care using mobile health technology: protocol for an implementation research study},
	Volume = {9},
	Year = {2014},
	Bdsk-Url-1 = {http://dx.doi.org/10.1186/1748-5908-9-65}}

@article{Gustafson2014,
	Author = {Gustafson, David and McTavish FM and Chih M and et al},
	Bdsk-Tempurl-1 = {+%20http://dx.doi.org/10.1001/jamapsychiatry.2013.4642},
	Bdsk-Tempurl-2 = {http://dx.doi.org/10.1001/jamapsychiatry.2013.4642},
	Date-Added = {2015-01-17 22:56:01 +0000},
	Date-Modified = {2015-01-17 22:56:01 +0000},
	Doi = {10.1001/jamapsychiatry.2013.4642},
	Eprint = {/data/Journals/PSYCH/930079/yoi130118.pdf},
	Journal = {JAMA Psychiatry},
	Number = {5},
	Pages = {566-572},
	Tempurl = {+ http://dx.doi.org/10.1001/jamapsychiatry.2013.4642},
	Title = {A smartphone application to support recovery from alcoholism: A randomized clinical trial},
	Volume = {71},
	Year = {2014},
	Bdsk-Url-1 = {http://dx.doi.org/10.1001/jamapsychiatry.2013.4642}}

@article{AlssiandPetry2013,
	Abstract = {
	Aims
	Contingency management (CM) uses tangible incentives to systematically reinforce abstinence and is among the most efficacious psychosocial substance abuse treatments. This study assessed the feasibility and initial efficacy of a portable CM procedure designed to address technical limitations for detecting drinking that have prevented using CM for alcohol problems.
	
	Design
	Participants received a cellphone, breathalyzer and training on video-recording alcohol breath tests (BrACs) and texting results. For 4 weeks, staff texted participants one to three times daily indicating that a breath alcohol concentration test (BrAC) was due within the hour. Participants were randomized to (1) modest compensation for submitting dated time-stamped BrAC videos regardless of results or (2) the same plus CM with escalating vouchers for on-time alcohol-negative tests (n-BrAC; <02 g/dl). `Thank-you' texts were sent, with CM patients also informed of results-based earnings.
	
	Setting
	Participants' natural environment.
	
	Participants
	Adults (n = 30; ≥ 21 years) who drank frequently but were not physiologically dependent.
	
	Measurements
	Drinking and related problems were assessed at intake and week 4. BrACs and self-reports of drinking were collected throughout. The primary outcome was the percentage of n-BrACs. Other outcomes were the longest duration of consecutive n-BrACs (longest duration of abstinence) and self-reports of drinking.
	
	Findings
	On average, 88.6% (10.4%) of BrACs were submitted on time, without group differences (P = 0.18). The percentage of n-BrACs and LDA were greater with CM, and there was an interaction effect on drinking frequency and negative consequences, with decreases over time with CM (P = 0.00; effect sizes d = 0.52--0.62).
	
	Conclusion
	Cellphone technology may be useful for extending contingency management to treatment for alcohol problems.
	},
	Author = {Alessi, Sheila M. and Petry, Nancy M.},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1111/add.12093},
	Date-Added = {2015-01-17 22:56:01 +0000},
	Date-Modified = {2015-01-17 22:56:01 +0000},
	Doi = {10.1111/add.12093},
	Issn = {1360-0443},
	Journal = {Addiction},
	Keywords = {Abstinence, alcohol, cellphone, contingency management, mobile, reinforcement},
	Number = {5},
	Pages = {900--909},
	Tempurl = {http://dx.doi.org/10.1111/add.12093},
	Title = {A randomized study of cellphone technology to reinforce alcohol abstinence in the natural environment},
	Volume = {108},
	Year = {2013},
	Bdsk-Url-1 = {http://dx.doi.org/10.1111/add.12093}}

@article{Cucciare2012,
	Abstract = {Issues: By allowing for the efficient delivery of instructional content and the secure collection of self-report data regarding substance use and related problems, the Internet has tremendous potential to improve the effectiveness and accessibility of Substance Use Disorder (SUD) treatment
	and recovery-oriented services. <br></br> Approach: This article discusses some of the ways in which Internet and mobile technology can facilitate, complement and support the process of traditional clinician-delivered treatment for individuals with SUDs. <br></br> Key Findings: Internet applications
	are being used to support a range of activities including (a) the assessment and feedback process that constitutes a key feature of brief motivational interventions; and (b) the concurrent monitoring of patients who are receiving treatment for SUDs, to support continuing care, and the ongoing
	recovery of SUD patients who have completed face-to-face treatment. Internet technology is also being used to (c) support efficient delivery of clinical training in evidence-based practices for treating individuals who may have SUDs. <br></br> Implications: This emerging body of literature suggests
	that SUD treatment providers and program administrators can enhance the quality of clinician-delivered treatment by incorporating internet applications into existing processes of care and recovery oriented services. <br></br> Conclusion: Internet applications provide an unparalleled opportunity
	to engage patients in the treatment process, incorporate real-time data into treatment planning, prevent relapse, and promote evidence-based treatment approaches.},
	Author = {A. Cucciare, Michael and R. Weingardt, Kenneth and J. Greene, Carolyn and Hoffman, Julia},
	Bdsk-Tempurl-1 = {http://www.ingentaconnect.com/content/ben/cdar/2012/00000005/00000003/art00002},
	Date-Added = {2015-01-17 22:56:01 +0000},
	Date-Modified = {2015-01-17 22:56:01 +0000},
	Journal = {Current Drug Abuse Reviews},
	Keyword = {Internet technology, SUD treatment, brief alcohol intervention, brief motivational interventions, e-Mental Health, e-Therapy, evidence-based mental health practices, mental health recovery, mobile technology, substance use disorders},
	Number = {3},
	Pages = {172-177},
	Tempurl = {http://www.ingentaconnect.com/content/ben/cdar/2012/00000005/00000003/art00002},
	Title = {Current Trends in Using Internet and Mobile Technology to Support the Treatment of Substance Use Disorders},
	Volume = {5},
	Year = {2012}}

@article{Boyer2012,
	Author = {Boyer, EdwardW. and Fletcher, Rich and Fay, RichardJ. and Smelson, David and Ziedonis, Douglas and Picard, RosalindW.},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1007/s13181-011-0200-4},
	Date-Added = {2015-01-17 22:56:01 +0000},
	Date-Modified = {2015-01-17 22:56:01 +0000},
	Doi = {10.1007/s13181-011-0200-4},
	Issn = {1556-9039},
	Journal = {Journal of Medical Toxicology},
	Language = {English},
	Number = {1},
	Pages = {5-9},
	Publisher = {Springer-Verlag},
	Tempurl = {http://dx.doi.org/10.1007/s13181-011-0200-4},
	Title = {Preliminary Efforts Directed Toward the Detection of Craving of Illicit Substances: The iHeal Project},
	Volume = {8},
	Year = {2012},
	Bdsk-Url-1 = {http://dx.doi.org/10.1007/s13181-011-0200-4}}

@article{Marsch2012,
	Abstract = { Technology such as the Internet and mobile phones offers considerable promise for affecting the assessment, prevention, and treatment of and recovery from substance use disorders. Technology may enable entirely new models of behavioral health care within and outside of formal systems of care. This article reviews the promise of technology-based therapeutic tools for affecting the quality and reach of addiction treatment and recovery support systems, as well as the empirical support to date for this approach. Potential models for implementing technology-based interventions targeting substance use disorders are described. Opportunities to optimize the effectiveness and impact of technology-based interventions targeting addiction and recovery, along with outstanding research needs, are discussed. },
	Author = {Marsch, Lisa A.},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1080/10550887.2012.694606},
	Date-Added = {2015-01-17 22:56:01 +0000},
	Date-Modified = {2015-01-17 22:56:01 +0000},
	Doi = {10.1080/10550887.2012.694606},
	Eprint = {http://dx.doi.org/10.1080/10550887.2012.694606},
	Journal = {Journal of Addictive Diseases},
	Note = {PMID: 22873192},
	Number = {3},
	Pages = {313-318},
	Tempurl = {http://dx.doi.org/10.1080/10550887.2012.694606},
	Title = {Leveraging Technology to Enhance Addiction Treatment and Recovery},
	Volume = {31},
	Year = {2012},
	Bdsk-Url-1 = {http://dx.doi.org/10.1080/10550887.2012.694606}}

@article{KaplanStone2013,
	Author = {Kaplan, Robert M. and Stone, Arthur A.},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1146/annurev-psych-113011-143736},
	Date-Added = {2015-01-17 19:58:24 +0000},
	Date-Modified = {2015-01-17 19:58:37 +0000},
	Doi = {10.1146/annurev-psych-113011-143736},
	Eprint = {http://dx.doi.org/10.1146/annurev-psych-113011-143736},
	Journal = {Annual Review of Psychology},
	Note = {PMID: 22994919},
	Number = {1},
	Pages = {471-498},
	Tempurl = {http://dx.doi.org/10.1146/annurev-psych-113011-143736},
	Title = {Bringing the Laboratory and Clinic to the Community: Mobile Technologies for Health Promotion and Disease Prevention},
	Volume = {64},
	Year = {2013},
	Bdsk-Url-1 = {http://dx.doi.org/10.1146/annurev-psych-113011-143736}}

@article{Lewis2013,
	Author = {Lewis, Megan A and Uhrig, Jennifer D and Bann, Carla M and Harris, Jennie L and Furberg, Robert D and Coomes, Curtis and Kuhns, Lisa M},
	Bdsk-Tempurl-1 = {http://dx.doi.org/10.1037/a0028109},
	Date-Added = {2015-01-17 19:54:57 +0000},
	Date-Modified = {2015-01-17 19:55:19 +0000},
	Doi = {10.1037/a0028109},
	Issn = {0278-6133},
	Journal = {Health psychology : official journal of the Division of Health Psychology, American Psychological Association},
	Month = {March},
	Number = {3},
	Pages = {248---253},
	Tempurl = {http://dx.doi.org/10.1037/a0028109},
	Title = {Tailored text messaging intervention for HIV adherence: a proof-of-concept study},
	Volume = {32},
	Year = {2013},
	Bdsk-Url-1 = {http://dx.doi.org/10.1037/a0028109}}



@article{Brumback2003,
	AUTHOR = {Brumback, B. and Greenland, S. and Redman, M. and Kiviat, N. and Diehr, P.},
	TITLE = {The Intensity-Score Approach to Adjusting for Confounding},
	JOURNAL = {Biometrics},
	YEAR = {2003},
	VOLUME = {59},
	PAGES  = {274--285},
}

@article{Goetgeluk2008,
	AUTHOR = {Goetgeluk, S. and Vansteelandt, S.},
	TITLE = {Conditional Generalized Estimating Equations for the Analysis of Clusetered and Longitudinal Data},
	JOURNAL = {Biometrics},
	YEAR = {2008},
	VOLUME = {64},
	PAGES  = {772--780},
}

@article{Bojinov2017,
	AUTHOR = {Bojinov, I. and Shephard, N.},
	TITLE = {Time series experiments and causal estimands: exact randomization tests and trading},
	JOURNAL = {Journal of the American Statistical Association (Forthcoming)},
	YEAR = 2018,
}

@incollection{Tate2015,
	author = {Tate, R. L. and Perdices, M.},
	title = {N-of-1 Trials in the Behavioral Sciences.},
	booktitle = {The essential guide to N-of-1 trials in health.},
	editor = {Nikles, J. and Mitchell G.},
	pages = {19--41},
	publisher = {Springer},
	address = {Netherlands},
	year = {2015},
}

@article{Guyatt1990,
	author = {Guyatt, G.H. and Keller, J.L. and Jaeschke, R. and Rosenbloom, D. and Adachi, J.D. and Newhouse, M.T.},
	title = {The n-of-1 randomized controlled trial: clinical usefulness: our three-year experience.},
	journal = {Annals of Internal Medicine},
	volume = {112},
	pages = {293--299},
	year = {1990},
}

@article{Scuffham2010,
	author = {Scuffha, P.A. and Nikles, J. and Mitchell, G.K. and Yelland, M.J. and Vine, N. and Poulos, C.J. and
	Pillans, P. and Bashford, G. and del Mar, C. and Schluter, P. and Glasziou, P.},
	title = {Using N-of-1 trials to improve patient management and save costs.},
	journal = {Journal of general internal medicine.},
	volume = {9},
	pages = {906--913},
	year = {2010},
}

@article{Zucker1997,
	author = {Zucker, D.R. and Schmid, C.H. and McIntosh, M.W. and D'Agostino, R.B. and Selker, H.P. and Lau, J.},
	title = {Combining single patient (N-of-1) trials to estimate population treatment effects and to evaluate individual patient responses to treatment.},
	journal = {Journal of clinical epidemiology},
	volume = {50},
	pages = {401--410},
	year = {1997},
}

@article{Zucker2010,
	author = {Zucker, D.R. and Ruthazer, R. and Schmid, C.H.},
	title = {Individual (N-of-1) trials can be combined to give population comparative treatment effect estimates: methodologic considerations.},
	journal = {Journal of clinical epidemiology},
	volume = {63},
	pages = {1312--1323},
	year = {2010},
}

@article{Nikles2011,
	author = {Nikles, J. and Mitchell, G.K. and Schluter, P. and Good, P. and Hardy, J. and Rowett, D. and Shelby-James, T. and Vohra, S. and Currow, D.},
	title = {Aggregating single patient (n-of-1) trials in populations where recruitment and retention was difficult: the case of palliative care.},
	journal = {Journal of clinical epidemiology},
	volume = {64},
	pages = {471--480},
	year = {2011},
}

@article{Liang1986,
	AUTHOR = {Liang, KY and Zeger, SL},
	TITLE = {Longitudinal data analysis using generalized linear models},
	JOURNAL = {Biometrika},
	YEAR = {1986},
	VOLUME = {73},
	number = {1},
	PAGES = {13--22}
}

@misc{Autosense,
	title = {Autosense: A wireless sensor system to quantify personal
	exposures to psychosocial stress and addictive substances in
	natural environments.},
	year = {2017}, 
	howpublished = {\url{http://sites.google.com/site/autosenseproject}},
}

@inproceedings{Autosense:Ertin:2011,
	author = {Ertin, E. and Stohs, N. and Kumar, S. and Raij, A. and al'Absi, M. and Shah, S.},
	title = {AutoSense: Unobtrusively Wearable Sensor Suite for Inferring the Onset, Causality, and Consequences of Stress in the Field},
	booktitle = {Proceedings of the 9th ACM Conference on Embedded Networked Sensor Systems},
	year = {2011},
	pages = {274--287},
	numpages = {14},
	address = {New York, NY, USA},
} 


@article{Sankaran63,
	AUTHOR = {Sankaran , M.},
	TITLE = {Approximations to the non-central chi-squared distribution},
	JOURNAL = {Biometrika},
	YEAR = 1963,
	VOLUME = 50,
	PAGES = {199--204},
}

@article{Pearl2009,
	AUTHOR = {Pearl, J.}, 
	TITLE = {Causal inference in statistics: An overview},
	JOURNAL = {Statistics Surveys},
	VOLUME = {3},
	PAGES = {96--146}, 
	YEAR = {2009},
}

@book{Imbens2015,
	author = {Imbens, G.W. and Rubin, D.B.},
	title = {Causal Inference for Statistics, Social, and Biomedical Sciences: An Introduction},
	year = {2015},
	publisher = {Cambridge University Press},
	address = {New York, NY, USA},
} 

@article{Vanderweele2013,
	AUTHOR = {Vanderweele, T. J. and Hong, G. and Jones, S.M. and Brown, J.L.},
	TITLE = {Mediation and spillover effects in group-randomized
	trials: A case study of the 4rs educational intervention.},
	JOURNAL = {Journal of the American Statistical Association},
	NUMBER = {502},
	VOLUME = {108},
	PAGES = {469--482},
	YEAR = {2013}
}

@article{Hong2006,
	AUTHOR = {Hong, G. and Raudenbush, S. W.},
	TITLE = {Evaluating kindergarten retention policy},
	JOURNAL = {Journal of the American Statistical Association},
	NUMBER = {475},
	VOLUME = {101},
	PAGES = {901--910},
	YEAR = {2006}
}

@article{BushLipari2015,
	AUTHOR = {Bush, D. and Lipari, R.},
	TITLE = {Substance use and and substance use disorder by industry},
	JOURNAL = {National Survey on Drug Use and Health},
	YEAR = {2015}
}

@article{Mancl2001,
	AUTHOR = {LA. Mancl and T.A. DeRouen},
	TITLE = {A covariance estimator for {GEE} with improved small-sample properties.},
	JOURNAL = {Biometrics},
	VOLUME = {57},
	NUMBER = {1},
	PAGES = {126--134},
	YEAR = {2001}
}

@article{Mancl1996,
	AUTHOR = {Mancl, L.A. and Leroux, B.G.},
	TITLE = {Efficiency of Regression Estimates for Clustered data},
	JOURNAL = {Biometrics},
	VOLUME = {52},
	PAGES = {500--511},
	YEAR = {1996}
}

@Article{Dallery2013,
	author="Dallery, J.
	and Cassidy, N. R.
	and Raiff, R. B.",
	title="Single-Case Experimental Designs to Evaluate Novel Technology-Based Health Interventions",
	journal="J Med Internet Res",
	year="2013",
	volume="15",
	number="2",
	pages="e22",
	url="http://www.jmir.org/2013/2/e22/",
}


@article{MarschDallery2012,
	AUTHOR = {Marsch, L. and Dallery, J.},
	TITLE = {Advances in the psychosocial treatment of addiction: the role of technology in the delivery of evidence-based psychosocial treatment.},
	JOURNAL = {Psychiatr Clin North Am.},
	VOLUME = {35},
	NUMBER = {2},
	PAGES = {481--493},
	YEAR = {2012},
}

@article{Hotelling1931,
	AUTHOR = {Hotelling, H.},
	TITLE = {The generalization of Student's ratio.},
	JOURNAL = {Annals of Mathematical Sciences},
	VOLUME = {2},
	NUMBER = {3},
	PAGES = {360--378},
	YEAR = {1931},
}

@article{psych1,
	AUTHOR={Geiser, Christian  and  Bishop, Jacob  and  Lockhart, Ginger  and  Shiffman, Saul  and  Grenard, Jerry L.},   
	TITLE={Analyzing Latent State-Trait and Multiple-Indicator Latent Growth Curve Models as Multilevel Structural Equation Models},      
	JOURNAL={Frontiers in Psychology},      
	VOLUME={4},      
	YEAR={2013},      
	NUMBER={975},     
}

@article{psych2,
	author = {Steyer, Rolf and Schmitt, Manfred and Eid, Michael},
	title = {Latent state–trait theory and research in personality and individual differences},
	journal = {European Journal of Personality},
	volume = 13,
	number = 5,
	publisher = {John Wiley & Sons, Ltd.},
	pages = {389--408},
	year = 1999,
}


@article{psych3,
	title = {Analyzing person, situation and person × situation interaction effects: Latent state-trait models for the combination of random and fixed situations.},
	author = {Geiser, Christian and Litson, Kaylee and Bishop, Jacoba and Keller, Brian T. and Burns, G. Leonard and Servera, Mateu and Shiffman, Saul},
	journal = {Psychological Methods},
	volume  = 20,
	number = 2,
	year = 2015,
	pages = {165--192},
}


@article{BenZeev2013,
	AUTHOR = {D. Ben-Zeev and S.M. Kaiser and C.J. Brenner and M. Begale and J. Duffecy and D.C. Mohr},
	TITLE = {Development and usability testing of FOCUS: a smartphone system for self-management of schizophrenia.},
	JOURNAL = {Psychiatr Rehabil J.},
	VOLUME = {36},
	NUMBER = {4},
	PAGES = {289--296},
	YEAR = {2013},
}

@article{Marsch2012,
	AUTHOR = {L. Marsch},
	TITLE = {Leveraging technology to enhance addiction treatment and recovery.},
	JOURNAL = {J Addict Dis.},
	VOLUME = {31},
	NUMBER = {3},
	PAGES = {313--318},
	YEAR = {2012},
}

@article{SAMHSA2014,
	AUTHOR = {US Dept of Health and Human Services},
	TITLE = {Results from the 2014 National Survey on Drug Use and Health: Summary of National Findings},
	YEAR = {2014},
}

@article{Scottetal2011,
	AUTHOR = {Scott, C. and Dennis M. and Laudet A. and Funk R. and Simeone R.},
	TITLE = {Surviving drug addiction: the effect of treatment and abstinence on mortality},
	JOURNAL = {Am J Public Health.},
	VOLUME = {101},
	NUMBER = {4},
	PAGES = {737--744},
	YEAR = {2011},
}

@article{CDC2014,
	AUTHOR = {Centers for Disease Control and Prevention},
	TITLE = {Current Cigarette Smoking Among Adults--United States, 2005-2013},
	JOURNAL = {Morbidity and Mortality Weekly Report},
	VOLUME = {63},
	NUMBER = {47},
	PAGES = {1108-12},
	YEAR = {2013},
}

@article{KapSton2013,
	AUTHOR = {R. M. Kaplan and A. A. Stone},
	TITLE = {Bringing the laboratory and clinic to the community: mobile technologies for health promotion and disease prevention},
	JOURNAL = {Annual Review of Psychology},
	VOLUME = {64},
	PAGES = {471--498},
	YEAR = {2013},
}

@TECHREPORT{Makovsky2015,
	TITLE = {Fifth Annual Makovsky/Kelton ``Pulse of Online Health" Survey},
	MONTH = {January},
	YEAR = {2015},
}

@TECHREPORT{Pew2015,
	AUTHOR = {The Pew Research Center},
	TITLE = {US Smartphone Use in 2015},
	MONTH = {April},
	YEAR = {2015},
}

@online{FlurryAnalytics,
	author = {P. Farago},
	title = {App Engagement: The Matrix Reloaded},
	year = {2012},
	url = {http://flurrymobile.tumblr.com/post/113379517625/app-engagement-the-matrix-reloaded},
	urldate = {2012-10-22}
}

@online{md2k_website,
	author = {MD2K},
	title = {MD2K},
	year = 2015,
	url = {http://md2k.org},
	urldate = {2015-11-30},
}

@online{Localytics,
	author = {D. Hoch},
	title = {APP RETENTION IMPROVES - APPS USED ONLY ONCE DECLINES TO 20\%},
	year = {2014},
	url = {http://info.localytics.com/blog/app-retention-improves},
	urldate = {2014-06-11},
}

@article{Beckeretal2015,
	AUTHOR = {Becker S. and Brandl C. and Meister S. and Nagel E. and Miron-Shatz T. and Mitchell A.},
	TITLE = {Demographic and Health Related Data of Users of a Mobile Application to Support Drug Adherence is Associated with Usage Duration and Intensity},
	JOURNAL = {PLoS ONE},
	VOLUME = {10},
	NUMBER = {1},
	YEAR = {2015},
	DOI = {10.1371/journal.pone.0116980},
}


@book{LiggettBook,
	YEAR = {1985},
	PUBLISHER = {Springer},
	AUTHOR = {Liggett, T.M.},
	TITLE = {Interacting {P}article {S}ystems},
	SERIES  = {Grundlehren der mathematischen Wissenschaften},
	VOLUME = {276},
}

@book{McCullagh_GLM,
	YEAR = {1989},
	PUBLISHER = {Springer},
	AUTHOR = {McCullagh, P. and Nelder, J.A.},
	TITLE = {Generalized Linear Models},
	EDITION = {2nd},
}

@book{Kalbfleisch92,
	YEAR = {2002},
	PUBLISHER = {John Wiley \& Sons},
	AUTHOR = {J. Kalbfleisch and R. Prentice},
	TITLE = {The Statistical Analysis of Failure Time Data.},
}


@article{Kumaretal2013,
	AUTHOR = {S. Kumar and W. Nilsen and A. Abernethy
	and A. Atienza and K. Patrick and M. Pavel and 
	W. Riley and A. Shar and B. Spring and
	D. Spruijt-Metz and D. Hedeker and V. Honavar and 
	R. Kravitz and R. Lefebvre and D. Mohr and 
	S. Murphy and C. Quinn and V. Shusterman and D. Swendeman},
	TITLE = {Mobile Health Technology Evaluation: The mHealth Evidence Workshop},
	JOURNAL = {American Journal of Preventive Medicine},
	VOLUME = {45},
	NUMBER = {2},
	YEAR = {2013},
	PAGES = {228 -- 236}
}

@article{Kratochwill2010,
	AUTHOR = {Kratochwill, T.R. and Levin, J.R.},
	TITLE = {Enhancing the scientific credibility of single-case intervention research: randomization to the rescue.},
	JOURNAL = {Psychol Methods},
	YEAR = {2010},
	VOLUME = {15},
	NUMBER = {2},
	PAGES = {124-144},
}


@article{McClureetal2013,
	AUTHOR = {J.B. McClure and S. Shortreed and A. Bogart and H. Derry and K. Riggs and J. John and V. Nair and L. An},
	TITLE = {The effect of program design on engagement with an internet-based smoking intervention: randomized factorial trial},
	JOURNAL = {J. Med. Internet Res.},
	YEAR = {2013},
	VOLUME = {15},
	NUMBER = {3},
}

@article{klasnja2012,
	AUTHOR = {P. Klasnja and W. Pratt},
	TITLE = {Healthcare in the pocket: Mapping the space of mobile-phone health interventions},
	JOURNAL = {Journal of Biomedical Informatics},
	VOLUME = {45},
	PAGES = {184 -- 198}
}

@article{Franklinetal2008,
	AUTHOR = {VL. Franklin and A. Green and A. Waller and SA. Greene and C. Pagliari and W. Pratt},
	TITLE = {Patients' engagement with "Sweet Talk": a text messaging support system for young people with diabetes},
	JOURNAL = {Journal of Medical Internet Research},
	VOLUME = {10},
	NUMBER = {2}
}

@misc{mrt,
	author = {{The Methodology Center}},
	title = {{Micro-Randomized Trials (MRTs)}},
	howpublished = {{\tt https://www.methodology.psu.edu/ra/adap-inter/mrt-projects} (Jun. 2019)}
}


@article{kizakevich2014personal,
	title={A personal health information toolkit for health intervention research},
	author={Kizakevich, Paul N and Eckhoff, Randall and Weger, Stacey and Weeks, Adam and Brown, Janice and Bryant, Stephanie and Bakalov, Vesselina and Zhang, Yuying and Lyden, Jennifer and Spira, James},
	journal={Stud Health Technol Inform},
	volume={199},
	pages={35--39},
	year={2014}
}


@article{klasnja2015microrandomized,
	title={Micro-randomized trials: An experimental design for developing just-in-time adaptive interventions.},
	author={Klasnja, P. and Hekler, E.B. and Shiffman, S. and Boruvka, A. and Almirall, D. and Tewari, A. and Murphy, S.A.},
	journal={Health Psychology},
	volume={34},
	number={S},
	pages={1220},
	year={2015},
	publisher={American Psychological Association}
}

@article{KlasnjaAnnals,
	title={Efficacy of Contextually Tailored Suggestions for Physical Activity: A Micro-randomized Optimization Trial of HeartSteps},
	author={Klasnja, Predrag and Smith, Shawna and Seewald, Nicholas J and Lee, Andy and Hall, Kelly and Luers, Brook and Hekler, Eric B and Murphy, Susan A},
	journal={Annals of Behavioral Medicine},
	year={2018}
}
@inproceedings{Klasnja2008,
	author = {Klasnja, P. and Harrison, B. L. and LeGrand, L. and LaMarca, A. and Froehlich, J. and Hudson, S.E.},
	title = {Using Wearable Sensors and Real Time Inference to Understand Human Recall of Routine Activities},
	booktitle = {Proceedings of the 10th International Conference on Ubiquitous Computing},
	series = {UbiComp '08},
	year = {2008},
	pages = {154--163},
	numpages = {10},
}

@misc{wiki:KL,
	author = "Wikipedia",
	title = "{Kullback–Leibler divergence} --- {W}ikipedia{,} The Free Encyclopedia",
	year = "2018",
	howpublished = {\url{http://en.wikipedia.org/wiki/Kullback–Leibler_divergence}},
	note = "[Online; accessed 26-July-2018]"
}


@article{McDonaldetal,
	AUTHOR = {McDonald, S. and Quinn, F. and Vieira, R.
	and O'Brien, N. and White, M. and Johnston, D.W. and
	Sniehotta, F.F.},
	TITLE = {The state of the art and future opportunities for using longitudinal n-of-1 methods in health behaviour research: a systematic literature overview},
	JOURNAL = {Health Psychology Rev.},
	YEAR = {2017},
	VOLUME = {0},
	NUMBEr = {0},
	PAGES = {1--17},
	eprint = {http://dx.doi.org/10.1080/17437199.2017.1316672},
}

@unpublished{Boruvkaetal,
	AUTHOR = {A. Boruvka and D. Almirall and K. Witkiewitz and S.A. Murphy},
	YEAR = {2017},
	TITLE = {Assessing Time-Varying Causal Effect Moderation in Mobile Health},
	note = "To appear in the {\it Journal of the American Statistical Association}",
}



@article{Muessigetal2013,
	AUTHOR = {E.K. Muessig and C.E. Pike and S. LeGrand and B.L. Hightow-Weidman},
	TITLE = {Mobile phone applications for the care and prevention
	of {HIV} and other sexually transmitted diseases: A review.},
	JOURNAL = {Journal of Medical Internet Research},
	YEAR = 2013,
	VOLUME = 15,
	NUMBER = 1,
	PAGES = {103-124},
}




@article{Rizoetal2009,
	AUTHOR = {D. Rizopoulos and G. Verbeke and E. Lesaffre},
	TITLE = {Fully exponential Laplace approximations for the 
	joint modelling of survival and longitudinal data},
	JOURNAL = {Journal of the Royal Statistical Society: Series B},
	VOLUME = {71},
	NUMBER = {3},
	PAGES = {637--654},
	YEAR = {2009},
}

@book{Rizopoulos2012,
	YEAR = {2012},
	PUBLISHER = {CRC Press},
	AUTHOR = {D. Rizopoulos},
	TITLE = {Joint models for longitudinal and time-to-event data: With applications in R},
}

@article{ProustLimaetal2014,
	AUTHOR = {C. Proust-Lima and M. Sene and J.M. Taylor and H. Jacqmin-Gadda},
	TITLE = {Joint latent class models for longitudinal and time-to-event data: a review},
	JOURNAL = {Statistical Methods in Medical Research},
	VOLUME = {23},
	NUMBER = {1},
	PAGES = {74--90},
	YEAR = {2014},
}

@article{Salfneretal2010,
	AUTHOR = {F. Salfner and M. Lenk and M. Malek},
	TITLE = {A survey of online failure prediction methods},
	JOURNAL = {ACM Computing Surveys},
	VOLUME = {42},
	NUMBER = {3},
	YEAR = {2010},
}

@online{Brebissonetal2015,
	AUTHOR = {A. Brebisson and E. Simon and A. Auvolat and P. Vincent and Y. Bengio},
	TITLE = {Artificial Neural Networks Applied to Taxi Destination Prediction},
	eprinttype = {arxiv},
	eprint = {1508.00021},
	date = {2015-09-21},
	ee = {http://arxiv.org/abs/1508.00021},
}

@article{Ionides2015,
	AUTHOR = {E. L. Ionides and D. Nguyen and Y. Atchade and S. Stoev and A.A. King},
	TITLE = {Inference for dynamic and latent variable models via iterated, perturbed Bayes maps.},
	JOURNAL = {Proceedings of the National Academy of Sciences of the USA},
	VOLUME = {112},
	PAGES = {719--724},
	YEAR = {2015},
}

@article{Andrieuetal2010,
	AUTHOR = {C. Andrieu and A. Doucet and R. Holenstein},
	TITLE = {Particle Markov chain Monte Carlo methods},
	JOURNAL = {J.R. Statist. Soc. B},
	VOLUME = {72},
	PAGES = {269--342},
	YEAR = {2010},
}

@article{Nosyketal2010,
	AUTHOR = {B. Nosyk and H. Sun and D. Guh and E. Oviedo-Joekes and D.C. Marsh and S. Brissette and M.T. Schechter and A.H. Anis},
	TITLE = {An empirical comparison of eight health status measures for chronic opioid dependence.},
	JOURNAL = {J. Clin. Epidemiol.},
	VOLUME = {63},
	PAGES = {1132 -- 1144},
	YEAR = {2010},
}

@article{Litvinetal2013,
	AUTHOR = {EB Litvin and AM Abrantes and RA Brown},
	TITLE = {Computer and mobile technology-based interentions for substance use disorders: An organizing framework},
	JOURNAL = {Addictive Behaviors},
	VOLUME = {38},
	PAGES = {1747--1756},
	YEAR = {2013},
}

@article{Dempsey_Significance,
	AUTHOR = {W. Dempsey and P. Liao and P. Klasnja I. Nahun-Shani and S.A. Murphy},
	TITLE = {Randomised trials for the Fitbit generation},
	JOURNAL = {Significance},
	VOLUME = {12},
	NUMBER = {6},
	PAGES = {20--23},
	YEAR = {2015},
}

@unpublished{Dempsey_AOAS,
	AUTHOR = {Walter Dempsey, Peng Liao, Santosh Kumar, Susan A. Murphy},
	TITLE = {he stratified micro-randomized trial design: sample size considerations for testing nested causal effects of time-varying treatments},
	note = "To appear in Annals of Applied Statistics, 2020",
}

@misc{wiki:f1score,
	author = {Wikipedia},
	title = "F1 score --- {W}ikipedia{,} The Free Encyclopedia",
	year = "2017",
	url = "https://en.wikipedia.org/wiki/F1_score",
	note = "[Online; accessed 23-May-2017]"
}

@inproceedings{Hovsepian:2015,
	author = {Hovsepian, K. and al'Absi, M. and Ertin, E. and Kamarck, T. and Nakajima, M. and Kumar, S.},
	title = {cStress: Towards a Gold Standard for Continuous Stress Assessment in the Mobile Environment},
	booktitle = {Proceedings of the 2015 ACM International Joint Conference on Pervasive and Ubiquitous Computing},
	series = {UbiComp '15},
	year = {2015},
	pages = {493--504},
	publisher = {ACM},
	address = {New York, NY, USA},
	keywords = {mobile health (mHealth), modeling, stress, wearable sensors},
} 

@inproceedings{Mash2018,
	author = {Rabbi, M. and Philyaw-Kotov, M. and Lee, J. and Mansour, A. and Dent, L. and Wang, X. and Cunningham, R. and Bonar, E. and Nahum-Shani, I. and Klasnja, P. and Walton, M. and Murphy, S.A.},
	title = {SARA: A Mobile App to Engage Users in Health Data Collection},
	booktitle = {Proceedings of the 2017 ACM International Joint Conference on Pervasive and Ubiquitous Computing},
	series = {UbiComp '17},
	year = {2017},
	pages = {781--789},
	publisher = {ACM},
	address = {New York, NY, USA},
} 


@inproceedings{Sarker:2016,
	title = {Finding Significant Stress Episodes in a Discontinuous Time Series of Rapidly Varying Mobile Sensor Data},
	author = {Sarker, H. and Tyburski, M. and Rahman, M.M. and Hovsepian, K. and Sharmin, M. and Epstein, D.H. and Preston, K.L. and Furr-Holden, C.D. and Milam, A. and Nahum-Shani, I. and al'Absi, M. and Kumar, S.},
	year = {2016},
	booktitle = {Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems},
	pages = {4489-4501},
	publisher = {ACM},
	address = {Santa Clara, California, USA},
	series = {CHI '16},
	tppubtype = {inproceedings}
}

@article{Nahum2017,
	title = {Just-in-Time Adaptive Interventions (JITAIs) in Mobile Health: Key Components and Design Principles for Ongoing Health Behavior Support},
	author = {Nahum-Shani, I. and Smith, S.N. and Spring, B.J. and Collins, L.M. and Witkiewitz, K. and Tewari, A. and Murphy, S.A.},
	journal = {Ann Behav Med},
	year = {2017}
}

@inproceedings{Chatterjee:2016,
	author = {Chatterjee, Soujanya and Hovsepian, Karen and Sarker, Hillol and Saleheen, Nazir and al'Absi, Mustafa and Atluri, Gowtham and Ertin, Emre and Lam, Cho and Lemieux, Andrine and Nakajima, Motohiro and Spring, Bonnie and Wetter, David W. and Kumar, Santosh},
	title = {mCrave: Continuous Estimation of Craving During Smoking Cessation},
	booktitle = {Proceedings of the 2016 ACM International Joint Conference on Pervasive and Ubiquitous Computing},
	series = {UbiComp '16},
	year = {2016},
	pages = {863--874},
	numpages = {12},
	url = {http://doi.acm.org/10.1145/2971648.2971672},
	publisher = {ACM},
	address = {New York, NY, USA},
}

@inproceedings{Hossain2017mcerebrum,
	title={mCerebrum: An mHealth Software Platform for Development and Validation of Digital Biomarkers and Interventions},
	author={Hossain, Syed Monowar and Hnat, Timothy and Saleheen, Nazir and Nasrin, Nusrat Jahan and Noor, Joseph and Ho, Bo-Jhang and Condie, Tyson and Srivastava, Mani and Kumar, Santosh},
	booktitle={The ACM Conference on Embedded Networked Sensor Systems (SenSys)},
	year={2017},
	organization={ACM}
}

@incollection{Sarker:book:2017,
	author      = {Sarker, H. and Hovsepian, K. and Chatterjee, S. and Nahum-Shani, I. and Murphy, S.A.
	and Spring, B. and Ertin, E. and al'Absi, M. and Nakajima, M. and Kumar, S.},
	title       = {From Markers to Interventions: The Case of Just-in-Time Stress Intervention},
	editor      = {Regh, J.M. and Murphy, S.A. and Kumar, S.},
	booktitle   = {Mobile Health Sensors, Analytic Methods, and Applications},
	publisher   = {Springer International Publishing},
	year        = 2017,
}

@inproceedings{Saleheen:2015,
	author = {Saleheen, N. and Ali, A.A. and Hossain, S.M. and Sarker, H. and Chatterjee, S. and Marlin, B.
	and Ertin, E. and al'Absi, M. and Kumar, S.},
	title = {puffMarker: A Multi-sensor Approach for Pinpointing the Timing of First Lapse in Smoking Cessation},
	booktitle = {Proceedings of the 2015 ACM International Joint Conference on Pervasive and Ubiquitous Computing},
	series = {UbiComp '15},
	year = {2015},
	pages = {999--1010},
	numpages = {12},
	url = {http://doi.acm.org/10.1145/2750858.2806897},
	publisher = {ACM},
	address = {New York, NY, USA},
} 



@article{Rubin,
	AUTHOR = {DB. Rubin},
	TITLE = {Bayesian inference for causal effects: The role of randomization},
	JOURNAL = {The Annals of Statistics},
	VOLUME = {6},
	NUMBER = {1},
	PAGES = {34--58},
	YEAR = {1978},
}

@article{Robins,
	AUTHOR = {J. Robins},
	TITLE = {A new approach to causal inference in mortality studies with 
	a sustained exposure period-application to control of the healthy worker survivor effect},
	JOURNAL = {Mathematical Modelling},
	VOLUME = {7},
	NUMBER = {9},
	PAGES = {1393--1512},
	YEAR = {1986},
}

@unpublished{Dempsey_MSP,
	AUTHOR = {W. Dempsey and P. McCullagh},
	TITLE = {Weak continuity of predictive distribution for Markov survival processes},
	note = "Submitted",
}

@unpublished{Dempsey_StatSci,
	AUTHOR = {W. Dempsey and P. McCullagh},
	TITLE = {The Pilgrim's Process},
	note = "Submitted",
}

@unpublished{DempseyPMCC,
	AUTHOR = {W. Dempsey and P. McCullagh},
	TITLE = {Survival models and health sequences},
	note = "Submitted",
}

@unpublished{DempseyPMCC2,
	AUTHOR = {W. Dempsey and P. McCullagh},
	TITLE = {Vital variables and survival processes},
	note = "Submitted",
}

@article{HeronSmyth2010,
	AUTHOR = {K.E. Heron and J.M. Smyth},
	TITLE = {Ecological momentary interventions: incorporating mobile technology into psychosocial and health behaviour treatments},
	JOURNAL = {Br. J. Health Psychol.},
	VOLUME = {15},
	PAGES = {1--39},
}

@article{Doksum74,
	AUTHOR = {K.A. Doksum},
	TITLE = {Tailfree and neutral random probabilities and their posterior distributions},
	JOURNAL = {Annals of Probability},
	VOLUME = {2},
	PAGES = {183--201},
	YEAR = {1974},
}

@article{Hjort90,
	AUTHOR = {N.L. Hjort},
	TITLE = {Nonparametric Bayes estimators based on beta processes in models for life history data},
	JOURNAL = {Annals of Statistics},
	VOLUME = {18},
	PAGES = {1259--1294},
	YEAR = {1990},
}

@incollection{James07,
	AUTHOR = {L.F. James},
	YEAR = {2007},
	TITLE = {Neutral-to-the-right species sampling mixture models},
	booktitle = {Advances in Statistical Modeling and Inference. Essays in honor of K. Doksum},
}

@article{Diggle2008,
	AUTHOR = {P.J. Diggle and I. Sousa and A. Chetwynd},
	YEAR = {2008},
	TITLE = {Joint modeling of repeated measurements and tome-to-event outcomes: The fourth Armitage lecture},
	JOURNAL = {Statistics in Medicine},
	VOLUME = {27},
	PAGES = {2981--2998},
}

@article{Henderson2000,
	AUTHOR = {R. Henderson and P. Diggle and A. Dobson},
	YEAR = {2000},
	TITLE = {Joint modeling of longitudinal measurements and event time data},
	JOURNAL = {Biostatistics},
	VOLUME = {1},
	PAGES = {465-480},
}

@article{Kurlandetal2009,
	AUTHOR = {B.F. Kurland AND L.L. Johnson AND B.L. Egleston and P.H. DiehR},
	YEAR = {2009},
	TITLE = {Longitudinal data with follow-up truncated by
	death: match the analysis method to the research aims},
	JOURNAL = {Statistical Science},
	VOLUME = {24},
	PAGES = {211--222},
}

@article{Andersen1991,
	AUTHOR = {P.K. Andersen and L.H. Hansen and N. Keiding},
	TITLE = {Assessing the influence of reversible disease indicators on survival},
	JOURNAL = {Statistics in Medicine},
	VOLUME = {10}, 
	NUMBER = {},
	PAGES = {1061--1067},
	YEAR = {1991},
}

@article{DeGruttola1994,
	AUTHOR = {V. DeGruttola and X.M. Xu},
	TITLE = {Modeling progression of CD4 lymphocyte count and its relation to survival time},
	JOURNAL = {Biometrics},
	VOLUME = {50}, 
	NUMBER = {},
	PAGES = {1003 -- 1014},
	YEAR = {1994},
}

@article{Fieuws2008,
	AUTHOR = {S. Fieuws and G. Verbeke and Y. Vanrenterghem},
	TITLE = {Predicting renal graft failure using multivariate longitudinal profiles.},
	JOURNAL = {Biostatistics},
	VOLUME = {9}, 
	NUMBER = {},
	PAGES = {419--431},
	YEAR = {2008},
}

@article{Sweeting2011,
	AUTHOR = {M.J. Sweeting and S.G. Thompson},
	TITLE = {Joint modeling of longitudinal and time-to-event data with application to predicting abdominal aortic aneurysm growth and rupture},
	JOURNAL = {Biometrical Journal},
	VOLUME = {53}, 
	NUMBER = {},
	PAGES = {750--763},
	YEAR = {2011},
}

@article{Tsiatis1995,
	AUTHOR = {A.A. Tsiatis and V. DeGruttola and M.S. Wulfsohn},
	TITLE = {Modeling the relationship of survival to longitudinal data measured with error: applications to survival and CD4 counts in patients with AIDS},
	JOURNAL = {J. Amer. Statist. Assoc.},
	VOLUME = {90}, 
	NUMBER = {},
	PAGES = {27--37},
	YEAR = {1995},
}

@article{Diggle2007,
	AUTHOR = {P.J. Diggle and D. Farewell and R. Henderson},
	TITLE = {Analysis of longitudinal data with drop-out: objectives, assumptions, and a proposal (with discussion)},
	JOURNAL = {Applied Statistics},
	VOLUME = {56}, 
	NUMBER = {},
	PAGES = {499--550},
	YEAR = {2007},
}

@article{Liaoetal2015,
	AUTHOR = {P. Liao and P. Klasjna and A. Tewari and S.A. Murphy},
	TITLE = {Micro-Randomized Trials in mHealth},
	JOURNAL = {Statistics in Medicine},
	VOLUME = {35}, 
	NUMBER = {12},
	PAGES = {1944-71},
	YEAR = {2016},
}

@article{Andersen1993,
	AUTHOR = {P.K. Andersen and O. Borgan and R.D. Gill and N. Keiding},
	TITLE = {Statistical Models Based on Counting Processes},
	PUBLISHER = {Springer},
	VOLUME = {}, 
	EDITION = {},
	YEAR = {1993},
}

@article{Breslow1974,
	AUTHOR = {N. Breslow},
	TITLE = {Covariance analysis of censored survival data},
	JOURNAL = {Biometrics},
	VOLUME = {30}, 
	NUMBER = {},
	PAGES = {89--99},
	YEAR = {1974},
}

@article{Cox1972,
	AUTHOR = {D.R. Cox},
	TITLE = {Regression models and life tables (with discussion)},
	JOURNAL = {J. Roy. Statist. Soc. B.},
	VOLUME = {39}, 
	NUMBER = {},
	PAGES = {264--296},
	YEAR = {1972},
}

@book{Kalbfleisch2002,
	AUTHOR = {J.D. Kalbfleisch and R. Prentice},
	TITLE = {Statistical analysis of failure time data},
	publisher = {Wiley},
	address = {New York},
	Edition = {2}, 
	YEAR = {2002},
}

@article{KaplanMeier1958,
	AUTHOR = {E.L. Kaplan and P. Meier},
	TITLE = {Nonparametric estimation from incomplete observations},
	JOURNAL = {J. Amer. Statist. Assoc.},
	VOLUME = {53}, 
	NUMBER = {},
	PAGES = {457--481},
	YEAR = {1958},
}

@article{Ngiametal2011,
	AUTHOR = {J. Ngiam and A. Khosia and M. Kim and J. Nam and H. Lee and A. Ng},
	TITLE = {Multimodal deep learning},
	JOURNAL = {In Proceedings of the 29th International Conference on Machine Learning},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {2011},
}

@article{Sohnetal2014,
	AUTHOR = {K. Sohn and W. Shang and H. Lee},
	TITLE = {Improved Multimodal Deep Learning with Variation of Information},
	JOURNAL = {In Advances in Neural Information Processing Systems (NIPS)},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {2014},
}

@article{Sohnetal2015,
	AUTHOR = {K. Sohn, X. Yan, H. Lee},
	TITLE = {Learning Structured Output Representation using Deep Conditional Generative Models},
	JOURNAL = {In Advances in Neural Information Processing Systems (NIPS)},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {2015},
}


@article{Rehgvision,
	AUTHOR = {A. Humayun and F. Li and J. M. Reh},
	TITLE = {The Middle Child Problem: Revisiting Parametric Min-cut and Seeds for Object Proposals},
	JOURNAL = {In Proceedings of the International Conference on Computer Vision (ICCV) 2015},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {},
}

@article{Leevision,
	AUTHOR = {Y. Zhang and K. Sohn and R. Villegas and G. Pan and H. Lee},
	TITLE = {Improving Object Detection with Deep Convolutional Networks via Bayesian Optimization and
	Structured Prediction},
	JOURNAL = {In Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {2015},
}

@inproceedings{jin2021pessimism,
	title={Is Pessimism Provably Efficient for Offline RL?},
	author={Jin, Ying and Yang, Zhuoran and Wang, Zhaoran},
	booktitle={International Conference on Machine Learning},
	pages={5084--5096},
	year={2021},
	organization={PMLR}
}

@InProceedings{pmlr-v139-shi21d,
	title = 	 {Deeply-Debiased Off-Policy Interval Estimation},
	author =       {Shi, Chengchun and Wan, Runzhe and Chernozhukov, Victor and Song, Rui},
	booktitle = 	 {Proceedings of the 38th International Conference on Machine Learning},
	pages = 	 {9580--9591},
	year = 	 {2021},
	volume = 	 {139},
	series = 	 {Proceedings of Machine Learning Research},
	publisher =    {PMLR},
}


@article{deeplearningvision,
	AUTHOR = {K. Sohn and D. Jung and H. Lee and A. Hero},
	TITLE = {Efficient Learning of Sparse, Distributed, Convolutional Feature Representations for Object Recognition},
	JOURNAL = {In Proceedings of 13th International Conference on Computer Vision (ICCV)},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {2011},
}

@article{McGovern2011,
	AUTHOR = {A. McGovern and D. Rosendahl and R. Brown and K. Droegemeier},
	TITLE = {Identifying predictive multidimensional time series motifs: an application to severe weather prediction},
	JOURNAL = {Data Min Knowl Disc},
	VOLUME = {22}, 
	NUMBER = {},
	PAGES = {232--258},
	YEAR = {2011},
}

@article{Li2007,
	AUTHOR = {Z. Li and S. Zhou and S. Choubey and C. Sievenpiper},
	TITLE = {Failure event prediction using the Cox proportional hazard model driven by frequent failure signatures},
	JOURNAL = {IIE Transactions},
	VOLUME = {39}, 
	NUMBER = {3},
	PAGES = {303--315},
	YEAR = {2007},
}

@article{Nilsen2012,
	AUTHOR = {W. Nilsen and S. Kumar and A. Shar and C. Varoquiers and T. Wiley and WT. Riley
	and M. Pavel and AA. Atienza},
	TITLE = {Advancing the science of mHealth},
	JOURNAL = {Journal of Health Communication},
	VOLUME = {17}, 
	number = {sup1},
	PAGES = {5--10},
	YEAR = {2012},
}

@article{Faucett1996,
	AUTHOR = {C.L. Faucett and D.C. Thomas},
	TITLE = {Simultaneously modeling censored survival data and repeatedly measured
	covariates: a Gibbs sampling approach},
	JOURNAL = {Statistics in Medicine},
	VOLUME = {15}, 
	NUMBER = {},
	PAGES = {1663--1685},
	YEAR = {1996},
}

@article{Guo2004,
	AUTHOR = {X. Guo and B. Carlin},
	TITLE = {Separate and joint modeling of longitudinal and event time data using standard computer packages.},
	JOURNAL = {American Statistician},
	VOLUME = {58}, 
	NUMBER = {},
	PAGES = {1--10},
	YEAR = {2004},
}

@article{ProustLimaetal2013,
	AUTHOR = {C. Proust-Lima and H Amieva and H. Jacqmin-Gadda},
	TITLE = {Analysis of multivariate mixed longitudinal data: flexible latent process approach},
	JOURNAL = {The British Journal of Mathematical and Statistical Psychology},
	VOLUME = {66}, 
	NUMBER = {},
	PAGES = {470--487},
	YEAR = {2013},
}

@article{Tsiatis2004,
	AUTHOR = {A.A. Tsiatis and M. Davidian},
	TITLE = {Joint modeling of longitudinal and time-to-event data: an overview},
	JOURNAL = {Statistica Sinica},
	VOLUME = {14}, 
	NUMBER = {},
	PAGES = {809--834},
	YEAR = {2004},
}

@book{vanHouwelingen2012, 
	AUTHOR = {H.C. van Houwelingen and H. Putter},
	TITLE = {Dynamic Prediction in Clinical Survival Analysis},
	publisher = {CRC Press},
	YEAR = {2012},
}

@article{Wulfsohn1997, 
	AUTHOR = {M.S. Wulfsohn and A.A. Tsiatis},
	TITLE = {A joint model for survival time and longitudinal data measured with error},
	JOURNAL = {Biometrics},
	VOLUME = {53}, 
	NUMBER = {},
	PAGES = {330--339},
	YEAR = {1997},
}

@article{Xu2001,
	AUTHOR = {J. Xu and S.L. Zeger},
	TITLE = {Joint analysis of longitudinal data comprising repeated measures and time to events},
	JOURNAL = {Applied Statistics},
	VOLUME = {50}, 
	NUMBER = {},
	PAGES = {375--387},
	YEAR = {2001},
}

@online{ProustLimaetal2015, 
	AUTHOR = {C. Proust-Lima and V. Philipps and B. Liquet},
	TITLE = {Estimation of extended mixed models using latent classes and latent processes: the R package lcmm},
	eprinttype = {arxiv},
	eprint = {1503.00890},
	date = {2015-09-21},
	ee = {http://arxiv.org/abs/1503.00890},
}

@article{Rizopoulos2010,
	AUTHOR = {D. Rizopoulos},
	TITLE = {JM: An R package for the joint modeling of longitudinal and time-to-event data},
	JOURNAL = {Journal of Statistical Software},
	VOLUME = {35}, 
	NUMBER = {},
	PAGES = {1--33},
	YEAR = {2010},
}

@article{LiuRehg2015,
	AUTHOR = {Y. Liu and L. Song and F. Li and S. Li and J. Rehg},
	TITLE = {Efficient Continuous-Time Hidden Markov Model for Disease Modeling},
	JOURNAL = {In Proceedings for Advances in Neural Information Processing Systems (NIPS)},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {2015},
}

@article{Rehg3,
	AUTHOR = {J. Hernandez, Yin Li, J. M. Rehg and R. W. Picard},
	TITLE = {Physiological Parameter Estimation Using a Head-mounted Wearable Device.},
	JOURNAL = {In Proceedings for MOBIHEALTH},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {2014},
}

@article{deeplearning,
	AUTHOR = {G. Hinton and S. Osindero and Y.W. Teh},
	TITLE = {A fast leearning algorithm for deep belief nets},
	JOURNAL = {Neural Computation},
	VOLUME = {18}, 
	NUMBER = {},
	PAGES = {1527--1554},
	YEAR = {2006},
}

@article{statespacemodels,
	AUTHOR = {},
	TITLE = {},
	JOURNAL = {},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {},
}

@book{Diggle2002,
	AUTHOR = {P.J. Diggle and P. Heagerty and K.Y. Liang and S.L. Zeger},
	TITLE = {Analysis of Longitudinal Data},
	publisher = {Clarendon press},
	SERIES = {Oxford Science Publications}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {2002},
}

@book{Fitzmaurice2011,
	AUTHOR = {G. Fitzmaurice and N.N. Laird and J.H. Ware},
	TITLE = {Applied Longitudinal Data Analysis},
	edition = {2},
	publisher = {Wiley}, 
	series = {New York},
	YEAR = {2011},
}

@book{Fitzmaurice2009,
	AUTHOR = {G. Fitzmaurice and M. Davidian and G. Verbeke and G. Molenberghs},
	TITLE = {Longitudinal Data Analysis},
	publisher = {Chapman and Hall},
	YEAR = {2009},
}

@article{Mohlenberghs2005,
	AUTHOR = {G. Molenberghs and G. Verbeke},
	TITLE = {Models for Discrete Longitudinal Data},
	publisher = {Springer},
	YEAR = {2005},
}

@article{Arulampalam2002,
	AUTHOR = {M. Arulampalam and S. Maskell and N. Gordon and T. Clapp},
	TITLE = {A Particle Filters for Online Nonlinear/Non-Gaussian Bayesian Tracking},
	JOURNAL = {IEEE transactions on signal processing},
	VOLUME = {50}, 
	NUMBER = {},
	PAGES = {174--188},
	YEAR = {2002},
}

@article{Andrieu2010,
	AUTHOR = {C. Andrieu and A. Doucet and R. Holenstein},
	TITLE = {Particle markov chain monte carlo methods},
	JOURNAL = {J. R. Stat. Soc. Ser. B Stat. Methodol. },
	VOLUME = {72}, 
	NUMBER = {},
	PAGES = {269--342},
	YEAR = {2010},
}

@article{Doucet2000,
	AUTHOR = {A. Doucet and S. Godsill and C. Andrieu},
	TITLE = {On sequential Monte Carlo sampling methods for Bayesian filtering},
	JOURNAL = {Stat. Comput.},
	VOLUME = {10}, 
	NUMBER = {},
	PAGES = {197--208},
	YEAR = {2000},
}

@article{Doucet2009,
	AUTHOR = {A. Doucet and A.M. Johansen},
	TITLE = {A tutorial on particle filtering and smoothing: Fifteen years later},
	JOURNAL = {Handbook of Nonlinear Filtering},
	VOLUME = {12}, 
	NUMBER = {},
	PAGES = {656--704},
	YEAR = {2009},
}

@article{IonidesJSS,
	AUTHOR = {A. King and D. Nguyen and E. Ionides},
	TITLE = {Statistical Inference for Partially Observed Markov Processes via the R Package pomp},
	JOURNAL = {Journal of Statistical Software},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {{2015}forthcoming},
}

@article{IonidesPNAS,
	AUTHOR = {E. Ionides and D. Nguyen and Y. Atchade and S. Stoev and A. King},
	TITLE = {Inference for dynamic and latent variable models via iterated, perturbed Bayes maps},
	JOURNAL = {Proceedings of the National Academy of Sciences of the USA},
	VOLUME = {112}, 
	NUMBER = {},
	PAGES = {719--724},
	YEAR = {2015},
}

@article{IonidesAOS,
	AUTHOR = {E. Ionides and A. Bhadra and Y. Atchade and A. King},
	TITLE = {Iterated Filtering},
	JOURNAL = {Annals of Statistics},
	VOLUME = {39}, 
	NUMBER = {},
	PAGES = {1776--1802},
	YEAR = {2011},
}

@article{Fasiola2015,
	AUTHOR = {M. Fasiolo and N. Pya and S. Wood},
	TITLE = {A comparison of inferential methods for highly non-linear state space models
	in ecology and epidemiology.},
	JOURNAL = {Statistical Science},
	NUMBER = {},
	PAGES = {},
	YEAR = {{2015}forthcoming},
}

@article{sparsepca1,
	AUTHOR = {},
	TITLE = {},
	JOURNAL = {},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {},
}

@article{sparsepca2,
	AUTHOR = {},
	TITLE = {},
	JOURNAL = {},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {},
}

@article{ctshmm,
	AUTHOR = {},
	TITLE = {},
	JOURNAL = {},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {},
}

@book{Cook2007,
	AUTHOR = {R. Cook and J. Lawless},
	TITLE = {The statistical analysis of recurrent events},
	publisher = {Springer},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {2007},
}

@book{Aalen,
	AUTHOR = {O. Aalen and O. Borgan and H. Gjessing},
	TITLE = {Survival and event history analysis: a process point of view},
	publisher = {Springer-Verlag},
	series = {New York},
	NUMBER = {},
	PAGES = {},
	YEAR = {2008},
}

@article{Choi2015,
	AUTHOR = {E. Choi and N. Du and L. Song and J. Sun},
	TITLE = {Constructing Disease Network and Temporal Progression Model via Context-Sensitive Hawkes Process},
	JOURNAL = {International Conference on Data Mining (ICDM)},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {},
}

@inproceedings{conditionalintensity,
	author = {Du, Nan and Farajtabar, Mehrdad and Ahmed, Amr and Smola, Alexander J. and Song, Le},
	title = {Dirichlet-Hawkes Processes with Applications to Clustering Continuous-Time Document Streams},
	booktitle = {Proceedings of the 21th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining},
	series = {KDD '15},
	year = {2015},
	isbn = {978-1-4503-3664-2},
	location = {Sydney, NSW, Australia},
	pages = {219--228},
	numpages = {10},
	url = {http://doi.acm.org/10.1145/2783258.2783411},
	doi = {10.1145/2783258.2783411},
	acmid = {2783411},
	publisher = {ACM},
	address = {New York, NY, USA},
	keywords = {dirichlet process, document modeling, hawkes process},
}

@incollection{sga1,
	year={2012},
	isbn={978-3-642-35288-1},
	booktitle={Neural Networks: Tricks of the Trade},
	volume={7700},
	series={Lecture Notes in Computer Science},
	doi={10.1007/978-3-642-35289-8_25},
	title={Stochastic Gradient Descent Tricks},
	url={http://dx.doi.org/10.1007/978-3-642-35289-8_25},
	publisher={Springer Berlin Heidelberg},
	author={Bottou, L{\'e}on},
	pages={421-436},
	language={English}
}

@incollection{sga2,
	author = {Bottou, L{\'e}on},
	chapter = {On-line Learning and Stochastic Approximations},
	title = {On-line Learning in Neural Networks},
	editor = {Saad, David},
	year = {1998},
	isbn = {0-521-65263-4},
	pages = {9--42},
	numpages = {34},
	url = {http://dl.acm.org/citation.cfm?id=304710.304720},
	acmid = {304720},
	publisher = {Cambridge University Press},
	address = {New York, NY, USA},
} 


@inproceedings{sga3,
	title={Solving large scale linear prediction problems using stochastic gradient descent algorithms},
	author={Zhang, Tong},
	booktitle={Proceedings of the twenty-first international conference on Machine learning},
	pages={116},
	year={2004},
	organization={ACM}
}

@incollection{sga4,
	title = {Hogwild: A Lock-Free Approach to Parallelizing Stochastic Gradient Descent},
	author = {Recht, Benjamin and Re, Christopher and Wright, Stephen and Feng Niu},
	booktitle = {Advances in Neural Information Processing Systems 24},
	editor = {J. Shawe-Taylor and R.S. Zemel and P.L. Bartlett and F. Pereira and K.Q. Weinberger},
	pages = {693--701},
	year = {2011},
	publisher = {Curran Associates, Inc.},
	url = {http://papers.nips.cc/paper/4390-hogwild-a-lock-free-approach-to-parallelizing-stochastic-gradient-descent.pdf}
}


@article{Blanche2015,
	AUTHOR = {P. Blanche and C. Proust-Lima and L. Loub\`{e}re, C. Berr and J.F. Dartiguesand and H. Jacqmin-Gadda},
	TITLE = {Quantifying and comparing dynamic predictive accuracy of joint models for longitudinal marker and time-to-event in presence of censoring and competing risks},
	JOURNAL = {Biometrics},
	VOLUME = {71}, 
	NUMBER = {1},
	PAGES = {102--113},
	YEAR = {2015},
} 

@article{Rizopoulos2011,
	AUTHOR = {D. Rizopoulos},
	TITLE = {Dynamic predictions and prospective accuracy in joint models for longitudinal and time-to-event data},
	JOURNAL = {Biometrics},
	VOLUME = {67}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {819--829},
} 

@article{Murphy1,
	AUTHOR = {S.A. Murphy },
	TITLE = {Optimal dynamic treatment regimes (with discussion)},
	JOURNAL = {J. Roy. Statist. Soc. B.},
	VOLUME = {25}, 
	NUMBER = {},
	PAGES = {331--366},
	YEAR = {2003},
}

@article{Murphy2,
	AUTHOR = {L.M. Collins, S.A. Murphy, V. Strecher},
	TITLE = {The Multiphase Optimization Strategy (MOST) and the Sequential Multiple Assignment Randomized Trial (SMART): New Methods for More Potent e-Health Interventions},
	JOURNAL = {American Journal of Preventive Medicine},
	VOLUME = {32}, 
	NUMBER = {5},
	PAGES = {112--118},
	YEAR = {2007},
}

@article{Murphy3,
	AUTHOR = {Laber,E. and Lizotte, D. and Qian, M. and Pelham, W.E. and S.A. Murphy},
	YEAR= {2014},
	TITLE = {Dynamic treatment regimes: technical challenges and applications.},
	JOURNAL = {Electronic Journal of Statistics, with discussion.},
	VOLUME = {8},
	NUMBER = {0},
	PAGES = {1225--1272},
}

@article{LanzaCollins2006,
	author = {Lanza, S. T. and Collins, L. M.},
	year = {2006},
	title = {A mixture model of discontinuous development in heavy drinking from ages 18 to 30: The role of college enrollment.},
	journal = {Journal of Studies on Alcohol},
	volume = {67},
	pages = {552--561},
}

@article{BethanyLanza,
	author = {Lanza, S. T., and Bray, B. C.},
	year = {2010},
	title = {Transitions in drug use among high-risk women: An application of latent class and latent transition analysis.},
	journal = {Advances and Applications in Statistical Sciences},
	volume = {3},
	number = {2},
	pages = {203--236},
}

@article{Lanza1,
	AUTHOR = {Lanza, S. and Rhoades, B.},
	YEAR= {2013},
	TITLE = {Latent class analysis: an alternative perspective on subgroup analysis in prevention and treatment},
	JOURNAL = {Preventative Science},
	VOLUME = {14},
	NUMBER = {2},
	PAGES = {157--168},
}

@article{Lanza2,
	AUTHOR = {Shiyko, M. and Lanza, S. and Tan, X.and Li, R. and Shiffman, S.},
	YEAR= 2012,
	TITLE = {Using the time-varying effect model (TVEM) to examine dynamic associations between negative
	affect and self confidence on smoking urges: differences between successful quitters and relapsers},
	JOURNAL = {Preventative Science},
	VOLUME = 13,
	PAGES = {288--299},
}

@article{Lanza3,
	AUTHOR = {Lanza, S. and Rhoades, B. and Greenberg, M. and Cox, M.},
	YEAR= {2011},
	TITLE = {Modeling Multiple Risks During Infancy to Predict Quality of the
	Caregiving Environment: Contributions of a Person-Centered Approach},
	JOURNAL = {Infant Behav Dev.},
	VOLUME = {34},
	NUMBER = {3},
	PAGES = {390--406},
}

@article{TaylorandProustLima,
	AUTHOR = {C. Proust-Lima and M. Sne and J.M. Taylor and H. Jacqmin-Gadda},
	TITLE = {Joint latent class models for longitudinal and
	time-to-event data: A review},
	JOURNAL = {Statistical Methods in Medical Research},
	VOLUME = {23}, 
	NUMBER = {},
	PAGES = {74--90},
	YEAR = {2014},
}

@article{ProustlimaTaylor2009,
	AUTHOR = {C. Proust-Lima and J.M. Taylor},
	TITLE = {Development and validation of a dynamic prognostic tool for prostate cancer recurrence using repeated measures of posttreatment PSA: a joint modeling approach},
	JOURNAL = {Biostatistics},
	VOLUME = {10}, 
	NUMBER = {3},
	PAGES = {535--549},
	YEAR = {2009},
}

@article{ROC1, 
	AUTHOR = {},
	TITLE = {},
	JOURNAL = {},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {},
}

@article{ROC2, 
	AUTHOR = {},
	TITLE = {},
	JOURNAL = {},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {},
}

@article{Heagerty2005, 
	AUTHOR = {P. Heagerty and Y. Zheng},
	TITLE = {Survival model predictive accuracy and ROC curves},
	JOURNAL = {Biometrics},
	VOLUME = {61}, 
	NUMBER = {},
	PAGES = {92--105},
	YEAR = {2005},
}

@article{Wangetal2011,
	AUTHOR = {Y. Wang and H. Chen and R. Li and N. Duan and R. Lewis-Fernandez},
	TITLE = {Prediction-based Structured Variable Selection
	through the receiver operating characteristic curve},
	JOURNAL = {Biometrics},
	VOLUME = {67}, 
	NUMBER = {},
	PAGES = {896--905},
	YEAR = {2011},
}

@article{RizopolousMA, 
	AUTHOR = {D. Rizopoulos and L. Hatfield and B. Carlin and J. Takkenberg},
	TITLE = {Combining Dynamic Predictions from Joint Models for Longitudinal and Time-to-Event Data using Bayesian Model Averaging},
	JOURNAL = {Journal of the American Statistical Association},
	VOLUME = {109}, 
	NUMBER = {},
	PAGES = {1385--1397},
	YEAR = {2014},
}

@article{tibshirani,
	AUTHOR = {},
	TITLE = {},
	JOURNAL = {},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {},
}

@article{Ionides, 
	AUTHOR = {},
	TITLE = {},
	JOURNAL = {},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {},
}

@article{Breto2009, 
	AUTHOR = {C. Breto and H. Daihei and E. Ionides and A. King},
	TITLE = {Time series analysis via mechanistic models},
	JOURNAL = {Annals of Applied Statistics},
	VOLUME = {3}, 
	NUMBER = {},
	PAGES = {319--348},
	YEAR = {2009},
}

@article{Breto2006, 
	AUTHOR = {C. Breto and E. Ionides and A. King},
	TITLE = {Inference for nonlinear dynamical systems},
	JOURNAL = {Proceedings of the National
	Academy of Sciences},
	VOLUME = {103}, 
	NUMBER = {},
	PAGES = {18438--18443},
}

@article{King,
	AUTHOR = {},
	TITLE = {},
	JOURNAL = {},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {},
}

@article{Carvalho1, 
	AUTHOR = {C. Carvalho and H. Lopes and N. Polson and M. Taddy},
	TITLE = {Particle learning for general mixtures},
	JOURNAL = {Bayesian Analysis},
	VOLUME = {5}, 
	NUMBER = {},
	PAGES = {709--740},
	YEAR = {2010},
}

@article{Carvalho2, 
	AUTHOR = {C. Carvalho and M. Johannes and H. Lopes and N. Polson},
	TITLE = {Particle learning and smoothing},
	JOURNAL = {Bayesian Analysis},
	VOLUME = {5}, 
	NUMBER = {},
	PAGES = {709--740},
	YEAR = {2010},
}


@article{Schoop2011, 
	AUTHOR = {R. Schoop and M. Schumacher and E. Graf},
	TITLE = {Measures of prediction error for survival data with longitudinal covariates},
	JOURNAL = {Biometrical Journal},
	VOLUME = {53}, 
	NUMBER = {2},
	PAGES = {275--293},
	YEAR = {2011},
}

@article{Henderson2002, 
	AUTHOR = {R. Henderson and P. Diggle and A. Dobson},
	TITLE = {Identification and efficacy of longitudinal markers for survival},
	JOURNAL = {Biostatistics},
	VOLUME = {3}, 
	NUMBER = {1},
	PAGES = {33--50},
	YEAR = {2002},
}


@article{Yuan2013, 
	AUTHOR = {L. Yuan and J. Liu and J. Ye},
	TITLE = {Efficient Methods for Overlapping Group Lasso},
	JOURNAL = {IEEE Transactions on Software Engineering},
	VOLUME = {35}, 
	NUMBER = {9},
	PAGES = {2104--2116},
	YEAR = {2013},
}


@article{Obozinski2011, 
	AUTHOR = {G. Obozinski and S. Inria},
	TITLE = {Group Lasso with Overlaps: the Latent Group Lasso approach},
	YEAR = {2011},
}

@inproceedings{Jacob2009,
	author = {L. Jacob and G. Obozinski and J.P. Vert},
	title = {Group Lasso with Overlap and Graph Lasso},
	booktitle = {Proceedings of the 26th Annual International Conference on Machine Learning},
	series = {ICML '09},
	year = 2009,
	isbn = {978-1-60558-516-1},
	location = {Montreal, Quebec, Canada},
	pages = {433--440},
	numpages = 8,
	url = {http://doi.acm.org/10.1145/1553374.1553431},
	doi = {10.1145/1553374.1553431},
	acmid = 1553431,
	publisher = {ACM},
	address = {New York, NY, USA},
}

@article{Cadima1995,
	author = {J. Cadima and I.T. Jolliffe},
	title = {Loadings and correlations in the interpretation of principal components},
	journal = {Journal of Applied Statistics},
	number = {22},
	pages = {203--214},
	year = {1995},
}


@article{Zou2006,
	author = {H. Zou and T. Hastie and R. Tibshirani},
	title = {Sparse Principal Component Analysis},
	journal = {Journal of Computational \& Graphical Statistics},
	number = {15},
	pages = {265--286},
	year = {2006},
}

@book{hastieetal,
	address = {New York, NY, USA},
	author = {Hastie, Trevor and Tibshirani, Robert and Friedman, Jerome},
	publisher = {Springer New York Inc.},
	series = {Springer Series in Statistics},
	title = {The Elements of Statistical Learning},
	year = 2001
}

@book{Kingman92,
	address = {Great Clarendon Street, Oxford OX2 6DP},
	author = {Kingman, J.F.C.},
	publisher = {Clarendon Press},
	series = {Oxford Science Publications},
	title = {Poisson Processes},
	year = 1992
}


@inbook {Stock2010,
	title = {Dynamic Factor Models},
	booktitle = {Oxford Handbook of Economic Forecasting},
	year = {2010},
	publisher = {Oxford University Press},
	organization = {Oxford University Press},
	address = {Oxford},
	url = {http://www.economics.harvard.edu/faculty/stock/files/dfm_oup_4.pdf},
	author = {James Stock and Mark Watson},
	editor = {Michael P. Clements and David F. Henry}
}

@article{Lanza2015,
	AUTHOR = {S.T. Lanza and B.R. Cooper},
	TITLE = {Latent class analysis: Contributions to developmental research and new directions},
	JOURNAL = {Child Development Perspectives},
	VOLUME = {}, 
	NUMBER = {},
	PAGES = {},
	YEAR = {{2015}in press},
}

@book{hagenaars2002applied,
	title={Applied latent class analysis},
	author={Hagenaars, Jacques A and McCutcheon, Allan L},
	year={2002},
	publisher={Cambridge University Press}
}

@book{mccutcheon1987latent,
	title={Latent class analysis},
	author={McCutcheon, Allan L},
	number={64},
	year={1987},
	publisher={Sage}
}

@article{brockman2016openai,
  title={Openai gym},
  author={Brockman, Greg and Cheung, Vicki and Pettersson, Ludwig and Schneider, Jonas and Schulman, John and Tang, Jie and Zaremba, Wojciech},
  journal={arXiv preprint arXiv:1606.01540},
  year={2016}
}

@inproceedings{maclaurin2015gradient,
  title={Gradient-based hyperparameter optimization through reversible learning},
  author={Maclaurin, Dougal and Duvenaud, David and Adams, Ryan},
  booktitle={International conference on machine learning},
  pages={2113--2122},
  year={2015},
  organization={PMLR}
}

@inproceedings{jiang2015abstraction,
  title={Abstraction selection in model-based reinforcement learning},
  author={Jiang, Nan and Kulesza, Alex and Singh, Satinder},
  booktitle={International Conference on Machine Learning},
  pages={179--188},
  year={2015},
  organization={PMLR}
}


@article{fard2010pac,
  title={PAC-Bayesian model selection for reinforcement learning},
  author={Fard, M and Pineau, Joelle},
  journal={Advances in Neural Information Processing Systems},
  volume={23},
  pages={1624--1632},
  year={2010}
}


@article{bergstra2011algorithms,
  title={Algorithms for hyper-parameter optimization},
  author={Bergstra, James and Bardenet, R{\'e}mi and Bengio, Yoshua and K{\'e}gl, Bal{\'a}zs},
  journal={Advances in neural information processing systems},
  volume={24},
  year={2011}
}


@article{zoph2016neural,
  title={Neural architecture search with reinforcement learning},
  author={Zoph, Barret and Le, Quoc V},
  journal={arXiv preprint arXiv:1611.01578},
  year={2016}
}

@inproceedings{fujimoto2019off,
  title={Off-policy deep reinforcement learning without exploration},
  author={Fujimoto, Scott and Meger, David and Precup, Doina},
  booktitle={International Conference on Machine Learning},
  pages={2052--2062},
  year={2019},
  organization={PMLR}
}

@inproceedings{bain1995framework,
  title={A Framework for Behavioural Cloning.},
  author={Bain, Michael and Sammut, Claude},
  booktitle={Machine Intelligence 15},
  pages={103--129},
  year={1995}
}


@inproceedings{ross2010efficient,
  title={Efficient reductions for imitation learning},
  author={Ross, St{\'e}phane and Bagnell, Drew},
  booktitle={Proceedings of the thirteenth international conference on artificial intelligence and statistics},
  pages={661--668},
  year={2010},
  organization={JMLR Workshop and Conference Proceedings}
}


@inproceedings{barth2018distributed,
  title={Distributed Distributional Deterministic Policy Gradients},
  author={Barth-Maron, Gabriel and Hoffman, Matthew W and Budden, David and Dabney, Will and Horgan, Dan and Dhruva, TB and Muldal, Alistair and Heess, Nicolas and Lillicrap, Timothy},
  booktitle={International Conference on Learning Representations},
  year={2018}
}


@article{gulcehre2020rl,
  title={Rl unplugged: Benchmarks for offline reinforcement learning},
  author={Gulcehre, Caglar and Wang, Ziyu and Novikov, Alexander and Le Paine, Tom and Gomez Colmenarejo, Sergio and Zolna, Konrad and Agarwal, Rishabh and Merel, Josh and Mankowitz, Daniel and Paduraru, Cosmin and others},
  journal={arXiv e-prints},
  pages={arXiv--2006},
  year={2020}
}



@inproceedings{agarwal2020optimistic,
  title={An optimistic perspective on offline reinforcement learning},
  author={Agarwal, Rishabh and Schuurmans, Dale and Norouzi, Mohammad},
  booktitle={International Conference on Machine Learning},
  pages={104--114},
  year={2020},
  organization={PMLR}
}


@inproceedings{siegel2019keep,
  title={Keep Doing What Worked: Behavior Modelling Priors for Offline Reinforcement Learning},
  author={Siegel, Noah and Springenberg, Jost Tobias and Berkenkamp, Felix and Abdolmaleki, Abbas and Neunert, Michael and Lampe, Thomas and Hafner, Roland and Heess, Nicolas and Riedmiller, Martin},
  booktitle={International Conference on Learning Representations},
  year={2019}
}



@inproceedings{lee2021online,
  title={Online model selection for reinforcement learning with function approximation},
  author={Lee, Jonathan and Pacchiano, Aldo and Muthukumar, Vidya and Kong, Weihao and Brunskill, Emma},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={3340--3348},
  year={2021},
  organization={PMLR}
}

@article{chen2021survey,
  title={A Survey of Deep Reinforcement Learning in Recommender Systems: A Systematic Review and Future Directions},
  author={Chen, Xiaocong and Yao, Lina and McAuley, Julian and Zhou, Guanglin and Wang, Xianzhi},
  journal={arXiv preprint arXiv:2109.03540},
  year={2021}
}


@inproceedings{yang2020enhanced,
  title={Enhanced adversarial strategically-timed attacks against deep reinforcement learning},
  author={Yang, Chao-Han Huck and Qi, Jun and Chen, Pin-Yu and Ouyang, Yi and Hung, I-Te Danny and Lee, Chin-Hui and Ma, Xiaoli},
  booktitle={ICASSP 2020-2020 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
  pages={3407--3411},
  year={2020},
  organization={IEEE}
}

@article{bennett2021proximal,
  title={Proximal Reinforcement Learning: Efficient Off-Policy Evaluation in Partially Observed Markov Decision Processes},
  author={Bennett, Andrew and Kallus, Nathan},
  journal={arXiv preprint arXiv:2110.15332},
  year={2021}
}


@inproceedings{lee2014exact,
  title={Exact post model selection inference for marginal screening},
  author={Lee, Jason D and Taylor, Jonathan E},
  booktitle={Proceedings of the 27th International Conference on Neural Information Processing Systems-Volume 1},
  pages={136--144},
  year={2014}
}

@article{yang2021causal,
  title={Causal Inference Q-Network: Toward Resilient Reinforcement Learning},
  author={Yang, Chao-Han Huck and Hung, I and Danny, Te and Ouyang, Yi and Chen, Pin-Yu},
  journal={arXiv preprint arXiv:2102.09677},
  year={2021}
}



@book{hutter2019automated,
  title={Automated machine learning: methods, systems, challenges},
  author={Hutter, Frank and Kotthoff, Lars and Vanschoren, Joaquin},
  year={2019},
  publisher={Springer Nature}
}

@article{schaul2015prioritized,
  title={Prioritized experience replay},
  author={Schaul, Tom and Quan, John and Antonoglou, Ioannis and Silver, David},
  journal={arXiv preprint arXiv:1511.05952},
  year={2015}
}


@inproceedings{van2016deep,
  title={Deep reinforcement learning with double q-learning},
  author={Van Hasselt, Hado and Guez, Arthur and Silver, David},
  booktitle={Proceedings of the AAAI conference on artificial intelligence},
  volume={30},
  year={2016}
}



@article{abbasi2011improved,
  title={Improved algorithms for linear stochastic bandits},
  author={Abbasi-Yadkori, Yasin and P{\'a}l, D{\'a}vid and Szepesv{\'a}ri, Csaba},
  journal={Advances in neural information processing systems},
  volume={24},
  pages={2312--2320},
  year={2011}
}

@article{gottesman2018evaluating,
  title={Evaluating reinforcement learning algorithms in observational health settings},
  author={Gottesman, Omer and Johansson, Fredrik and Meier, Joshua and Dent, Jack and Lee, Donghun and Srinivasan, Srivatsan and Zhang, Linying and Ding, Yi and Wihl, David and Peng, Xuefeng and others},
  journal={arXiv preprint arXiv:1805.12298},
  year={2018}
}


@inproceedings{chu2011contextual,
  title={Contextual bandits with linear payoff functions},
  author={Chu, Wei and Li, Lihong and Reyzin, Lev and Schapire, Robert},
  booktitle={Proceedings of the Fourteenth International Conference on Artificial Intelligence and Statistics},
  pages={208--214},
  year={2011},
  organization={JMLR Workshop and Conference Proceedings}
}

@article{pacchiano2020optimism,
  title={On optimism in model-based reinforcement learning},
  author={Pacchiano, Aldo and Ball, Philip and Parker-Holder, Jack and Choromanski, Krzysztof and Roberts, Stephen},
  journal={arXiv preprint arXiv:2006.11911},
  year={2020}
}


@article{singh1996reinforcement,
  title={Reinforcement learning with replacing eligibility traces},
  author={Singh, Satinder P and Sutton, Richard S},
  journal={Machine learning},
  volume={22},
  number={1},
  pages={123--158},
  year={1996},
  publisher={Springer}
}


@book{LanzaCollins,
	YEAR = {2009},
	PUBLISHER = {Wiley},
	AUTHOR = {Collins, L.M. and Lanza, S.T.},
	TITLE = {Latent Class and Latent Transition Analysis: With Applications in the Social, Behavioral, and Health Sciences},
	SERIES  = {Wiley Series in Probability and Statistics}
}

@INBOOK{FoxJordan:14,
	AUTHOR = {Fox, E.B. and Jordan, M.I.},
	Chapter = {Mixed Membership Models for Time Series},
	Editor = {Airoldi, E. and Blei, D. and Erosheva, E. and Fienberg, S.E., and Bokalders, K.},
	Publisher = {Chapman \& Hall},
	Title = {Handbook on Mixed Membership Models},
	Year = {2014},
}

@article{mnih2013playing,
  title={Playing atari with deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Graves, Alex and Antonoglou, Ioannis and Wierstra, Daan and Riedmiller, Martin},
  journal={arXiv preprint arXiv:1312.5602},
  year={2013}
}

@article{juliani2018unity,
  title={Unity: A general platform for intelligent agents},
  author={Juliani, Arthur and Berges, Vincent-Pierre and Teng, Ervin and Cohen, Andrew and Harper, Jonathan and Elion, Chris and Goy, Chris and Gao, Yuan and Henry, Hunter and Mattar, Marwan and others},
  journal={arXiv preprint arXiv:1809.02627},
  year={2018}
}

@article{jaquette1973markov,
  title={Markov decision processes with a new optimality criterion: Discrete time},
  author={Jaquette, Stratton C},
  journal={The Annals of Statistics},
  pages={496--505},
  year={1973},
  publisher={JSTOR}
}


@inproceedings{bellemare2017distributional,
  title={A distributional perspective on reinforcement learning},
  author={Bellemare, Marc G and Dabney, Will and Munos, R{\'e}mi},
  booktitle={International Conference on Machine Learning},
  pages={449--458},
  year={2017},
  organization={PMLR}
}


@inproceedings{tang2020clinician,
  title={Clinician-in-the-loop decision making: Reinforcement learning with near-optimal set-valued policies},
  author={Tang, Shengpu and Modi, Aditya and Sjoding, Michael and Wiens, Jenna},
  booktitle={International Conference on Machine Learning},
  pages={9387--9396},
  year={2020},
  organization={PMLR}
}

@inproceedings{kidambi2020morel,
  title={MOReL: Model-Based Offline Reinforcement Learning},
  author={Kidambi, Rahul and Rajeswaran, Aravind and Netrapalli, Praneeth and Joachims, Thorsten},
  booktitle={NeurIPS},
  year={2020}
}


@inproceedings{jaques2020human,
  title={Human-centric Dialog Training via Offline Reinforcement Learning},
  author={Jaques, Natasha and Shen, Judy Hanwen and Ghandeharioun, Asma and Ferguson, Craig and Lapedriza, Agata and Jones, Noah and Gu, Shixiang and Picard, Rosalind},
  booktitle={Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
  pages={3985--4003},
  year={2020}
}


@inproceedings{hase2020ultrasound,
  title={Ultrasound-guided robotic navigation with deep reinforcement learning},
  author={Hase, Hannes and Azampour, Mohammad Farid and Tirindelli, Maria and Paschali, Magdalini and Simson, Walter and Fatemizadeh, Emad and Navab, Nassir},
  booktitle={2020 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},
  pages={5534--5541},
  year={2020},
  organization={IEEE}
}


@inproceedings{kahn2018self,
  title={Self-supervised deep reinforcement learning with generalized computation graphs for robot navigation},
  author={Kahn, Gregory and Villaflor, Adam and Ding, Bosen and Abbeel, Pieter and Levine, Sergey},
  booktitle={2018 IEEE International Conference on Robotics and Automation (ICRA)},
  pages={5129--5136},
  year={2018},
  organization={IEEE}
}


@inproceedings{henderson2018deep,
  title={Deep reinforcement learning that matters},
  author={Henderson, Peter and Islam, Riashat and Bachman, Philip and Pineau, Joelle and Precup, Doina and Meger, David},
  booktitle={Proceedings of the AAAI conference on artificial intelligence},
  volume={32},
  year={2018}
}


@article{Fox:IEEETSP2011,
	title={Bayesian Nonparametric Inference of Switching Dynamic Linear Models},
	author={Fox, E.B. and Sudderth, E.B. and Jordan, M.I. and Willsky, A.S.},
	journal={IEEE Transactions on Signal Processing},
	volume=59,
	number=4,
	year=2011,
}

@article{Fox:IEEESPM2010,
	title={Bayesian Nonparametric Methods for Learning {M}arkov Switching Processes},
	author={Fox, E.B. and Sudderth, E.B. and Jordan, M.I. and Willsky, A.S.},
	journal={IEEE Signal Processing Magazine},
	volume={27},
	number={6},
	pages={43--54},
	year={2010},
}

@article{Loo2015,
	author = {B. Loo and K.M. Griffiths and A. Gulliver},
	title = {Effectiveness of mobile technologies delivering Ecological Momentary Interventions for stress and anxiety: a systematic review},
	journal = {J Am Med Inform Assoc.},
	journal = {Journal of the American Medical Informatics Association},
	volume = 23,
	number = 1,
	pages = {221--229},
	year = 2016,
}


@article{Duncan2014,
	author = {M. Duncan and C. Vandelanotte and G.S. Kolt and R.R. Rosenkranz and C.M. Caperchione
	and E.S. George and H. Ding and C. Hooker and M. Karunanithi and A.J. Maeder and M. Noakes 
	and R. Tague and P. Taylor and P. Viljoen and W.K. Mummery},
	title = {Effectiveness of a web- and mobile phone-based intervention to promote physical activity and healthy eating in middle-aged males: randomized controlled trial of the ManUp study},
	journal = {J Med Internet Res},
	VOLUME = {16},
	NUMBER = {6},
	PAGES = {e136},
	year = {2014},
}

@article{Pfaeffli2015, 
	author= {L. Pfaeffli and R. Dobson and R. Whittaker and R. Maddison},
	title = {The effectiveness of mobile-health behaviour change interventions for cardiovascular disease self-management: A systematic review},
	journal = {European Journal of Preventive Cardiology},
	volume = {23},
	number = {8},
	pages = {801-817},
	year = {2016},
}


## Auto car related work

@inproceedings{43849,
	title = {Pedestrian Detection with a Large-Field-Of-View Deep Network},
	author  = {Anelia Angelova and Alex Krizhevsky and Vincent Vanhoucke},
	year  = 2015,
	booktitle = {Proceedings of ICRA 2015}
}

@article{Gharamani2015,
	title = {Probabilistic machine learning and artificial intelligence},
	author = {Z. Gharamani},
	year = {2015},
	journal = {Nature},
	doi = {10.1038/nature14541},
}

@incollection{kundu2014joint,
	title={Joint semantic segmentation and 3d reconstruction from monocular video},
	author={Kundu, Abhijit and Li, Yin and Dellaert, Frank and Li, Fuxin and Rehg, James M},
	booktitle={Computer Vision--ECCV 2014},
	pages={703--718},
	year={2014},
	publisher={Springer International Publishing}
}

@inproceedings{Lmn2006,
	author = "Lamon, Pierre and Stachniss, Cyrill and Triebel, Rudolph and Pfaff, Patrick and Plagemann, Christian and Grisetti, Giorgio and Kolski, Sascha and Burgard, Wolfram and Siegwart, Roland",
	title = "{M}apping with an {A}utonomous {C}ar",
	booktitle = "Proceedings of The Workshop on Safe Navigation in Open and Dynamic Environments (IROS)",
	year = 2006,
}

@inproceedings{kolter2010probabilistic,
	title={A probabilistic approach to mixed open-loop and closed-loop control, with application to extreme autonomous driving},
	author={Kolter, J Zico and Plagemann, Christian and Jackson, David T and Ng, Andrew Y and Thrun, Sebastian},
	booktitle={Robotics and Automation (ICRA), 2010 IEEE International Conference on},
	pages={839--845},
	year={2010},
	organization={IEEE}
}

### HMM and Moble in Santosh space

@inproceedings{Witayangkurn2013,
	author = {Witayangkurn, Apichon and Horanont, Teerayut and Sekimoto, Yoshihide and Shibasaki, Ryosuke},
	title = {Anomalous Event Detection on Large-scale GPS Data from Mobile Phones Using Hidden Markov Model and Cloud Platform},
	booktitle = {Proceedings of the 2013 ACM Conference on Pervasive and Ubiquitous Computing Adjunct Publication},
	series = {UbiComp '13 Adjunct},
	year = {2013},
	isbn = {978-1-4503-2215-7},
	location = {Zurich, Switzerland},
	pages = {1219--1228},
	numpages = {10},
	url = {http://doi.acm.org/10.1145/2494091.2497352},
	doi = {10.1145/2494091.2497352},
	acmid = {2497352},
	publisher = {ACM},
	address = {New York, NY, USA},
}

@article{Kravitz2014,
	AUTHOR = {Kravitz, R.L. and Duan, N. and eds and the DEcIDE Methods Center N-of-1 Guidance Panel},
	TITLE = {Design and Implementation of N-of-1 Trials: A User’s Guide.},
	JOURNAL = {AHRQ Publication},
	VOLUME = 13,
	NUMBER = 14,
	address = {Rockville, MD: Agency for Healthcare Research and Quality},
	MONTH = {January},
	YEAR = 2014,
	url = {http://www.effectivehealthcare.ahrq.gov/N-1-Trials.cfm},
}

@article{PMID:23867031,
	Title = {Mobile health technology evaluation: the mHealth evidence workshop},
	Author = {Kumar, Santosh and Nilsen, Wendy J and Abernethy, Amy and Atienza, Audie and Patrick, Kevin and Pavel, Misha and Riley, William T and Shar, Albert and Spring, Bonnie and Spruijt-Metz, Donna and Hedeker, Donald and Honavar, Vasant and Kravitz, Richard and Lefebvre, R Craig and Mohr, David C and Murphy, Susan A and Quinn, Charlene and Shusterman, Vladimir and Swendeman, Dallas},
	DOI = {10.1016/j.amepre.2013.03.017},
	Number = {2},
	Volume = {45},
	Month = {August},
	Year = {2013},
	Journal = {American journal of preventive medicine},
	ISSN = {0749-3797},
	Pages = {228—236},
	URL = {http://europepmc.org/articles/PMC3803146},
}

@article{Pepe1994,
	author = {Pepe, M. S. and G. L. Anderson},
	year = {1994},
	title = {A cautionary note on inference for marginal regression
	models with longitudinal data and general correlated response data},
	journal = {Communications in Statistics - Simulation and Computation},
	volume = {23},
	number = {4},
	pages = {939--951}
}

@article{Vansteelandt2007,
	author = {Vansteelandt, S.},
	year = {2007},
	title = {On confounding, prediction and efficiency in the analysis of longitudinal and cross-sectional clustered data},
	journal = {Scandinavian Journal of Statistics},
	volume = {34},
	number = {3},
	pages = {478--498}
}

@article{Tchetgen2012,
	author = {Tchetgen Tchetgen, E. J. and Glymour, M.M. and Weuve, J. and Robins J.},
	year = {2012},
	title = {Specifying the correlation structure in inverse-probability-weighting estimation for repeated measures},
	journal = {Epidemiology},
	volume = {23},
	number = {4},
	pages = {644--646}
}


@article{Spruijt2014,
	AUTHOR = {Spruijt-Metz, D. and Nilsen, W.},
	TITLE = {Dynamic models of behavior for just-in-time adaptive interventions.},
	JOURNAL = {Pervasive Computing, IEEE},
	YEAR = {2014},
	NUMBER = {46},
	PAGES = {28--35},
}

@inproceedings{Mathew2012,
	author = {Mathew, Wesley and Raposo, Ruben and Martins, Bruno},
	title = {Predicting Future Locations with Hidden Markov Models},
	booktitle = {Proceedings of the 2012 ACM Conference on Ubiquitous Computing},
	series = {UbiComp '12},
	year = {2012},
	isbn = {978-1-4503-1224-0},
	location = {Pittsburgh, Pennsylvania},
	pages = {911--918},
	numpages = {8},
	url = {http://doi.acm.org/10.1145/2370216.2370421},
	doi = {10.1145/2370216.2370421},
	acmid = {2370421},
	publisher = {ACM},
	address = {New York, NY, USA},
} 

@article{bagnell2003covariant,
	title={Covariant policy search},
	author={Bagnell, J Andrew and Schneider, Jeff},
	year={2003},
	publisher={Carnegie Mellon University}
}

@article{peters2008natural,
	title={Natural actor-critic},
	author={Peters, Jan and Schaal, Stefan},
	journal={Neurocomputing},
	volume={71},
	number={7-9},
	pages={1180--1190},
	year={2008},
	publisher={Elsevier}
}

@inproceedings{peters2010relative,
	title={Relative entropy policy search},
	author={Peters, Jan and Mulling, Katharina and Altun, Yasemin},
	booktitle={Twenty-Fourth AAAI Conference on Artificial Intelligence},
	year={2010}
}

@inproceedings{Modayil2008,
	author = {Modayil, Joseph and Bai, Tongxin and Kautz, Henry},
	title = {Improving the Recognition of Interleaved Activities},
	booktitle = {Proceedings of the 10th International Conference on Ubiquitous Computing},
	series = {UbiComp '08},
	year = {2008},
	isbn = {978-1-60558-136-1},
	location = {Seoul, Korea},
	pages = {40--43},
	numpages = {4},
	url = {http://doi.acm.org/10.1145/1409635.1409641},
	doi = {10.1145/1409635.1409641},
	acmid = {1409641},
	publisher = {ACM},
	address = {New York, NY, USA},
}

@inproceedings{vanKasteren2008,
	author = {van Kasteren, Tim and Noulas, Athanasios and Englebienne, Gwenn and Kr\"{o}se, Ben},
	title = {Accurate Activity Recognition in a Home Setting},
	booktitle = {Proceedings of the 10th International Conference on Ubiquitous Computing},
	series = {UbiComp '08},
	year = {2008},
	isbn = {978-1-60558-136-1},
	location = {Seoul, Korea},
	pages = {1--9},
	numpages = {9},
	url = {http://doi.acm.org/10.1145/1409635.1409637},
	doi = {10.1145/1409635.1409637},
	acmid = {1409637},
	publisher = {ACM},
	address = {New York, NY, USA},
} 

@INPROCEEDINGS{Samejima2004,
	author = {Kazuyuki Samejima and Kenji Doya and Yasumasa Ueda and Minoru Kimura},
	title = {Estimating internal variables and parameters of a learning agent by a particle filter},
	booktitle = {Advances in Neural Information Processing Systems},
	year = {2004},
	pages = {04},
	publisher = {MIT Press}
}

@article{hunter2004tutorial,
	title={A tutorial on MM algorithms},
	author={Hunter, David R and Lange, Kenneth},
	journal={The American Statistician},
	volume={58},
	number={1},
	pages={30--37},
	year={2004},
	publisher={Taylor \& Francis}
}

@article{hoeting1999bayesian,
	title={Bayesian model averaging: a tutorial},
	author={Hoeting, Jennifer A and Madigan, David and Raftery, Adrian E and Volinsky, Chris T},
	journal={Statistical science},
	pages={382--401},
	year={1999},
	publisher={Institute of Mathematical Statistics}
}

@article{Crane1,
	AUTHOR = {Crane, Harry and Dempsey, Walter},
	TITLE = {A framework for statistical network modeling},
	YEAR = {2015},
	JOURNAL = {Under review},
}

@article{Crane2,
	AUTHOR = {Crane, Harry and Dempsey, Walter},
	TITLE = {Atypical scaling behavior persists in real world interaction networks},
	YEAR = {2015},
	archivePrefix = "arXiv",
	eprint = {1509.08184},
}

@article{Crane3,
	AUTHOR = {Crane, Harry and Dempsey, Walter},
	TITLE = {Community detection for interaction networks},
	YEAR = {2015},
	JOURNAL = {Under review},
}

@article{Crane4,
	AUTHOR = {Crane, Harry and Dempsey, Walter},
	TITLE = {Edge exchangeable models for sparse, scale-free network datasets},
	YEAR = {2015},
	JOURNAL = {In Preparation},
}
= {Crane, Harry and Dempsey, Walter},
	TITLE = {Atypical scaling behavior persists in real world interaction networks},
	YEAR = {2015},
	archivePrefix = "arXiv",
	eprint = {1509.08184},
}

@article{Crane3,
	AUTHOR = {Crane, Harry and Dempsey, Walter},
	TITLE = {Community detection for interaction networks},
	YEAR = {2015},
	JOURNAL = {Under review},
}

@article{Crane4,
	AUTHOR = {Crane, Harry and Dempsey, Walter},
	TITLE = {Edge exchangeable models for sparse, scale-free network datasets},
	YEAR = {2015},
	JOURNAL = {In Preparation},
}

@article{robins1994,
 ISSN = {01621459},
 URL = {http://www.jstor.org/stable/2290910},
 author = {James M. Robins and Andrea Rotnitzky and Lue Ping Zhao},
 journal = {Journal of the American Statistical Association},
 number = {427},
 pages = {846--866},
 publisher = {[American Statistical Association, Taylor & Francis, Ltd.]},
 title = {Estimation of Regression Coefficients When Some Regressors Are Not Always Observed},
 volume = {89},
 year = {1994}
}


