\relax 
\providecommand\hyper@newdestlabel[2]{}
\providecommand\HyperFirstAtBeginDocument{\AtBeginDocument}
\HyperFirstAtBeginDocument{\ifx\hyper@anchor\@undefined
\global\let\oldcontentsline\contentsline
\gdef\contentsline#1#2#3#4{\oldcontentsline{#1}{#2}{#3}}
\global\let\oldnewlabel\newlabel
\gdef\newlabel#1#2{\newlabelxx{#1}#2}
\gdef\newlabelxx#1#2#3#4#5#6{\oldnewlabel{#1}{{#2}{#3}}}
\AtEndDocument{\ifx\hyper@anchor\@undefined
\let\contentsline\oldcontentsline
\let\newlabel\oldnewlabel
\fi}
\fi}
\global\let\hyper@last\relax 
\gdef\HyperFirstAtBeginDocument#1{#1}
\providecommand\HyField@AuxAddToFields[1]{}
\providecommand\HyField@AuxAddToCoFields[2]{}
\providecommand\babel@aux[2]{}
\@nameuse{bbl@beforestart}
\bibstyle{plainnat}
\citation{bowman2015generating}
\citation{Deng2020Residual,NEURIPS2020_pang}
\citation{calixto2019latent}
\citation{Deng2020Residual}
\citation{shen-etal-2017-conditional}
\citation{9265406,jhamtani-berg-kirkpatrick-2020-narrative}
\citation{Sutskever2014SequenceTS}
\citation{Fang_iLVM_2019_EMNLP}
\citation{Casale2018GaussianPP,Tomczak2018VAEWA}
\citation{bowman2015generating}
\citation{Fang_iLVM_2019_EMNLP}
\citation{NEURIPS2020_Khosla,LKhc2020ContrastiveRL}
\providecommand \oddpage@label [2]{}
\babel@aux{american}{}
\@writefile{toc}{\contentsline {section}{\numberline {1}Introduction}{1}{section.1}\protected@file@percent }
\@writefile{lof}{\contentsline {figure}{\numberline {1}{\ignorespaces Contrastive learning over latent variables.\relax }}{1}{figure.caption.1}\protected@file@percent }
\providecommand*\caption@xref[2]{\@setref\relax\@undefined{#1}}
\newlabel{fig:clvae}{{1}{1}{Contrastive learning over latent variables.\relax }{figure.caption.1}{}}
\newlabel{fig:clvae@cref}{{[figure][1][]1}{[1][1][]1}}
\citation{rockafellar1966extension,NIPS2018_dai}
\citation{Sun2020CircleLA}
\citation{bowman2015generating}
\citation{Tomczak2018VAEWA,wang-wang-2019-riemannian,ding-gimpel-2021-flowprior}
\citation{bowman2015generating,higgins2017beta,he2019lagging,Fu2019CyclicalAS}
\citation{Fang_iLVM_2019_EMNLP}
\citation{NIPS2018_dai}
\citation{Tomczak2018VAEWA}
\citation{Oord2018RepresentationLW,Hjelm2019LearningDR,He2020MomentumCF,Chen2020ASF}
\citation{zhang2020unsupervised,Giorgi2021DeCLUTRDC,Carlsson2021SemanticRW,Gao2021SimCSESC}
\citation{Gao2021SimCSESC}
\citation{liang2021rdrop}
\citation{Dai2021DialogueRG}
\citation{Su2022ACF}
\citation{lee2021contrastive}
\@writefile{toc}{\contentsline {section}{\numberline {2}Related Work}{2}{section.2}\protected@file@percent }
\citation{bowman2015generating}
\citation{Fu2019CyclicalAS}
\citation{hochreiter1997long}
\citation{Fang_iLVM_2019_EMNLP}
\citation{rockafellar1966extension,NIPS2018_dai}
\@writefile{lof}{\contentsline {figure}{\numberline {2}{\ignorespaces Relationships among the models.\relax }}{3}{figure.caption.2}\protected@file@percent }
\newlabel{fig:modelrel}{{2}{3}{Relationships among the models.\relax }{figure.caption.2}{}}
\newlabel{fig:modelrel@cref}{{[figure][2][]2}{[1][3][]3}}
\@writefile{toc}{\contentsline {section}{\numberline {3}Method}{3}{section.3}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {3.1}Variational Auto-Encoder Baseline}{3}{subsection.3.1}\protected@file@percent }
\newlabel{sec:vae}{{3.1}{3}{Variational Auto-Encoder Baseline}{subsection.3.1}{}}
\newlabel{sec:vae@cref}{{[subsection][1][3]3.1}{[1][3][]3}}
\newlabel{eq:elbo}{{1}{3}{Variational Auto-Encoder Baseline}{equation.3.1}{}}
\newlabel{eq:elbo@cref}{{[equation][1][]1}{[1][3][]3}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.2}Implicit VAE Baseline (iVAE)}{3}{subsection.3.2}\protected@file@percent }
\newlabel{sec:ivae}{{3.2}{3}{Implicit VAE Baseline (iVAE)}{subsection.3.2}{}}
\newlabel{sec:ivae@cref}{{[subsection][2][3]3.2}{[1][3][]3}}
\newlabel{eq:implicitz}{{2}{3}{Implicit VAE Baseline (iVAE)}{equation.3.2}{}}
\newlabel{eq:implicitz@cref}{{[equation][2][]2}{[1][3][]3}}
\newlabel{eq:dualkl}{{3}{3}{Implicit VAE Baseline (iVAE)}{equation.3.3}{}}
\newlabel{eq:dualkl@cref}{{[equation][3][]3}{[1][3][]3}}
\citation{Fang_iLVM_2019_EMNLP}
\citation{Chen2020ASF}
\citation{Gao2021SimCSESC}
\citation{Gao2021SimCSESC}
\citation{Sun2020CircleLA}
\newlabel{eq:ivae}{{4}{4}{Implicit VAE Baseline (iVAE)}{equation.3.4}{}}
\newlabel{eq:ivae@cref}{{[equation][4][]4}{[1][3][]4}}
\newlabel{eq:lvaed}{{5}{4}{Implicit VAE Baseline (iVAE)}{equation.3.5}{}}
\newlabel{eq:lvaed@cref}{{[equation][5][]5}{[1][4][]4}}
\newlabel{eq:livaed}{{6}{4}{Implicit VAE Baseline (iVAE)}{equation.3.6}{}}
\newlabel{eq:livaed@cref}{{[equation][6][]6}{[1][4][]4}}
\newlabel{eq:livaemid}{{7}{4}{Implicit VAE Baseline (iVAE)}{equation.3.7}{}}
\newlabel{eq:livaemid@cref}{{[equation][7][]7}{[1][4][]4}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.3}Contrastive Learning over Latent Variables for Text Generation}{4}{subsection.3.3}\protected@file@percent }
\newlabel{eq:clobjective}{{8}{4}{Contrastive Learning over Latent Variables for Text Generation}{equation.3.8}{}}
\newlabel{eq:clobjective@cref}{{[equation][8][]8}{[1][4][]4}}
\newlabel{eq:sim}{{9}{4}{Contrastive Learning over Latent Variables for Text Generation}{equation.3.9}{}}
\newlabel{eq:sim@cref}{{[equation][9][]9}{[1][4][]4}}
\newlabel{eq:clloss}{{10}{4}{Contrastive Learning over Latent Variables for Text Generation}{equation.3.10}{}}
\newlabel{eq:clloss@cref}{{[equation][10][]10}{[1][4][]4}}
\newlabel{eq:clvae}{{11}{4}{Contrastive Learning over Latent Variables for Text Generation}{equation.3.11}{}}
\newlabel{eq:clvae@cref}{{[equation][11][]11}{[1][4][]4}}
\newlabel{eq:clvaemi}{{12}{4}{Contrastive Learning over Latent Variables for Text Generation}{equation.3.12}{}}
\newlabel{eq:clvaemi@cref}{{[equation][12][]12}{[1][4][]4}}
\newlabel{eq:newdualkl}{{13}{4}{Contrastive Learning over Latent Variables for Text Generation}{equation.3.13}{}}
\newlabel{eq:newdualkl@cref}{{[equation][13][]13}{[1][4][]4}}
\citation{marcus1993building}
\citation{yang2017improved}
\citation{he2019lagging}
\citation{Fang_iLVM_2019_EMNLP}
\newlabel{eq:newdualkl2}{{14}{5}{Contrastive Learning over Latent Variables for Text Generation}{equation.3.14}{}}
\newlabel{eq:newdualkl2@cref}{{[equation][14][]14}{[1][5][]5}}
\newlabel{eq:clvaeplus}{{15}{5}{Contrastive Learning over Latent Variables for Text Generation}{equation.3.15}{}}
\newlabel{eq:clvaeplus@cref}{{[equation][15][]15}{[1][5][]5}}
\newlabel{eq:clvaemiplus}{{16}{5}{Contrastive Learning over Latent Variables for Text Generation}{equation.3.16}{}}
\newlabel{eq:clvaemiplus@cref}{{[equation][16][]16}{[1][5][]5}}
\newlabel{eq:sq}{{17}{5}{Contrastive Learning over Latent Variables for Text Generation}{equation.3.17}{}}
\newlabel{eq:sq@cref}{{[equation][17][]17}{[1][5][]5}}
\@writefile{toc}{\contentsline {section}{\numberline {4}Experiments}{5}{section.4}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {4.1}Data and Settings}{5}{subsection.4.1}\protected@file@percent }
\@writefile{loa}{\contentsline {algocf}{\numberline {1}{\ignorespaces The training algorithm (a single step SGD) for contrastive learning over latent variables for text generation. \relax }}{5}{algocf.1}\protected@file@percent }
\newlabel{alg:clvae}{{1}{5}{Contrastive Learning over Latent Variables for Text Generation}{algocf.1}{}}
\newlabel{alg:clvae@cref}{{[algocf][1][]1}{[1][5][]5}}
\newlabel{eq:klupdate}{{18}{5}{Contrastive Learning over Latent Variables for Text Generation}{equation.3.18}{}}
\newlabel{eq:klupdate@cref}{{[equation][18][]18}{[1][5][]5}}
\newlabel{eq:modelupdate}{{19}{5}{Contrastive Learning over Latent Variables for Text Generation}{equation.3.19}{}}
\newlabel{eq:modelupdate@cref}{{[equation][19][]19}{[1][5][]5}}
\citation{bowman2015generating}
\citation{higgins2017beta}
\citation{kim2018semi}
\citation{Fu2019CyclicalAS}
\citation{he2019lagging}
\citation{Fang_iLVM_2019_EMNLP}
\citation{Fang_iLVM_2019_EMNLP}
\@writefile{lot}{\contentsline {table}{\numberline {1}{\ignorespaces Language modeling on PTB. \relax }}{6}{table.caption.4}\protected@file@percent }
\newlabel{tab:lmptb}{{1}{6}{Language modeling on PTB. \relax }{table.caption.4}{}}
\newlabel{tab:lmptb@cref}{{[table][1][]1}{[1][6][]6}}
\@writefile{toc}{\contentsline {subsection}{\numberline {4.2}Language Modeling Results}{6}{subsection.4.2}\protected@file@percent }
\@writefile{lot}{\contentsline {table}{\numberline {2}{\ignorespaces Language modeling on Yahoo and Yelp. \relax }}{6}{table.caption.5}\protected@file@percent }
\newlabel{tab:lmyy}{{2}{6}{Language modeling on Yahoo and Yelp. \relax }{table.caption.5}{}}
\newlabel{tab:lmyy@cref}{{[table][2][]2}{[1][6][]6}}
\citation{Fang_iLVM_2019_EMNLP}
\citation{kim2017adversarially}
\citation{Heafield-estimate}
\@writefile{lot}{\contentsline {table}{\numberline {3}{\ignorespaces Contrastive learning using $\mathbf  {h}$ or $\mathbf  {z}$. \relax }}{7}{table.caption.6}\protected@file@percent }
\newlabel{tab:clhz}{{3}{7}{Contrastive learning using $\mathbf {h}$ or $\mathbf {z}$. \relax }{table.caption.6}{}}
\newlabel{tab:clhz@cref}{{[table][3][]3}{[1][7][]7}}
\@writefile{lot}{\contentsline {table}{\numberline {4}{\ignorespaces Forward and reverse PPL on PTB test set.\relax }}{7}{table.caption.7}\protected@file@percent }
\newlabel{tab:ppl}{{4}{7}{Forward and reverse PPL on PTB test set.\relax }{table.caption.7}{}}
\newlabel{tab:ppl@cref}{{[table][4][]4}{[1][7][]7}}
\@writefile{toc}{\contentsline {subsection}{\numberline {4.3}Analysis}{7}{subsection.4.3}\protected@file@percent }
\newlabel{fig:kltrend}{{3a}{7}{Estimated $KL(Q(\mathbf {z}|\mathbf {x}, \phiv )||P(\mathbf {z}, \thetav ))$ w.r.t training epoch.\relax }{figure.caption.8}{}}
\newlabel{fig:kltrend@cref}{{[subfigure][1][3]3a}{[1][7][]7}}
\newlabel{sub@fig:kltrend}{{a}{7}{Estimated $KL(Q(\mathbf {z}|\mathbf {x}, \phiv )||P(\mathbf {z}, \thetav ))$ w.r.t training epoch.\relax }{figure.caption.8}{}}
\newlabel{sub@fig:kltrend@cref}{{[subfigure][1][3]3a}{[1][7][]7}}
\newlabel{fig:mitrend}{{3b}{7}{Mutual information $I(\mathbf {x}, \mathbf {z})$ w.r.t training epoch.\relax }{figure.caption.8}{}}
\newlabel{fig:mitrend@cref}{{[subfigure][2][3]3b}{[1][7][]7}}
\newlabel{sub@fig:mitrend}{{b}{7}{Mutual information $I(\mathbf {x}, \mathbf {z})$ w.r.t training epoch.\relax }{figure.caption.8}{}}
\newlabel{sub@fig:mitrend@cref}{{[subfigure][2][3]3b}{[1][7][]7}}
\@writefile{lof}{\contentsline {figure}{\numberline {3}{\ignorespaces  KL and MI w.r.t the training epoch.\relax }}{7}{figure.caption.8}\protected@file@percent }
\newlabel{fig:trend}{{3}{7}{KL and MI w.r.t the training epoch.\relax }{figure.caption.8}{}}
\newlabel{fig:trend@cref}{{[figure][3][]3}{[1][7][]7}}
\citation{bowman2015generating}
\@writefile{lot}{\contentsline {table}{\numberline {5}{\ignorespaces Comparisons of iVAE and $\mathbf  {cl}$VAE$^+$on toy data.\relax }}{8}{table.caption.9}\protected@file@percent }
\newlabel{tab:conspeed}{{5a}{8}{$K=4$\relax }{table.caption.9}{}}
\newlabel{tab:conspeed@cref}{{[subtable][1][5]5a}{[1][8][]8}}
\newlabel{sub@tab:conspeed}{{a}{8}{$K=4$\relax }{table.caption.9}{}}
\newlabel{sub@tab:conspeed@cref}{{[subtable][1][5]5a}{[1][8][]8}}
\newlabel{tab:conk8}{{5b}{8}{$K>4$\relax }{table.caption.9}{}}
\newlabel{tab:conk8@cref}{{[subtable][2][5]5b}{[1][8][]8}}
\newlabel{sub@tab:conk8}{{b}{8}{$K>4$\relax }{table.caption.9}{}}
\newlabel{sub@tab:conk8@cref}{{[subtable][2][5]5b}{[1][8][]8}}
\@writefile{toc}{\contentsline {subsection}{\numberline {4.4}Results on Synthetic Data}{8}{subsection.4.4}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {4.5}Sentence Interpolation}{8}{subsection.4.5}\protected@file@percent }
\citation{Gao2021SimCSESC}
\bibdata{teng_729}
\bibcite{bowman2015generating}{{1}{2015}{{Bowman et~al.}}{{Bowman, Vilnis, Vinyals, Dai, Jozefowicz, and Bengio}}}
\@writefile{lot}{\contentsline {table}{\numberline {6}{\ignorespaces Interpolation of latent representation.\relax }}{9}{table.caption.10}\protected@file@percent }
\newlabel{tab:interpolation}{{6}{9}{Interpolation of latent representation.\relax }{table.caption.10}{}}
\newlabel{tab:interpolation@cref}{{[table][6][]6}{[1][9][]9}}
\@writefile{toc}{\contentsline {subsection}{\numberline {4.6}Effect of Different Dropout Rates}{9}{subsection.4.6}\protected@file@percent }
\@writefile{lof}{\contentsline {figure}{\numberline {4}{\ignorespaces PPL and ELBO values w.r.t dropout rate.\relax }}{9}{figure.caption.11}\protected@file@percent }
\newlabel{fig:dropout}{{4}{9}{PPL and ELBO values w.r.t dropout rate.\relax }{figure.caption.11}{}}
\newlabel{fig:dropout@cref}{{[figure][4][]4}{[1][9][]9}}
\@writefile{toc}{\contentsline {subsection}{\numberline {4.7}Different Data Augmentation Methods}{9}{subsection.4.7}\protected@file@percent }
\@writefile{lot}{\contentsline {table}{\numberline {7}{\ignorespaces Comparisons with different augmentation methods. \relax }}{9}{table.caption.12}\protected@file@percent }
\newlabel{tab:aug}{{7}{9}{Comparisons with different augmentation methods. \relax }{table.caption.12}{}}
\newlabel{tab:aug@cref}{{[table][7][]7}{[1][9][]9}}
\@writefile{toc}{\contentsline {section}{\numberline {5}Conclusion}{9}{section.5}\protected@file@percent }
\bibcite{calixto2019latent}{{2}{2019}{{Calixto et~al.}}{{Calixto, Rios, and Aziz}}}
\bibcite{Carlsson2021SemanticRW}{{3}{2021}{{Carlsson et~al.}}{{Carlsson, Gyllensten, Gogoulou, Hellqvist, and Sahlgren}}}
\bibcite{Casale2018GaussianPP}{{4}{2018}{{Casale et~al.}}{{Casale, Dalca, Saglietti, Listgarten, and Fusi}}}
\bibcite{9265406}{{5}{2021}{{Chen et~al.}}{{Chen, Liu, Luan, Zhang, Liu, and Sun}}}
\bibcite{Chen2020ASF}{{6}{2020}{{Chen et~al.}}{{Chen, Kornblith, Norouzi, and Hinton}}}
\bibcite{NIPS2018_dai}{{7}{2018}{{Dai et~al.}}{{Dai, Dai, He, Liu, Liu, Chen, Xiao, and Song}}}
\bibcite{Dai2021DialogueRG}{{8}{2021}{{Dai et~al.}}{{Dai, Wang, Park, and Lee}}}
\bibcite{Deng2020Residual}{{9}{2020}{{Deng et~al.}}{{Deng, Bakhtin, Ott, Szlam, and Ranzato}}}
\bibcite{ding-gimpel-2021-flowprior}{{10}{2021}{{Ding and Gimpel}}{{}}}
\bibcite{Fang_iLVM_2019_EMNLP}{{11}{2019}{{Fang et~al.}}{{Fang, Li, Gao, Dong, and Chen}}}
\bibcite{Fu2019CyclicalAS}{{12}{2019}{{Fu et~al.}}{{Fu, Li, Liu, Gao, Celikyilmaz, and Carin}}}
\bibcite{Gao2021SimCSESC}{{13}{2021}{{Gao et~al.}}{{Gao, Yao, and Chen}}}
\bibcite{Giorgi2021DeCLUTRDC}{{14}{2021}{{Giorgi et~al.}}{{Giorgi, Nitski, Bader, and Wang}}}
\bibcite{he2019lagging}{{15}{2019}{{He et~al.}}{{He, Spokoyny, Neubig, and Berg-Kirkpatrick}}}
\bibcite{He2020MomentumCF}{{16}{2020}{{He et~al.}}{{He, Fan, Wu, Xie, and Girshick}}}
\bibcite{Heafield-estimate}{{17}{2013}{{Heafield et~al.}}{{Heafield, Pouzyrevsky, Clark, and Koehn}}}
\bibcite{higgins2017beta}{{18}{2017}{{Higgins et~al.}}{{Higgins, Matthey, Pal, Burgess, Glorot, Botvinick, Mohamed, and Lerchner}}}
\bibcite{Hjelm2019LearningDR}{{19}{2019}{{Hjelm et~al.}}{{Hjelm, Fedorov, Lavoie-Marchildon, Grewal, Trischler, and Bengio}}}
\bibcite{hochreiter1997long}{{20}{1997}{{Hochreiter and Schmidhuber}}{{}}}
\bibcite{jhamtani-berg-kirkpatrick-2020-narrative}{{21}{2020}{{Jhamtani and Berg-Kirkpatrick}}{{}}}
\bibcite{NEURIPS2020_Khosla}{{22}{2020}{{Khosla et~al.}}{{Khosla, Teterwak, Wang, Sarna, Tian, Isola, Maschinot, Liu, and Krishnan}}}
\bibcite{kim2017adversarially}{{23}{2017}{{Kim et~al.}}{{Kim, Zhang, Rush, LeCun, et~al.}}}
\bibcite{kim2018semi}{{24}{2018}{{Kim et~al.}}{{Kim, Wiseman, Miller, Sontag, and Rush}}}
\bibcite{LKhc2020ContrastiveRL}{{25}{2020}{{Le-Khac et~al.}}{{Le-Khac, Healy, and Smeaton}}}
\bibcite{lee2021contrastive}{{26}{2021}{{Lee et~al.}}{{Lee, Lee, and Hwang}}}
\bibcite{liang2021rdrop}{{27}{2021}{{Liang et~al.}}{{Liang, Wu, Li, Wang, Meng, Qin, Chen, Zhang, and Liu}}}
\bibcite{marcus1993building}{{28}{1993}{{Marcus et~al.}}{{Marcus, Santorini, and Marcinkiewicz}}}
\bibcite{Oord2018RepresentationLW}{{29}{2018}{{Oord et~al.}}{{Oord, Li, and Vinyals}}}
\bibcite{NEURIPS2020_pang}{{30}{2020}{{Pang et~al.}}{{Pang, Han, Nijkamp, Zhu, and Wu}}}
\bibcite{rockafellar1966extension}{{31}{1966}{{Rockafellar et~al.}}{{}}}
\bibcite{shen-etal-2017-conditional}{{32}{2017}{{Shen et~al.}}{{Shen, Su, Li, Li, Niu, Zhao, Aizawa, and Long}}}
\bibcite{Su2022ACF}{{33}{2022}{{Su et~al.}}{{Su, Lan, Wang, Yogatama, Kong, and Collier}}}
\bibcite{Sun2020CircleLA}{{34}{2020}{{Sun et~al.}}{{Sun, Cheng, Zhang, Zhang, Zheng, Wang, and Wei}}}
\bibcite{Sutskever2014SequenceTS}{{35}{2014}{{Sutskever et~al.}}{{Sutskever, Vinyals, and Le}}}
\bibcite{Tomczak2018VAEWA}{{36}{2018}{{Tomczak and Welling}}{{}}}
\bibcite{wang-wang-2019-riemannian}{{37}{2019}{{Wang and Wang}}{{}}}
\bibcite{yang2017improved}{{38}{2017}{{Yang et~al.}}{{Yang, Hu, Salakhutdinov, and Berg-Kirkpatrick}}}
\bibcite{zhang2020unsupervised}{{39}{2020}{{Zhang et~al.}}{{Zhang, He, Liu, Lim, and Bing}}}
\newlabel{LastPage}{{}{11}{}{page.11}{}}
\xdef\lastpage@lastpage{11}
\xdef\lastpage@lastpageHy{11}
\gdef \@abspage@last{11}
