@article{elazar2018adversarial,
  title={Adversarial removal of demographic attributes from text data},
  author={Elazar, Yanai and Goldberg, Yoav},
  journal={arXiv preprint arXiv:1808.06640},
  year={2018}
}
@misc{si2024design2code,
      title={Design2Code: How Far Are We From Automating Front-End Engineering?},
      author={Chenglei Si and Yanzhe Zhang and Zhengyuan Yang and Ruibo Liu and Diyi Yang},
      year={2024},
      eprint={2403.03163},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
  }
@article{chowdhury2021adversarial,
  title={Adversarial scrubbing of demographic information for text classification},
  author={Chowdhury, Somnath Basu Roy and Ghosh, Sayan and Li, Yiyuan and Oliva, Junier B and Srivastava, Shashank and Chaturvedi, Snigdha},
  journal={arXiv preprint arXiv:2109.08613},
  year={2021}
}
@article{claude3,
  title={The Claude 3 Model Family: Opus, Sonnet, Haiku},
  author={Anthropic},
  journal={https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model\_Card\_Claude\_3.pdf},
  year={2024}
}
@article{han2021diverse,
  title={Diverse adversaries for mitigating bias in training},
  author={Han, Xudong and Baldwin, Timothy and Cohn, Trevor},
  journal={arXiv preprint arXiv:2101.10001},
  year={2021}
}
@article{coavoux2018privacy,
  title={Privacy-preserving neural representations of text},
  author={Coavoux, Maximin and Narayan, Shashi and Cohen, Shay B},
  journal={arXiv preprint arXiv:1808.09408},
  year={2018}
}
@article{li2018towards,
  title={Towards robust and privacy-preserving text representations},
  author={Li, Yitong and Baldwin, Timothy and Cohn, Trevor},
  journal={arXiv preprint arXiv:1805.06093},
  year={2018}
}
@inproceedings{ganin2015unsupervised,
  title={Unsupervised domain adaptation by backpropagation},
  author={Ganin, Yaroslav and Lempitsky, Victor},
  booktitle={International conference on machine learning},
  pages={1180--1189},
  year={2015},
  organization={PMLR}
}
@InProceedings{pmlr-v139-yu21f,
  title = 	 {Large Scale Private Learning via Low-rank Reparametrization},
  author =       {Yu, Da and Zhang, Huishuai and Chen, Wei and Yin, Jian and Liu, Tie-Yan},
  booktitle = 	 {Proceedings of the 38th International Conference on Machine Learning},
  pages = 	 {12208--12218},
  year = 	 {2021},
  editor = 	 {Meila, Marina and Zhang, Tong},
  volume = 	 {139},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {18--24 Jul},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v139/yu21f/yu21f.pdf},
  url = 	 {https://proceedings.mlr.press/v139/yu21f.html},
  abstract = 	 {We propose a reparametrization scheme to address the challenges of applying differentially private SGD on large neural networks, which are 1) the huge memory cost of storing individual gradients, 2) the added noise suffering notorious dimensional dependence. Specifically, we reparametrize each weight matrix with two \emph{gradient-carrier} matrices of small dimension and a \emph{residual weight} matrix. We argue that such reparametrization keeps the forward/backward process unchanged while enabling us to compute the projected gradient without computing the gradient itself. To learn with differential privacy, we design \emph{reparametrized gradient perturbation (RGP)} that perturbs the gradients on gradient-carrier matrices and reconstructs an update for the original weight from the noisy gradients. Importantly, we use historical updates to find the gradient-carrier matrices, whose optimality is rigorously justified under linear regression and empirically verified with deep learning tasks. RGP significantly reduces the memory cost and improves the utility. For example, we are the first able to apply differential privacy on the BERT model and achieve an average accuracy of $83.9%$ on four downstream tasks with $\epsilon=8$, which is within $5%$ loss compared to the non-private baseline but enjoys much lower privacy leakage risk.}
}


@inproceedings{Anil2021LargeScaleDP,
  title={Large-Scale Differentially Private BERT},
  author={Rohan Anil and Badih Ghazi and Vineet Gupta and Ravi Kumar and Pasin Manurangsi},
  booktitle={Conference on Empirical Methods in Natural Language Processing},
  year={2021}
}

@article{maheshwari2022fair,
  title={Fair NLP Models with Differentially Private Text Encoders},
  author={Maheshwari, Gaurav and Denis, Pascal and Keller, Mikaela and Bellet, Aur{\'e}lien},
  journal={arXiv preprint arXiv:2205.06135},
  year={2022}
}

@article{Ginart2022SubmixPP,
  title={Submix: Practical Private Prediction for Large-Scale Language Models},
  author={Antonio A. Ginart and Laurens van der Maaten and James Y. Zou and Chuan Guo},
  journal={ArXiv},
  year={2022},
  volume={abs/2201.00971}
}

@article{Weggenmann2022DPVAEHT,
  title={DP-VAE: Human-Readable Text Anonymization for Online Reviews with Differentially Private Variational Autoencoders},
  author={Benjamin Weggenmann and Valentin Rublack and Michael Andrejczuk and Justus Mattern and Florian Kerschbaum},
  journal={Proceedings of the ACM Web Conference 2022},
  year={2022}
}


@InProceedings{pmlr-v75-dwork18a,
  title = 	 {Privacy-preserving Prediction},
  author =       {Dwork, Cynthia and Feldman, Vitaly},
  booktitle = 	 {Proceedings of the 31st  Conference On Learning Theory},
  pages = 	 {1693--1702},
  year = 	 {2018},
  editor = 	 {Bubeck, Sébastien and Perchet, Vianney and Rigollet, Philippe},
  volume = 	 {75},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {06--09 Jul},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v75/dwork18a/dwork18a.pdf},
  url = 	 {https://proceedings.mlr.press/v75/dwork18a.html},
  abstract = 	 {Ensuring differential privacy of models learned from sensitive user data is an important goal that has been studied extensively in recent years. It is now known that for some basic learning problems, especially those involving high-dimensional data, producing an accurate private model requires much more data than learning without privacy. At the same time, in many applications it is not necessary to expose the model itself. Instead users may be allowed to query the prediction model on their inputs only through an appropriate interface. Here we formulate the problem of ensuring privacy of individual predictions and investigate the overheads required to achieve it in several standard models of classification and regression. We first describe a simple baseline approach based on training several models on disjoint subsets of data and using standard private aggregation techniques to predict. We show that this approach has nearly optimal sample complexity for (realizable) PAC learning of any class of Boolean functions. At the same time, without strong assumptions on the data distribution, the aggregation step introduces a substantial overhead. We demonstrate that this overhead can be avoided for the well-studied class of thresholds on a line and for a number of standard settings of convex regression. The analysis of our algorithm for learning thresholds relies crucially on strong generalization guarantees that we establish for all differentially private prediction algorithms.}
}

@inproceedings{abadi2016deep,
  title={Deep learning with differential privacy},
  author={Abadi, Martin and Chu, Andy and Goodfellow, Ian and McMahan, H Brendan and Mironov, Ilya and Talwar, Kunal and Zhang, Li},
  booktitle={Proceedings of the 2016 ACM SIGSAC conference on computer and communications security},
  pages={308--318},
  year={2016}
}

@article{shi2021selective,
  title={Selective differential privacy for language modeling},
  author={Shi, Weiyan and Cui, Aiqi and Li, Evan and Jia, Ruoxi and Yu, Zhou},
  journal={arXiv preprint arXiv:2108.12944},
  year={2021}
}

@inproceedings{Dwork2006CalibratingNT,
  title={Calibrating Noise to Sensitivity in Private Data Analysis},
  author={Cynthia Dwork and Frank McSherry and Kobbi Nissim and Adam D. Smith},
  booktitle={Theory of Cryptography Conference},
  year={2006}
}

@inproceedings{garcia2020sensitive,
    title = "Sensitive Data Detection and Classification in {S}panish Clinical Text: Experiments with {BERT}",
    author = "Garc{\'\i}a Pablos, Aitor  and
      Perez, Naiara  and
      Cuadros, Montse",
    booktitle = "Proceedings of the Twelfth Language Resources and Evaluation Conference",
    month = may,
    year = "2020",
    address = "Marseille, France",
    publisher = "European Language Resources Association",
    url = "https://aclanthology.org/2020.lrec-1.552",
    pages = "4486--4494",
    abstract = "Massive digital data processing provides a wide range of opportunities and benefits, but at the cost of endangering personal data privacy. Anonymisation consists in removing or replacing sensitive information from data, enabling its exploitation for different purposes while preserving the privacy of individuals. Over the years, a lot of automatic anonymisation systems have been proposed; however, depending on the type of data, the target language or the availability of training documents, the task remains challenging still. The emergence of novel deep-learning models during the last two years has brought large improvements to the state of the art in the field of Natural Language Processing. These advancements have been most noticeably led by BERT, a model proposed by Google in 2018, and the shared language models pre-trained on millions of documents. In this paper, we use a BERT-based sequence labelling model to conduct a series of anonymisation experiments on several clinical datasets in Spanish. We also compare BERT with other algorithms. The experiments show that a simple BERT-based model with general-domain pre-training obtains highly competitive results without any domain specific feature engineering.",
    language = "English",
    ISBN = "979-10-95546-34-4",
}

@article{tovino2016hipaa,
  title={The HIPAA privacy rule and the EU GDPR: illustrative comparisons},
  author={Tovino, Stacey A},
  journal={Seton Hall L. Rev.},
  volume={47},
  pages={973},
  year={2016},
  publisher={HeinOnline}
}

@inproceedings{babakov-etal-2021-detecting,
    title = "Detecting Inappropriate Messages on Sensitive Topics that Could Harm a Company{'}s Reputation",
    author = "Babakov, Nikolay  and
      Logacheva, Varvara  and
      Kozlova, Olga  and
      Semenov, Nikita  and
      Panchenko, Alexander",
    booktitle = "Proceedings of the 8th Workshop on Balto-Slavic Natural Language Processing",
    month = apr,
    year = "2021",
    address = "Kiyv, Ukraine",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/2021.bsnlp-1.4",
    pages = "26--36",
    abstract = "Not all topics are equally {``}flammable{''} in terms of toxicity: a calm discussion of turtles or fishing less often fuels inappropriate toxic dialogues than a discussion of politics or sexual minorities. We define a set of sensitive topics that can yield inappropriate and toxic messages and describe the methodology of collecting and labelling a dataset for appropriateness. While toxicity in user-generated data is well-studied, we aim at defining a more fine-grained notion of inappropriateness. The core of inappropriateness is that it can harm the reputation of a speaker. This is different from toxicity in two respects: (i) inappropriateness is topic-related, and (ii) inappropriate message is not toxic but still unacceptable. We collect and release two datasets for Russian: a topic-labelled dataset and an appropriateness-labelled dataset. We also release pre-trained classification models trained on this data.",
}

@article{feyisetan2019privacypreserving,
  title={Privacy- and Utility-Preserving Textual Analysis via Calibrated Multivariate Perturbations},
  author={Oluwaseyi Feyisetan and Borja Balle and Thomas Drake and Tom Diethe},
  journal={Proceedings of the 13th International Conference on Web Search and Data Mining},
  year={2019}
}

@article{carvalho2021privacypreserving,
  title={TEM: High Utility Metric Differential Privacy on Text},
  author={Ricardo Silva Carvalho and Theodore Vasiloudis and Oluwaseyi Feyisetan},
  journal={ArXiv},
  year={2021},
  volume={abs/2107.07928}
}

@article{Xu2021DensityAwareDP,
  title={Density-Aware Differentially Private Textual Perturbations Using Truncated Gumbel Noise},
  author={Nan Xu and Oluwaseyi Feyisetan and Abhinav Aggarwal and Zekun Xu and Nathanael Teissier},
  journal={The International FLAIRS Conference Proceedings},
  year={2021}
}

@article{devlin2019bert,
  title={BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding},
  author={Jacob Devlin and Ming-Wei Chang and Kenton Lee and Kristina Toutanova},
  journal={ArXiv},
  year={2019},
  volume={abs/1810.04805}
}

@article{liu2019roberta,
  title={RoBERTa: A Robustly Optimized BERT Pretraining Approach},
  author={Yinhan Liu and Myle Ott and Naman Goyal and Jingfei Du and Mandar Joshi and Danqi Chen and Omer Levy and Mike Lewis and Luke Zettlemoyer and Veselin Stoyanov},
  journal={ArXiv},
  year={2019},
  volume={abs/1907.11692}
}

@article{alsentzer2019publicly,
  title={Publicly Available Clinical BERT Embeddings},
  author={Emily Alsentzer and John R. Murphy and Willie Boag and Wei-Hung Weng and Di Jin and Tristan Naumann and Matthew B. A. McDermott},
  journal={ArXiv},
  year={2019},
  volume={abs/1904.03323}
}

@article{Feyisetan2021PrivateRO,
  title={Private Release of Text Embedding Vectors},
  author={Oluwaseyi Feyisetan and Shiva Prasad Kasiviswanathan},
  journal={Proceedings of the First Workshop on Trustworthy Natural Language Processing},
  year={2021}
}

@inproceedings{Meehan2022SentencelevelPF,
  title={Sentence-level Privacy for Document Embeddings},
  author={Casey Meehan and Khalil Mrini and Kamalika Chaudhuri},
  booktitle={Annual Meeting of the Association for Computational Linguistics},
  year={2022}
}

@inproceedings{Subramani2020EnablingFD,
  title={Enabling Fast Differentially Private SGD via Just-in-Time Compilation and Vectorization},
  author={Pranav Subramani and Nicholas Vadivelu and Gautam Kamath},
  booktitle={Neural Information Processing Systems},
  year={2020}
}

@article{Goodfellow2015EfficientPG,
  title={Efficient Per-Example Gradient Computations},
  author={Ian J. Goodfellow},
  journal={ArXiv},
  year={2015},
  volume={abs/1510.01799}
}

@article{Tetko1995NeuralNS,
  title={Neural network studies, 1. Comparison of overfitting and overtraining},
  author={Igor V. Tetko and David J. Livingstone and Alexander I. Luik},
  journal={J. Chem. Inf. Comput. Sci.},
  year={1995},
  volume={35},
  pages={826-833}
}

@article{zhou2023webarena,
  title={Webarena: A realistic web environment for building autonomous agents},
  author={Zhou, Shuyan and Xu, Frank F and Zhu, Hao and Zhou, Xuhui and Lo, Robert and Sridhar, Abishek and Cheng, Xianyi and Bisk, Yonatan and Fried, Daniel and Alon, Uri and others},
  journal={arXiv preprint arXiv:2307.13854},
  year={2023}
}

@article{zhang2023mobile,
  title={Mobile-Env: A Universal Platform for Training and Evaluation of Mobile Interaction},
  author={Zhang, Danyang and Chen, Lu and Yu, Kai},
  journal={arXiv preprint arXiv:2305.08144},
  year={2023}
}

@article{li2023sheetcopilot,
  title={SheetCopilot: Bringing Software Productivity to the Next Level through Large Language Models},
  author={Li, Hongxin and Su, Jingran and Chen, Yuntao and Li, Qing and Zhang, Zhaoxiang},
  journal={arXiv preprint arXiv:2305.19308},
  year={2023}
}

@article{deng2023mind2web,
  title={Mind2Web: Towards a Generalist Agent for the Web},
  author={Deng, Xiang and Gu, Yu and Zheng, Boyuan and Chen, Shijie and Stevens, Samuel and Wang, Boshi and Sun, Huan and Su, Yu},
  journal={arXiv preprint arXiv:2306.06070},
  year={2023}
}

@article{rawles2023android,
  title={Android in the wild: A large-scale dataset for android device control},
  author={Rawles, Christopher and Li, Alice and Rodriguez, Daniel and Riva, Oriana and Lillicrap, Timothy},
  journal={arXiv preprint arXiv:2307.10088},
  year={2023}
}

@inproceedings{shi2017world,
  title={World of bits: An open-domain platform for web-based agents},
  author={Shi, Tianlin and Karpathy, Andrej and Fan, Linxi and Hernandez, Jonathan and Liang, Percy},
  booktitle={International Conference on Machine Learning},
  pages={3135--3144},
  year={2017},
  organization={PMLR}
}

@article{liu2018reinforcement,
  title={Reinforcement learning on web interfaces using workflow-guided exploration},
  author={Liu, Evan Zheran and Guu, Kelvin and Pasupat, Panupong and Shi, Tianlin and Liang, Percy},
  journal={arXiv preprint arXiv:1802.08802},
  year={2018}
}

@article{yao2022webshop,
  title={Webshop: Towards scalable real-world web interaction with grounded language agents},
  author={Yao, Shunyu and Chen, Howard and Yang, John and Narasimhan, Karthik},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={20744--20757},
  year={2022}
}

@article{toyama2021androidenv,
  title={Androidenv: A reinforcement learning platform for android},
  author={Toyama, Daniel and Hamel, Philippe and Gergely, Anita and Comanici, Gheorghe and Glaese, Amelia and Ahmed, Zafarali and Jackson, Tyler and Mourad, Shibl and Precup, Doina},
  journal={arXiv preprint arXiv:2105.13231},
  year={2021}
}

@article{wen2023empowering,
  title={Empowering llm to use smartphone for intelligent task automation},
  author={Wen, Hao and Li, Yuanchun and Liu, Guohong and Zhao, Shanhui and Yu, Tao and Li, Toby Jia-Jun and Jiang, Shiqi and Liu, Yunhao and Zhang, Yaqin and Liu, Yunxin},
  journal={arXiv preprint arXiv:2308.15272},
  year={2023}
}

@article{zhang2023appagent,
  title={AppAgent: Multimodal Agents as Smartphone Users},
  author={Zhang, Chi and Yang, Zhao and Liu, Jiaxuan and Han, Yucheng and Chen, Xin and Huang, Zebiao and Fu, Bin and Yu, Gang},
  journal={arXiv e-prints},
  pages={arXiv--2312},
  year={2023}
}
@article{li2020mapping,
  title={Mapping natural language instructions to mobile UI action sequences},
  author={Li, Yang and He, Jiacong and Zhou, Xin and Zhang, Yuan and Baldridge, Jason},
  journal={arXiv preprint arXiv:2005.03776},
  year={2020}
}

@article{sun2022meta,
  title={META-GUI: Towards Multi-modal Conversational Agents on Mobile GUI},
  author={Sun, Liangtai and Chen, Xingyu and Chen, Lu and Dai, Tianle and Zhu, Zichen and Yu, Kai},
  journal={arXiv preprint arXiv:2205.11029},
  year={2022}
}

@article{venkatesh2022ugif,
  title={UGIF: UI Grounded Instruction Following},
  author={Venkatesh, Sagar Gubbi and Talukdar, Partha and Narayanan, Srini},
  journal={arXiv preprint arXiv:2211.07615},
  year={2022}
}

@inproceedings{andreas2016neural,
  title={Neural module networks},
  author={Andreas, Jacob and Rohrbach, Marcus and Darrell, Trevor and Klein, Dan},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={39--48},
  year={2016}
}

@article{cheng2022binding,
  title={Binding language models in symbolic languages},
  author={Cheng, Zhoujun and Xie, Tianbao and Shi, Peng and Li, Chengzu and Nadkarni, Rahul and Hu, Yushi and Xiong, Caiming and Radev, Dragomir and Ostendorf, Mari and Zettlemoyer, Luke and others},
  journal={arXiv preprint arXiv:2210.02875},
  year={2022}
}

@inproceedings{gupta2023visual,
  title={Visual programming: Compositional visual reasoning without training},
  author={Gupta, Tanmay and Kembhavi, Aniruddha},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={14953--14962},
  year={2023}
}

@article{suris2023vipergpt,
  title={Vipergpt: Visual inference via python execution for reasoning},
  author={Sur{\'\i}s, D{\'\i}dac and Menon, Sachit and Vondrick, Carl},
  journal={arXiv preprint arXiv:2303.08128},
  year={2023}
}

@article{wu2023visual,
  title={Visual chatgpt: Talking, drawing and editing with visual foundation models},
  author={Wu, Chenfei and Yin, Shengming and Qi, Weizhen and Wang, Xiaodong and Tang, Zecheng and Duan, Nan},
  journal={arXiv preprint arXiv:2303.04671},
  year={2023}
}

@article{furuta2023multimodal,
  title={Multimodal Web Navigation with Instruction-Finetuned Foundation Models},
  author={Furuta, Hiroki and Nachum, Ofir and Lee, Kuang-Huei and Matsuo, Yutaka and Gu, Shixiang Shane and Gur, Izzeddin},
  journal={arXiv preprint arXiv:2305.11854},
  year={2023}
}

@inproceedings{lee2023pix2struct,
  title={Pix2struct: Screenshot parsing as pretraining for visual language understanding},
  author={Lee, Kenton and Joshi, Mandar and Turc, Iulia Raluca and Hu, Hexiang and Liu, Fangyu and Eisenschlos, Julian Martin and Khandelwal, Urvashi and Shaw, Peter and Chang, Ming-Wei and Toutanova, Kristina},
  booktitle={International Conference on Machine Learning},
  pages={18893--18912},
  year={2023},
  organization={PMLR}
}

@article{shaw2023pixels,
  title={From Pixels to UI Actions: Learning to Follow Instructions via Graphical User Interfaces},
  author={Shaw, Peter and Joshi, Mandar and Cohan, James and Berant, Jonathan and Pasupat, Panupong and Hu, Hexiang and Khandelwal, Urvashi and Lee, Kenton and Toutanova, Kristina},
  journal={arXiv preprint arXiv:2306.00245},
  year={2023}
}

@article{gur2023real,
  title={A real-world webagent with planning, long context understanding, and program synthesis},
  author={Gur, Izzeddin and Furuta, Hiroki and Huang, Austin and Safdari, Mustafa and Matsuo, Yutaka and Eck, Douglas and Faust, Aleksandra},
  journal={arXiv preprint arXiv:2307.12856},
  year={2023}
}

@article{nakano2021webgpt,
  title={Webgpt: Browser-assisted question-answering with human feedback},
  author={Nakano, Reiichiro and Hilton, Jacob and Balaji, Suchir and Wu, Jeff and Ouyang, Long and Kim, Christina and Hesse, Christopher and Jain, Shantanu and Kosaraju, Vineet and Saunders, William and others},
  journal={arXiv preprint arXiv:2112.09332},
  year={2021}
}

@inproceedings{wang2023enabling,
  title={Enabling conversational interaction with mobile ui using large language models},
  author={Wang, Bryan and Li, Gang and Li, Yang},
  booktitle={Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems},
  pages={1--17},
  year={2023}
}

@inproceedings{humphreys2022data,
  title={A data-driven approach for learning to control computers},
  author={Humphreys, Peter C and Raposo, David and Pohlen, Tobias and Thornton, Gregory and Chhaparia, Rachita and Muldal, Alistair and Abramson, Josh and Georgiev, Petko and Santoro, Adam and Lillicrap, Timothy},
  booktitle={International Conference on Machine Learning},
  pages={9466--9482},
  year={2022},
  organization={PMLR}
}

@article{sridhar2023hierarchical,
  title={Hierarchical Prompting Assists Large Language Model on Web Navigation},
  author={Sridhar, Abishek and Lo, Robert and Xu, Frank F and Zhu, Hao and Zhou, Shuyan},
  journal={arXiv preprint arXiv:2305.14257},
  year={2023}
}

@inproceedings{MirnaNachouki2019ISSPIT_RPAAcademicAdvising,
  author       = {Mirna Nachouki and
                  Mahmoud Abou Naaj},
  title        = {Process Automation Tool for Academic Advising},
  booktitle    = {{IEEE} International Symposium on Signal Processing and Information
                  Technology, {ISSPIT} 2019, Ajman, United Arab Emirates, December 10-12,
                  2019},
  pages        = {1--6},
  publisher    = {{IEEE}},
  year         = {2019},
  url          = {https://doi.org/10.1109/ISSPIT47144.2019.9001864},
  doi          = {10.1109/ISSPIT47144.2019.9001864},
  timestamp    = {Tue, 25 Feb 2020 17:16:38 +0100},
  biburl       = {https://dblp.org/rec/conf/isspit/NachoukiN19.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{JulianKoch2020ECIS_MirrorOnTheWall,
  author       = {Julian Koch and
                  Michael Trampler and
                  Ingo Kregel and
                  Andr{\'{e}} Coners},
  editor       = {Frantz Rowe and
                  Redouane El Amrani and
                  Moez Limayem and
                  Sue Newell and
                  Nancy Pouloudi and
                  Eric van Heck and
                  Ali El Quammah},
  title        = {'mirror, mirror, on the Wall': robotic Process Automation in the Public
                  Sector using a Digital twin},
  booktitle    = {28th European Conference on Information Systems - Liberty, Equality,
                  and Fraternity in a Digitizing World, {ECIS} 2020, Marrakech, Morocco,
                  June 15-17, 2020},
  year         = {2020},
  url          = {https://aisel.aisnet.org/ecis2020\_rip/2},
  timestamp    = {Wed, 10 Jun 2020 12:03:24 +0200},
  biburl       = {https://dblp.org/rec/conf/ecis/KochTKC20.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{ZelleM96,
  author    = {John M. Zelle and
               Raymond J. Mooney},
  title     = {Learning to Parse Database Queries Using Inductive Logic Programming},
  booktitle = {{AAAI} 1996},
  pages     = {1050--1055},
  year      = {1996},
  timestamp = {Wed, 10 Feb 2021 08:44:51 +0100},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{Zettlemoyer05,
  author    = {Luke S. Zettlemoyer and Michael Collins},
  title     = {Learning to Map Sentences to Logical Form: Structured Classification with Probabilistic Categorial Grammars},
  journal   = {UAI},
  year      = {2005},
}

@inproceedings{berant-etal-2013-semantic,
    title = "Semantic Parsing on {F}reebase from Question-Answer Pairs",
    author = "Berant, Jonathan  and
      Chou, Andrew  and
      Frostig, Roy  and
      Liang, Percy",
    booktitle = "Proc.\ of EMNLP",
    year = "2013",
    url = "https://aclanthology.org/D13-1160",
}

@inproceedings{yin-neubig-2017-syntactic,
    title = "A Syntactic Neural Model for General-Purpose Code Generation",
    author = "Yin, Pengcheng  and
      Neubig, Graham",
    booktitle = "Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    month = jul,
    year = "2017",
    address = "Vancouver, Canada",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/P17-1041",
    doi = "10.18653/v1/P17-1041",
    pages = "440--450",
    abstract = "We consider the problem of parsing natural language descriptions into source code written in a general-purpose programming language like Python. Existing data-driven methods treat this problem as a language generation task without considering the underlying syntax of the target programming language. Informed by previous work in semantic parsing, in this paper we propose a novel neural architecture powered by a grammar model to explicitly capture the target syntax as prior knowledge. Experiments find this an effective way to scale up to generation of complex programs from natural language descriptions, achieving state-of-the-art results that well outperform previous code generation and semantic parsing approaches.",
}

@inproceedings{TobyJiaJunLI2018Mobisys_Kite,
  author       = {Toby Jia{-}Jun Li and
                  Oriana Riva},
  editor       = {J{\"{o}}rg Ott and
                  Falko Dressler and
                  Stefan Saroiu and
                  Prabal Dutta},
  title        = {Kite: Building Conversational Bots from Mobile Apps},
  booktitle    = {Proceedings of the 16th Annual International Conference on Mobile
                  Systems, Applications, and Services, MobiSys 2018, Munich, Germany,
                  June 10-15, 2018},
  pages        = {96--109},
  publisher    = {{ACM}},
  year         = {2018},
  url          = {https://doi.org/10.1145/3210240.3210339},
  doi          = {10.1145/3210240.3210339},
  timestamp    = {Thu, 14 Oct 2021 09:56:16 +0200},
  biburl       = {https://dblp.org/rec/conf/mobisys/LiR18.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{TobyJiaJunLi2019_PUMICE,
  author       = {Toby Jia{-}Jun Li and
                  Marissa Radensky and
                  Justin Jia and
                  Kirielle Singarajah and
                  Tom M. Mitchell and
                  Brad A. Myers},
  editor       = {Fran{\c{c}}ois Guimbreti{\`{e}}re and
                  Michael S. Bernstein and
                  Katharina Reinecke},
  title        = {{PUMICE:} {A} Multi-Modal Agent that Learns Concepts and Conditionals
                  from Natural Language and Demonstrations},
  booktitle    = {Proceedings of the 32nd Annual {ACM} Symposium on User Interface Software
                  and Technology, {UIST} 2019, New Orleans, LA, USA, October 20-23,
                  2019},
  pages        = {577--589},
  publisher    = {{ACM}},
  year         = {2019},
  url          = {https://doi.org/10.1145/3332165.3347899},
  doi          = {10.1145/3332165.3347899},
  timestamp    = {Fri, 04 Feb 2022 08:27:42 +0100},
  biburl       = {https://dblp.org/rec/conf/uist/LiRJSMM19.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{TobyJiaJunLi2020ACLDemo_SUGILITE,
  author       = {Toby Jia{-}Jun Li and
                  Tom M. Mitchell and
                  Brad A. Myers},
  editor       = {Asli Celikyilmaz and
                  Tsung{-}Hsien Wen},
  title        = {Interactive Task Learning from GUI-Grounded Natural Language Instructions
                  and Demonstrations},
  booktitle    = {Proceedings of the 58th Annual Meeting of the Association for Computational
                  Linguistics: System Demonstrations, {ACL} 2020, Online, July 5-10,
                  2020},
  pages        = {215--223},
  publisher    = {Association for Computational Linguistics},
  year         = {2020},
  url          = {https://doi.org/10.18653/v1/2020.acl-demos.25},
  doi          = {10.18653/V1/2020.ACL-DEMOS.25},
  timestamp    = {Fri, 06 Aug 2021 00:41:00 +0200},
  biburl       = {https://dblp.org/rec/conf/acl/LiMM20.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{yu2018spider,
  title={Spider: A Large-Scale Human-Labeled Dataset for Complex and Cross-Domain Semantic Parsing and Text-to-SQL Task},
  author={Yu, Tao and Zhang, Rui and Yang, Kai and Yasunaga, Michihiro and Wang, Dongxu and Li, Zifan and Ma, James and Li, Irene and Yao, Qingning and Roman, Shanelle and others},
  booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},
  pages={3911--3921},
  year={2018}
}

@article{zhong2018seq2sql,
  title={Seq2SQL: Generating Structured Queries From Natural Language Using Reinforcement Learning},
  author={Zhong, Victor and Xiong, Caiming and Socher, Richard},
  year={2018}
}

@inproceedings{scholak2021picard,
    title = "{PICARD}: Parsing Incrementally for Constrained Auto-Regressive Decoding from Language Models",
    author = "Scholak, Torsten  and
      Schucher, Nathan  and
      Bahdanau, Dzmitry",
    booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing",
    month = nov,
    year = "2021",
    address = "Online and Punta Cana, Dominican Republic",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.emnlp-main.779",
    doi = "10.18653/v1/2021.emnlp-main.779",
    pages = "9895--9901",
    abstract = "Large pre-trained language models for textual data have an unconstrained output space; at each decoding step, they can produce any of 10,000s of sub-word tokens. When fine-tuned to target constrained formal languages like SQL, these models often generate invalid code, rendering it unusable. We propose PICARD (code available at https://github.com/ElementAI/picard), a method for constraining auto-regressive decoders of language models through incremental parsing. PICARD helps to find valid output sequences by rejecting inadmissible tokens at each decoding step. On the challenging Spider and CoSQL text-to-SQL translation tasks, we show that PICARD transforms fine-tuned T5 models with passable performance into state-of-the-art solutions.",
}

@inproceedings{shaw-etal-2021-compositional,
    title = "Compositional Generalization and Natural Language Variation: Can a Semantic Parsing Approach Handle Both?",
    author = "Shaw, Peter  and
      Chang, Ming-Wei  and
      Pasupat, Panupong  and
      Toutanova, Kristina",
    booktitle = "Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)",
    month = aug,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
}

@article{zhang2023you,
  title={You Only Look at Screens: Multimodal Chain-of-Action Agents},
  author={Zhang, Zhuosheng and Zhang, Aston},
  journal={arXiv e-prints},
  pages={arXiv--2309},
  year={2023}
}

@article{hong2023cogagent,
  title={CogAgent: A Visual Language Model for GUI Agents},
  author={Hong, Wenyi and Wang, Weihan and Lv, Qingsong and Xu, Jiazheng and Yu, Wenmeng and Ji, Junhui and Wang, Yan and Wang, Zihan and Dong, Yuxiao and Ding, Ming and others},
  journal={arXiv preprint arXiv:2312.08914},
  year={2023}
}

@article{liu2023visual,
  title={Visual instruction tuning},
  author={Liu, Haotian and Li, Chunyuan and Wu, Qingyang and Lee, Yong Jae},
  journal={arXiv preprint arXiv:2304.08485},
  year={2023}
}

@article{zhu2023minigpt,
  title={Minigpt-4: Enhancing vision-language understanding with advanced large language models},
  author={Zhu, Deyao and Chen, Jun and Shen, Xiaoqian and Li, Xiang and Elhoseiny, Mohamed},
  journal={arXiv preprint arXiv:2304.10592},
  year={2023}
}

@article{team2023gemini,
  title={Gemini: a family of highly capable multimodal models},
  author={Team, Gemini and Anil, Rohan and Borgeaud, Sebastian and Wu, Yonghui and Alayrac, Jean-Baptiste and Yu, Jiahui and Soricut, Radu and Schalkwyk, Johan and Dai, Andrew M and Hauth, Anja and others},
  journal={arXiv preprint arXiv:2312.11805},
  year={2023}
}

@article{DeborahFerreira2020_RealWoB,
  author       = {Deborah Ferreira and
                  Julia Rozanova and
                  Krishna Dubba and
                  Dell Zhang and
                  Andr{\'{e}} Freitas},
  title        = {On the Evaluation of Intelligence Process Automation},
  journal      = {CoRR},
  volume       = {abs/2001.02639},
  year         = {2020},
  url          = {http://arxiv.org/abs/2001.02639},
  eprinttype    = {arXiv},
  eprint       = {2001.02639},
  timestamp    = {Thu, 28 May 2020 10:01:25 +0200},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2001-02639.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{BryanWang2023ACMCHI_LLMMobileInteraction,
  author       = {Bryan Wang and
                  Gang Li and
                  Yang Li},
  editor       = {Albrecht Schmidt and
                  Kaisa V{\"{a}}{\"{a}}n{\"{a}}nen and
                  Tesh Goyal and
                  Per Ola Kristensson and
                  Anicia Peters and
                  Stefanie Mueller and
                  Julie R. Williamson and
                  Max L. Wilson},
  title        = {Enabling Conversational Interaction with Mobile {UI} using Large Language
                  Models},
  booktitle    = {Proceedings of the 2023 {CHI} Conference on Human Factors in Computing
                  Systems, {CHI} 2023, Hamburg, Germany, April 23-28, 2023},
  pages        = {432:1--432:17},
  publisher    = {{ACM}},
  year         = {2023},
  url          = {https://doi.org/10.1145/3544548.3580895},
  doi          = {10.1145/3544548.3580895},
  timestamp    = {Thu, 01 Jun 2023 13:14:33 +0200},
  biburl       = {https://dblp.org/rec/conf/chi/WangLL23.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{ShunyuYao2023ICLR_ReAct,
  author       = {Shunyu Yao and
                  Jeffrey Zhao and
                  Dian Yu and
                  Nan Du and
                  Izhak Shafran and
                  Karthik R. Narasimhan and
                  Yuan Cao},
  title        = {ReAct: Synergizing Reasoning and Acting in Language Models},
  booktitle    = {The Eleventh International Conference on Learning Representations,
                  {ICLR} 2023, Kigali, Rwanda, May 1-5, 2023},
  publisher    = {OpenReview.net},
  year         = {2023},
  url          = {https://openreview.net/pdf?id=WE\_vluYUL-X},
  timestamp    = {Wed, 16 Aug 2023 16:09:26 +0200},
  biburl       = {https://dblp.org/rec/conf/iclr/YaoZYDSN023.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{IzzeddinGur2023_WebAgent,
  author       = {Izzeddin Gur and
                  Hiroki Furuta and
                  Austin Huang and
                  Mustafa Safdari and
                  Yutaka Matsuo and
                  Douglas Eck and
                  Aleksandra Faust},
  title        = {A Real-World WebAgent with Planning, Long Context Understanding, and
                  Program Synthesis},
  journal      = {CoRR},
  volume       = {abs/2307.12856},
  year         = {2023},
  url          = {https://doi.org/10.48550/arXiv.2307.12856},
  doi          = {10.48550/ARXIV.2307.12856},
  eprinttype    = {arXiv},
  eprint       = {2307.12856},
  timestamp    = {Tue, 01 Aug 2023 14:49:51 +0200},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2307-12856.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{AmanMadaan2022EMNLP_MemPrompt,
  author       = {Aman Madaan and
                  Niket Tandon and
                  Peter Clark and
                  Yiming Yang},
  editor       = {Yoav Goldberg and
                  Zornitsa Kozareva and
                  Yue Zhang},
  title        = {Memory-assisted prompt editing to improve {GPT-3} after deployment},
  booktitle    = {Proceedings of the 2022 Conference on Empirical Methods in Natural
                  Language Processing, {EMNLP} 2022, Abu Dhabi, United Arab Emirates,
                  December 7-11, 2022},
  pages        = {2833--2861},
  publisher    = {Association for Computational Linguistics},
  year         = {2022},
  url          = {https://doi.org/10.18653/v1/2022.emnlp-main.183},
  doi          = {10.18653/V1/2022.EMNLP-MAIN.183},
  timestamp    = {Thu, 10 Aug 2023 12:35:25 +0200},
  biburl       = {https://dblp.org/rec/conf/emnlp/MadaanTCY22.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{DanyangZhang2023_Rememberer,
  author       = {Danyang Zhang and
                  Lu Chen and
                  Situo Zhang and
                  Hongshen Xu and
                  Zihan Zhao and
                  Kai Yu},
  title        = {Large Language Model Is Semi-Parametric Reinforcement Learning Agent},
  journal      = {CoRR},
  volume       = {abs/2306.07929},
  year         = {2023},
  url          = {https://doi.org/10.48550/arXiv.2306.07929},
  doi          = {10.48550/ARXIV.2306.07929},
  eprinttype    = {arXiv},
  eprint       = {2306.07929},
  timestamp    = {Sat, 17 Jun 2023 18:52:05 +0200},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2306-07929.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{AndrewZhao2023_ExpeL,
  author       = {Andrew Zhao and
                  Daniel Huang and
                  Quentin Xu and
                  Matthieu Lin and
                  Yong{-}Jin Liu and
                  Gao Huang},
  title        = {ExpeL: {LLM} Agents Are Experiential Learners},
  journal      = {CoRR},
  volume       = {abs/2308.10144},
  year         = {2023},
  url          = {https://doi.org/10.48550/arXiv.2308.10144},
  doi          = {10.48550/ARXIV.2308.10144},
  eprinttype    = {arXiv},
  eprint       = {2308.10144},
  timestamp    = {Wed, 30 Aug 2023 15:34:14 +0200},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2308-10144.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{ReiichiroNakano2021_WebGPT,
  author       = {Reiichiro Nakano and
                  Jacob Hilton and
                  Suchir Balaji and
                  Jeff Wu and
                  Long Ouyang and
                  Christina Kim and
                  Christopher Hesse and
                  Shantanu Jain and
                  Vineet Kosaraju and
                  William Saunders and
                  Xu Jiang and
                  Karl Cobbe and
                  Tyna Eloundou and
                  Gretchen Krueger and
                  Kevin Button and
                  Matthew Knight and
                  Benjamin Chess and
                  John Schulman},
  title        = {WebGPT: Browser-assisted question-answering with human feedback},
  journal      = {CoRR},
  volume       = {abs/2112.09332},
  year         = {2021},
  url          = {https://arxiv.org/abs/2112.09332},
  eprinttype    = {arXiv},
  eprint       = {2112.09332},
  timestamp    = {Mon, 03 Jan 2022 15:45:35 +0100},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2112-09332.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{YongliangShen2023_HuggingGPT,
  author       = {Yongliang Shen and
                  Kaitao Song and
                  Xu Tan and
                  Dongsheng Li and
                  Weiming Lu and
                  Yueting Zhuang},
  title        = {HuggingGPT: Solving {AI} Tasks with ChatGPT and its Friends in HuggingFace},
  journal      = {CoRR},
  volume       = {abs/2303.17580},
  year         = {2023},
  url          = {https://doi.org/10.48550/arXiv.2303.17580},
  doi          = {10.48550/ARXIV.2303.17580},
  eprinttype    = {arXiv},
  eprint       = {2303.17580},
  timestamp    = {Thu, 04 May 2023 17:10:57 +0200},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2303-17580.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{TimoSchick2023_Toolformer,
  author       = {Timo Schick and
                  Jane Dwivedi{-}Yu and
                  Roberto Dess{\`{\i}} and
                  Roberta Raileanu and
                  Maria Lomeli and
                  Luke Zettlemoyer and
                  Nicola Cancedda and
                  Thomas Scialom},
  title        = {Toolformer: Language Models Can Teach Themselves to Use Tools},
  journal      = {CoRR},
  volume       = {abs/2302.04761},
  year         = {2023},
  url          = {https://doi.org/10.48550/arXiv.2302.04761},
  doi          = {10.48550/ARXIV.2302.04761},
  eprinttype    = {arXiv},
  eprint       = {2302.04761},
  timestamp    = {Mon, 13 Feb 2023 14:23:40 +0100},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2302-04761.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{TianbaoXie2023_OpenAgents,
  author       = {Tianbao Xie and
                  Fan Zhou and
                  Zhoujun Cheng and
                  Peng Shi and
                  Luoxuan Weng and
                  Yitao Liu and
                  Toh Jing Hua and
                  Junning Zhao and
                  Qian Liu and
                  Che Liu and
                  Leo Z. Liu and
                  Yiheng Xu and
                  Hongjin Su and
                  Dongchan Shin and
                  Caiming Xiong and
                  Tao Yu},
  title        = {OpenAgents: An Open Platform for Language Agents in the Wild},
  journal      = {CoRR},
  volume       = {abs/2310.10634},
  year         = {2023},
  url          = {https://doi.org/10.48550/arXiv.2310.10634},
  doi          = {10.48550/ARXIV.2310.10634},
  eprinttype    = {arXiv},
  eprint       = {2310.10634},
  timestamp    = {Wed, 06 Dec 2023 16:22:07 +0100},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2310-10634.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{li2023silkie,
  title={Silkie: Preference Distillation for Large Visual Language Models},
  author={Li, Lei and Xie, Zhihui and Li, Mukai and Chen, Shunian and Wang, Peiyi and Chen, Liang and Yang, Yazheng and Wang, Benyou and Kong, Lingpeng},
  journal={arXiv preprint arXiv:2312.10665},
  year={2023}
}

@article{mackay2019state,
  title={The State of Work Life Balance in 2019: What we learned from studying 185 million hours of working time},
  author={MacKay, J},
  journal={RescueTime: blog},
  year={2019}
}

@article{brown2020language,
  title={Language models are few-shot learners},
  author={Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and others},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={1877--1901},
  year={2020}
}

@article{chen2021evaluating,
  title={Evaluating large language models trained on code},
  author={Chen, Mark and Tworek, Jerry and Jun, Heewoo and Yuan, Qiming and Pinto, Henrique Ponde de Oliveira and Kaplan, Jared and Edwards, Harri and Burda, Yuri and Joseph, Nicholas and Brockman, Greg and others},
  journal={arXiv preprint arXiv:2107.03374},
  year={2021}
}

@article{chowdhery2022palm,
  title={Palm: Scaling language modeling with pathways},
  author={Chowdhery, Aakanksha and Narang, Sharan and Devlin, Jacob and Bosma, Maarten and Mishra, Gaurav and Roberts, Adam and Barham, Paul and Chung, Hyung Won and Sutton, Charles and Gehrmann, Sebastian and others},
  journal={arXiv preprint arXiv:2204.02311},
  year={2022}
}

@misc{touvronLlamaOpenFoundation2023,
  title = {Llama 2: {{Open Foundation}} and {{Fine-Tuned Chat Models}}},
  shorttitle = {Llama 2},
  author = {Touvron, Hugo and Martin, Louis and Stone, Kevin and Albert, Peter and Almahairi, Amjad and Babaei, Yasmine and Bashlykov, Nikolay and Batra, Soumya and Bhargava, Prajjwal and Bhosale, Shruti and Bikel, Dan and Blecher, Lukas and Ferrer, Cristian Canton and Chen, Moya and Cucurull, Guillem and Esiobu, David and Fernandes, Jude and Fu, Jeremy and Fu, Wenyin and Fuller, Brian and Gao, Cynthia and Goswami, Vedanuj and Goyal, Naman and Hartshorn, Anthony and Hosseini, Saghar and Hou, Rui and Inan, Hakan and Kardas, Marcin and Kerkez, Viktor and Khabsa, Madian and Kloumann, Isabel and Korenev, Artem and Koura, Punit Singh and Lachaux, Marie-Anne and Lavril, Thibaut and Lee, Jenya and Liskovich, Diana and Lu, Yinghai and Mao, Yuning and Martinet, Xavier and Mihaylov, Todor and Mishra, Pushkar and Molybog, Igor and Nie, Yixin and Poulton, Andrew and Reizenstein, Jeremy and Rungta, Rashi and Saladi, Kalyan and Schelten, Alan and Silva, Ruan and Smith, Eric Michael and Subramanian, Ranjan and Tan, Xiaoqing Ellen and Tang, Binh and Taylor, Ross and Williams, Adina and Kuan, Jian Xiang and Xu, Puxin and Yan, Zheng and Zarov, Iliyan and Zhang, Yuchen and Fan, Angela and Kambadur, Melanie and Narang, Sharan and Rodriguez, Aurelien and Stojnic, Robert and Edunov, Sergey and Scialom, Thomas},
  year = {2023},
  month = jul,
  number = {arXiv:2307.09288},
  eprint = {2307.09288},
  primaryclass = {cs},
  publisher = {{arXiv}},
  doi = {10.48550/arXiv.2307.09288},
  urldate = {2023-09-28},
  abstract = {In this work, we develop and release Llama 2, a collection of pretrained and fine-tuned large language models (LLMs) ranging in scale from 7 billion to 70 billion parameters. Our fine-tuned LLMs, called Llama 2-Chat, are optimized for dialogue use cases. Our models outperform open-source chat models on most benchmarks we tested, and based on our human evaluations for helpfulness and safety, may be a suitable substitute for closed-source models. We provide a detailed description of our approach to fine-tuning and safety improvements of Llama 2-Chat in order to enable the community to build on our work and contribute to the responsible development of LLMs.},
  archiveprefix = {arxiv},
  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computation and Language},
  file = {/Users/ranpox/Zotero/storage/AUQI42X4/Touvron et al. - 2023 - Llama 2 Open Foundation and Fine-Tuned Chat Models.pdf;/Users/ranpox/Zotero/storage/WGJ69RTM/2307.html}
}

@incollection{engelbart2023augmenting,
  title={Augmenting human intellect: A conceptual framework},
  author={Engelbart, Douglas C},
  booktitle={Augmented Education in the Global Age},
  pages={13--29},
  year={2023},
  publisher={Routledge}
}

@misc{adept2022act1,
    title = {{ACT-1: Transformer for Actions}},
    howpublished = {\url{https://www.adept.ai/act}},
    year = {2022},
    author = {{Adept}}
}

@ARTICLE{licklider1960man,
  author={Licklider, J. C. R.},
  journal={IRE Transactions on Human Factors in Electronics}, 
  title={Man-Computer Symbiosis}, 
  year={1960},
  volume={HFE-1},
  number={1},
  pages={4-11},
  keywords={Symbiosis;Insects;Time sharing computer systems;Performance evaluation;Performance analysis;Computer languages},
  doi={10.1109/THFE2.1960.4503259}}


@inproceedings{sarsenbayeva2018situational,
  title={Situational impairments during mobile interaction},
  author={Sarsenbayeva, Zhanna},
  booktitle={Proceedings of the 2018 ACM International Joint Conference and 2018 International Symposium on Pervasive and Ubiquitous Computing and Wearable Computers},
  pages={498--503},
  year={2018}
}

@misc{stallbaumer2023introducing,
  title={Introducing Microsoft 365 Copilot—a whole new way to work},
  author={Stallbaumer, Colette},
  year={2023},
  publisher={Microsoft}
}

@article{liang2023taskmatrix,
  title={Taskmatrix. ai: Completing tasks by connecting foundation models with millions of apis},
  author={Liang, Yaobo and Wu, Chenfei and Song, Ting and Wu, Wenshan and Xia, Yan and Liu, Yu and Ou, Yang and Lu, Shuai and Ji, Lei and Mao, Shaoguang and others},
  journal={arXiv preprint arXiv:2303.16434},
  year={2023}
}

@article{alayrac2022flamingo,
  title={Flamingo: a visual language model for few-shot learning},
  author={Alayrac, Jean-Baptiste and Donahue, Jeff and Luc, Pauline and Miech, Antoine and Barr, Iain and Hasson, Yana and Lenc, Karel and Mensch, Arthur and Millican, Katherine and Reynolds, Malcolm and others},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={23716--23736},
  year={2022}
}

@article{li2023blip,
  title={Blip-2: Bootstrapping language-image pre-training with frozen image encoders and large language models},
  author={Li, Junnan and Li, Dongxu and Savarese, Silvio and Hoi, Steven},
  journal={arXiv preprint arXiv:2301.12597},
  year={2023}
}

@article{openai2023gpt,
  title={Gpt-4 technical report. arxiv 2303.08774},
  author={OpenAI, R},
  journal={View in Article},
  volume={2},
  pages={13},
  year={2023}
}

@article{yang2023set,
  title={Set-of-mark prompting unleashes extraordinary visual grounding in gpt-4v},
  author={Yang, Jianwei and Zhang, Hao and Li, Feng and Zou, Xueyan and Li, Chunyuan and Gao, Jianfeng},
  journal={arXiv preprint arXiv:2310.11441},
  year={2023}
}

@inproceedings{song2023llm,
  title={Llm-planner: Few-shot grounded planning for embodied agents with large language models},
  author={Song, Chan Hee and Wu, Jiaman and Washington, Clayton and Sadler, Brian M and Chao, Wei-Lun and Su, Yu},
  booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
  pages={2998--3009},
  year={2023}
}

@article{zheng2024gpt,
  title={GPT-4V (ision) is a Generalist Web Agent, if Grounded},
  author={Zheng, Boyuan and Gou, Boyu and Kil, Jihyung and Sun, Huan and Su, Yu},
  journal={arXiv preprint arXiv:2401.01614},
  year={2024}
}

@misc{gpt4vact,
  title = {{GPT-4V-Act: GPT-4 Variant for Active Learning}},
  author = {Dupont, D.},
  howpublished = {GitHub repository},
  year = {2023},
  url = {https://github.com/ddupont808/GPT-4V-Act}
}

@article{lin2018nl2bash,
  title={Nl2bash: A corpus and semantic parser for natural language interface to the linux operating system},
  author={Lin, Xi Victoria and Wang, Chenglong and Zettlemoyer, Luke and Ernst, Michael D},
  journal={arXiv preprint arXiv:1802.08979},
  year={2018}
}

@article{brockman2016openai,
  title={Openai gym},
  author={Brockman, Greg and Cheung, Vicki and Pettersson, Ludwig and Schneider, Jonas and Schulman, intercode and Tang, Jie and Zaremba, Wojciech},
  journal={arXiv preprint arXiv:1606.01540},
  year={2016}
}

@article{server2008using,
  title={Using vmrun to Control Virtual Machines},
  author={Server, VMware},
  year={2008}
}

@book{grinberg2018flask,
  title={Flask web development: developing web applications with python},
  author={Grinberg, Miguel},
  year={2018},
  publisher={" O'Reilly Media, Inc."}
}

@article{liu2023agentbench,
  title={Agentbench: Evaluating llms as agents},
  author={Liu, Xiao and Yu, Hao and Zhang, Hanchen and Xu, Yifan and Lei, Xuanyu and Lai, Hanyu and Gu, Yu and Ding, Hangliang and Men, Kaiwen and Yang, Kejuan and others},
  journal={arXiv preprint arXiv:2308.03688},
  year={2023}
}

@article{ma2024agentboard,
  title={AgentBoard: An Analytical Evaluation Board of Multi-turn LLM Agents},
  author={Ma, Chang and Zhang, Junlei and Zhu, Zhihao and Yang, Cheng and Yang, Yujiu and Jin, Yaohui and Lan, Zhenzhong and Kong, Lingpeng and He, Junxian},
  journal={arXiv preprint arXiv:2401.13178},
  year={2024}
}

@article{yang2023intercode,
  title={InterCode: Standardizing and Benchmarking Interactive Coding with Execution Feedback},
  author={Yang, John and Prabhakar, Akshara and Narasimhan, Karthik and Yao, Shunyu},
  journal={arXiv preprint arXiv:2306.14898},
  year={2023}
}

@article{Shridhar2020ALFWorldAT,
  title={ALFWorld: Aligning Text and Embodied Environments for Interactive Learning},
  author={Mohit Shridhar and Xingdi Yuan and Marc-Alexandre C{\^o}t{\'e} and Yonatan Bisk and Adam Trischler and Matthew J. Hausknecht},
  journal={ArXiv},
  year={2020},
  volume={abs/2010.03768},
  url={https://api.semanticscholar.org/CorpusID:222208810}
}

@article{fan2022minedojo,
  title={Minedojo: Building open-ended embodied agents with internet-scale knowledge},
  author={Fan, Linxi and Wang, Guanzhi and Jiang, Yunfan and Mandlekar, Ajay and Yang, Yuncong and Zhu, Haoyi and Tang, Andrew and Huang, De-An and Zhu, Yuke and Anandkumar, Anima},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={18343--18362},
  year={2022}
}

@inproceedings{shridhar2020alfred,
  title={Alfred: A benchmark for interpreting grounded instructions for everyday tasks},
  author={Shridhar, Mohit and Thomason, Jesse and Gordon, Daniel and Bisk, Yonatan and Han, Winson and Mottaghi, Roozbeh and Zettlemoyer, Luke and Fox, Dieter},
  booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
  pages={10740--10749},
  year={2020}
}

@inproceedings{searles2023empirical,
  title={An Empirical Study \& Evaluation of Modern $\{$CAPTCHAs$\}$},
  author={Searles, Andrew and Nakatsuka, Yoshimichi and Ozturk, Ercan and Paverd, Andrew and Tsudik, Gene and Enkoji, Ai},
  booktitle={32nd USENIX Security Symposium (USENIX Security 23)},
  pages={3081--3097},
  year={2023}
}

@article{lin2023vila,
  title={Vila: On pre-training for visual language models},
  author={Lin, Ji and Yin, Hongxu and Ping, Wei and Lu, Yao and Molchanov, Pavlo and Tao, Andrew and Mao, Huizi and Kautz, Jan and Shoeybi, Mohammad and Han, Song},
  journal={arXiv preprint arXiv:2312.07533},
  year={2023}
}

@article{yue2023mmmu,
  title={Mmmu: A massive multi-discipline multimodal understanding and reasoning benchmark for expert agi},
  author={Yue, Xiang and Ni, Yuansheng and Zhang, Kai and Zheng, Tianyu and Liu, Ruoqi and Zhang, Ge and Stevens, Samuel and Jiang, Dongfu and Ren, Weiming and Sun, Yuxuan and others},
  journal={arXiv preprint arXiv:2311.16502},
  year={2023}
}

@article{mialon2023gaia,
  title={Gaia: a benchmark for general ai assistants},
  author={Mialon, Gr{\'e}goire and Fourrier, Cl{\'e}mentine and Swift, Craig and Wolf, Thomas and LeCun, Yann and Scialom, Thomas},
  journal={arXiv preprint arXiv:2311.12983},
  year={2023}
}

@article{he2024webvoyager,
  title={WebVoyager: Building an End-to-End Web Agent with Large Multimodal Models},
  author={He, Hongliang and Yao, Wenlin and Ma, Kaixin and Yu, Wenhao and Dai, Yong and Zhang, Hongming and Lan, Zhenzhong and Yu, Dong},
  journal={arXiv preprint arXiv:2401.13919},
  year={2024}
}

@article{gao2023assistgui,
  title={ASSISTGUI: Task-Oriented Desktop Graphical User Interface Automation},
  author={Gao, Difei and Ji, Lei and Bai, Zechen and Ouyang, Mingyu and Li, Peiran and Mao, Dongxing and Wu, Qinchen and Zhang, Weichen and Wang, Peiyi and Guo, Xiangwu and others},
  journal={arXiv preprint arXiv:2312.13108},
  year={2023}
}

@inproceedings{puig2018virtualhome,
  title={Virtualhome: Simulating household activities via programs},
  author={Puig, Xavier and Ra, Kevin and Boben, Marko and Li, Jiaman and Wang, Tingwu and Fidler, Sanja and Torralba, Antonio},
  booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
  pages={8494--8502},
  year={2018}
}

@article{wu2023smartplay,
  title={SmartPlay: A Benchmark for LLMs as Intelligent Agents},
  author={Wu, Yue and Tang, Xuan and Mitchell, Tom M and Li, Yuanzhi},
  journal={arXiv preprint arXiv:2310.01557},
  year={2023}
}

@article{wang2022scienceworld,
  title={ScienceWorld: Is your Agent Smarter than a 5th Grader?},
  author={Wang, Ruoyao and Jansen, Peter and C{\^o}t{\'e}, Marc-Alexandre and Ammanabrolu, Prithviraj},
  journal={arXiv preprint arXiv:2203.07540},
  year={2022}
}

@article{chevalier2018babyai,
  title={Babyai: A platform to study the sample efficiency of grounded language learning},
  author={Chevalier-Boisvert, Maxime and Bahdanau, Dzmitry and Lahlou, Salem and Willems, Lucas and Saharia, Chitwan and Nguyen, Thien Huu and Bengio, Yoshua},
  journal={arXiv preprint arXiv:1810.08272},
  year={2018}
}

@inproceedings{li2023behavior,
  title={Behavior-1k: A benchmark for embodied ai with 1,000 everyday activities and realistic simulation},
  author={Li, Chengshu and Zhang, Ruohan and Wong, Josiah and Gokmen, Cem and Srivastava, Sanjana and Mart{\'\i}n-Mart{\'\i}n, Roberto and Wang, Chen and Levine, Gabrael and Lingelbach, Michael and Sun, Jiankai and others},
  booktitle={Conference on Robot Learning},
  pages={80--93},
  year={2023},
  organization={PMLR}
}

@article{guo2023pptc,
  title={PPTC Benchmark: Evaluating Large Language Models for PowerPoint Task Completion},
  author={Guo, Yiduo and Zhang, Zekai and Liang, Yaobo and Zhao, Dongyan and Nan, Duan},
  journal={arXiv preprint arXiv:2311.01767},
  year={2023}
}

@article{Cheng2024SeeClickHG,
  title={Seeclick: Harnessing gui grounding for advanced visual gui agents},
  author={Cheng, Kanzhi and Sun, Qiushi and Chu, Yougang and Xu, Fangzhi and Li, Yantao and Zhang, Jianbing and Wu, Zhiyong},
  journal={arXiv preprint arXiv:2401.10935},
  year={2024}
}

@article{Zheng2024GPT4VisionIA,
  title={GPT-4V(ision) is a Generalist Web Agent, if Grounded},
  author={Boyuan Zheng and Boyu Gou and Jihyung Kil and Huan Sun and Yu Su},
  journal={ArXiv},
  year={2024},
  volume={abs/2401.01614},
  url={https://api.semanticscholar.org/CorpusID:266741821}
}

@article{He2024WebVoyagerBA,
  title={WebVoyager: Building an End-to-End Web Agent with Large Multimodal Models},
  author={Hongliang He and Wenlin Yao and Kaixin Ma and Wenhao Yu and Yong Dai and Hongming Zhang and Zhenzhong Lan and Dong Yu},
  journal={ArXiv},
  year={2024},
  volume={abs/2401.13919},
  url={https://api.semanticscholar.org/CorpusID:267211622}
}

@article{Baechler2024ScreenAIAV,
  title={ScreenAI: A Vision-Language Model for UI and Infographics Understanding},
  author={Baechler, Gilles and Sunkara, Srinivas and Wang, Maria and Zubach, Fedir and Mansoor, Hassan and Etter, Vincent and C{\u{a}}rbune, Victor and Lin, Jason and Chen, Jindong and Sharma, Abhanshu},
  journal={arXiv preprint arXiv:2402.04615},
  year={2024}
}

@article{Kapoor2024OmniACTAD,
  title={OmniACT: A Dataset and Benchmark for Enabling Multimodal Generalist Autonomous Agents for Desktop and Web},
  author={Kapoor, Raghav and Butala, Yash Parag and Russak, Melisa and Koh, Jing Yu and Kamble, Kiran and Alshikh, Waseem and Salakhutdinov, Ruslan},
  journal={arXiv preprint arXiv:2402.17553},
  year={2024}
}

@article{Koh2024VisualWebArenaEM,
  title={Visualwebarena: Evaluating multimodal agents on realistic visual web tasks},
  author={Koh, Jing Yu and Lo, Robert and Jang, Lawrence and Duvvur, Vikram and Lim, Ming Chong and Huang, Po-Yu and Neubig, Graham and Zhou, Shuyan and Salakhutdinov, Ruslan and Fried, Daniel},
  journal={arXiv preprint arXiv:2401.13649},
  year={2024}
}

@article{Zhang2024UFOAU,
  title={UFO: A UI-Focused Agent for Windows OS Interaction},
  author={Zhang, Chaoyun and Li, Liqun and He, Shilin and Zhang, Xu and Qiao, Bo and Qin, Si and Ma, Minghua and Kang, Yu and Lin, Qingwei and Rajmohan, Saravan and others},
  journal={arXiv preprint arXiv:2402.07939},
  year={2024}
}

@article{shinn2024reflexion,
  title={Reflexion: Language agents with verbal reinforcement learning},
  author={Shinn, Noah and Cassano, Federico and Gopinath, Ashwin and Narasimhan, Karthik and Yao, Shunyu},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  year={2024}
}

@article{yan2023gpt,
  title={Gpt-4v in wonderland: Large multimodal models for zero-shot smartphone gui navigation},
  author={Yan, An and Yang, Zhengyuan and Zhu, Wanrong and Lin, Kevin and Li, Linjie and Wang, Jianfeng and Yang, Jianwei and Zhong, Yiwu and McAuley, Julian and Gao, Jianfeng and others},
  journal={arXiv preprint arXiv:2311.07562},
  year={2023}
}

@article{lu2024weblinx,
  title={Weblinx: Real-world website navigation with multi-turn dialogue},
  author={L{\`u}, Xing Han and Kasner, Zden{\v{e}}k and Reddy, Siva},
  journal={arXiv preprint arXiv:2402.05930},
  year={2024}
}

@article{jiang2024mixtral,
  title={Mixtral of experts},
  author={Jiang, Albert Q and Sablayrolles, Alexandre and Roux, Antoine and Mensch, Arthur and Savary, Blanche and Bamford, Chris and Chaplot, Devendra Singh and Casas, Diego de las and Hanna, Emma Bou and Bressand, Florian and others},
  journal={arXiv preprint arXiv:2401.04088},
  year={2024}
}

@misc{Anthropic2023Claude,
  author = {{Anthropic}},
  title = {Introducing the next generation of Claude},
  year = {2023},
  howpublished = {\url{https://www.anthropic.com/news/claude-3-family}},
  note = {Accessed: 2024-03-26}
}

@article{niu2024screenagent,
  title={ScreenAgent: A Vision Language Model-driven Computer Control Agent},
  author={Niu, Runliang and Li, Jindong and Wang, Shiqi and Fu, Yali and Hu, Xiyu and Leng, Xueyuan and Kong, He and Chang, Yi and Wang, Qi},
  journal={arXiv preprint arXiv:2402.07945},
  year={2024}
}

@article{zhang2024large,
  title={Large Language Models Are Semi-Parametric Reinforcement Learning Agents},
  author={Zhang, Danyang and Chen, Lu and Zhang, Situo and Xu, Hongshen and Zhao, Zihan and Yu, Kai},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  year={2024}
}

@inproceedings{zhao2022tie,
  title={TIE: Topological Information Enhanced Structural Reading Comprehension on Web Pages},
  author={Zhao, Zihan and Chen, Lu and Cao, Ruisheng and Xu, Hongshen and Chen, Xingyu and Yu, Kai},
  booktitle={Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
  pages={1808--1821},
  year={2022}
}

@article{brohan2023rt,
  title={Rt-2: Vision-language-action models transfer web knowledge to robotic control},
  author={Brohan, Anthony and Brown, Noah and Carbajal, Justice and Chebotar, Yevgen and Chen, Xi and Choromanski, Krzysztof and Ding, Tianli and Driess, Danny and Dubey, Avinava and Finn, Chelsea and others},
  journal={arXiv preprint arXiv:2307.15818},
  year={2023}
}

@article{brohan2022rt,
  title={Rt-1: Robotics transformer for real-world control at scale},
  author={Brohan, Anthony and Brown, Noah and Carbajal, Justice and Chebotar, Yevgen and Dabis, Joseph and Finn, Chelsea and Gopalakrishnan, Keerthana and Hausman, Karol and Herzog, Alex and Hsu, Jasmine and others},
  journal={arXiv preprint arXiv:2212.06817},
  year={2022}
}

@article{jimenez2023swe,
  title={Swe-bench: Can language models resolve real-world github issues?},
  author={Jimenez, Carlos E and Yang, John and Wettig, Alexander and Yao, Shunyu and Pei, Kexin and Press, Ofir and Narasimhan, Karthik},
  journal={arXiv preprint arXiv:2310.06770},
  year={2023}
}

@article{li2024devbench,
  title={DevBench: A Comprehensive Benchmark for Software Development},
  author={Li, Bowen and Wu, Wenhan and Tang, Ziwei and Shi, Lin and Yang, John and Li, Jinyang and Yao, Shunyu and Qian, Chen and Hui, Binyuan and Zhang, Qicheng and others},
  journal={arXiv preprint arXiv:2403.08604},
  year={2024}
}

@article{wang2024mobile,
  title={Mobile-Agent: Autonomous multi-modal mobile device agent with visual perception},
  author={Wang, Junyang and Xu, Haiyang and Ye, Jiabo and Yan, Ming and Shen, Weizhou and Zhang, Ji and Huang, Fei and Sang, Jitao},
  journal={arXiv preprint arXiv:2401.16158},
  year={2024}
}

@article{drouin2024workarena,
  title={WorkArena: How Capable Are Web Agents at Solving Common Knowledge Work Tasks?},
  author={Drouin, Alexandre and Gasse, Maxime and Caccia, Massimo and Laradji, Issam H and Del Verme, Manuel and Marty, Tom and Boisvert, L{\'e}o and Thakkar, Megh and Cappart, Quentin and Vazquez, David and others},
  journal={arXiv preprint arXiv:2403.07718},
  year={2024}
}

@article{reid2024gemini,
  title={Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context},
  author={Reid, Machel and Savinov, Nikolay and Teplyashin, Denis and Lepikhin, Dmitry and Lillicrap, Timothy and Alayrac, Jean-baptiste and Soricut, Radu and Lazaridou, Angeliki and Firat, Orhan and Schrittwieser, Julian and others},
  journal={arXiv preprint arXiv:2403.05530},
  year={2024}
}

@article{bai2023qwen,
  title={Qwen technical report},
  author={Bai, Jinze and Bai, Shuai and Chu, Yunfei and Cui, Zeyu and Dang, Kai and Deng, Xiaodong and Fan, Yang and Ge, Wenbin and Han, Yu and Huang, Fei and others},
  journal={arXiv preprint arXiv:2309.16609},
  year={2023}
}

@misc{meta2024llama3,
  title = {Introducing Meta {Llama 3}: The most capable openly available {LLM} to date},
  author = {{Meta AI}},
  year = {2024},
  month = apr,
  url = {https://ai.meta.com/blog/meta-llama-3/},
  note = {Accessed: 2024-04-18}
}

@article{tan2024towards,
  title={Towards general computer control: A multimodal agent for red dead redemption ii as a case study},
  author={Tan, Weihao and Ding, Ziluo and Zhang, Wentao and Li, Boyu and Zhou, Bohan and Yue, Junpeng and Xia, Haochong and Jiang, Jiechuan and Zheng, Longtao and Xu, Xinrun and others},
  journal={arXiv preprint arXiv:2403.03186},
  year={2024}
}

@article{imagination,
  title={\href{https://arxiv.org/pdf/2404.12253}{Toward Self-Improvement of LLMs via Imagination, Searching, and Criticizing}},
  author={Tian, Ye and Peng, Baolin and Song, Linfeng and Jin, Lifeng and Yu, Dian and Mi, Haitao and Yu, Dong},
  journal={arXiv preprint arXiv:2404.12253},
  year={2024}
}

@article{rl-self-correct,
  title={\href{https://arxiv.org/abs/2409.12917}{Training language models to self-correct via reinforcement learning}},
  author={Kumar, Aviral and Zhuang, Vincent and Agarwal, Rishabh and Su, Yi and Co-Reyes, John D and Singh, Avi and Baumli, Kate and Iqbal, Shariq and Bishop, Colton and Roelofs, Rebecca and others},
  journal={arXiv preprint arXiv:2409.12917},
  year={2024}
}

@article{info-van-roy,
  title={\href{https://arxiv.org/pdf/2407.12288}{Information-Theoretic Foundations for Machine Learning}},
  author={Jeon, Hong Jun and Van Roy, Benjamin},
  journal={arXiv preprint arXiv:2407.12288},
  year={2024}
}

@article{scalable-planning,
  title={\href{https://proceedings.neurips.cc/paper/2021/file/8ce8b102d40392a688f8c04b3cd6cae0-Paper.pdf}{Scalable online planning via reinforcement learning fine-tuning}},
  author={Fickinger, Arnaud and Hu, Hengyuan and Amos, Brandon and Russell, Stuart and Brown, Noam},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={16951--16963},
  year={2021}
}


@article{multi-rlhf,
  title={\href{https://arxiv.org/abs/2405.14655}{Multi-turn Reinforcement Learning from Preference Human Feedback}},
  author={Shani, Lior and Rosenberg, Aviv and Cassel, Asaf and Lang, Oran and Calandriello, Daniele and Zipori, Avital and Noga, Hila and Keller, Orgad and Piot, Bilal and Szpektor, Idan and others},
  journal={arXiv preprint arXiv:2405.14655},
  year={2024}
}

@article{multi-archer,
  title={\href{https://arxiv.org/abs/2402.19446}{Archer: Training language model agents via hierarchical multi-turn rl}},
  author={Zhou, Yifei and Zanette, Andrea and Pan, Jiayi and Levine, Sergey and Kumar, Aviral},
  journal={arXiv preprint arXiv:2402.19446},
  year={2024}
}

@article{multi-recursive,
  title={\href{https://arxiv.org/pdf/2407.18219}{Recursive introspection: Teaching language model agents how to self-improve}},
  author={Qu, Yuxiao and Zhang, Tianjun and Garg, Naman and Kumar, Aviral},
  journal={arXiv preprint arXiv:2407.18219},
  year={2024}
}

@article{rl-rewarding-process,
  title={\href{https://arxiv.org/abs/2410.08146}{Rewarding Progress: Scaling Automated Process Verifiers for LLM Reasoning}},
  author={Setlur, Amrith and Nagpal, Chirag and Fisch, Adam and Geng, Xinyang and Eisenstein, Jacob and Agarwal, Rishabh and Agarwal, Alekh and Berant, Jonathan and Kumar, Aviral},
  journal={arXiv preprint arXiv:2410.08146},
  year={2024}
}

@article{rl-8-fold,
  title={\href{https://arxiv.org/pdf/2406.14532}{RL on Incorrect Synthetic Data Scales the Efficiency of LLM Math Reasoning by Eight-Fold}},
  author={Setlur, Amrith and Garg, Saurabh and Geng, Xinyang and Garg, Naman and Smith, Virginia and Kumar, Aviral},
  journal={arXiv preprint arXiv:2406.14532},
  year={2024}
}

@article{alphago,
  title={\href{https://www.nature.com/articles/nature16961}{Mastering the game of Go with deep neural networks and tree search}},
  author={Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and Van Den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and others},
  journal={nature},
  volume={529},
  number={7587},
  pages={484--489},
  year={2016},
  publisher={Nature Publishing Group}
}

@inproceedings{attention,
 author = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, \L ukasz and Polosukhin, Illia},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {I. Guyon and U. Von Luxburg and S. Bengio and H. Wallach and R. Fergus and S. Vishwanathan and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {\href{https://proceedings.neurips.cc/paper_files/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf}{Attention is All you Need}},
 volume = {30},
 year = {2017}
}

@inproceedings{seq2seq,
 author = {Sutskever, Ilya and Vinyals, Oriol and Le, Quoc V},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {Z. Ghahramani and M. Welling and C. Cortes and N. Lawrence and K.Q. Weinberger},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {\href{https://proceedings.neurips.cc/paper_files/paper/2014/file/a14ac55a4f27472c5d894ec1c3c743d2-Paper.pdf}{Sequence to Sequence Learning with Neural Networks}},
 volume = {27},
 year = {2014}
}


@article{reinforce,
  title={\href{https://proceedings.neurips.cc/paper_files/paper/1999/file/464d828b85b0bed98e80ade0a5c43b0f-Paper.pdf}{Policy gradient methods for reinforcement learning with function approximation}},
  author={Sutton, Richard S and McAllester, David and Singh, Satinder and Mansour, Yishay},
  journal={Advances in neural information processing systems},
  volume={12},
  year={1999}
}


@article{dqn,
  title={\href{https://people.engr.tamu.edu/guni/csce642/files/dqn.pdf}{Playing atari with deep reinforcement learning}},
  author={Silver, David and Graves, Alex and Antonoglou, Ioannis and Riedmiller, Martin and Mnih, Volodymyr and Wierstra, D and Kavukcuoglu, K},
  journal={DeepMind Lab. arXiv},
  volume={1312},
  year={2013}
}

@article{gpt-3,
  title={\href{https://arxiv.org/abs/2005.14165}{Language models are few-shot learners}},
  author={Mann, Ben and Ryder, N and Subbiah, M and Kaplan, J and Dhariwal, P and Neelakantan, A and Shyam, P and Sastry, G and Askell, A and Agarwal, S and others},
  journal={arXiv preprint arXiv:2005.14165},
  volume={1},
  year={2020}
}

@article{gpt-4,
  title={Gpt-4 technical report},
  author={OpenAI and Achiam, Josh and Adler, Steven and Agarwal, Sandhini and Ahmad, Lama and Akkaya, Ilge and Aleman, Florencia Leoni and Almeida, Diogo and Altenschmidt, Janko and Altman, Sam and Anadkat, Shyamal and others},
  journal={arXiv preprint arXiv:2303.08774},
  year={2023}
}


@article{rlhf,
  title={\href{https://arxiv.org/abs/2203.02155}{Training language models to follow instructions with human feedback}},
  author={Ouyang, Long and Wu, Jeffrey and Jiang, Xu and Almeida, Diogo and Wainwright, Carroll and Mishkin, Pamela and Zhang, Chong and Agarwal, Sandhini and Slama, Katarina and Ray, Alex and others},
  journal={Advances in neural information processing systems},
  volume={35},
  pages={27730--27744},
  year={2022}
}


@article{livecodebench,
    title={\href{https://livecodebench.github.io/}{LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code}},
    author={Jain, Naman and Han, King and Gu, Alex and Li, Wen-Ding and Yan, Fanjia and Zhang, Tianjun and Wang, Sida and Solar-Lezama, Armando and Sen, Koushik and Stoica, Ion},
    journal={arXiv preprint arXiv:2403.07974},
    year={2024}
}

@article{silver-abstraction,
  title={\href{https://arxiv.org/abs/1501.03959}{Value iteration with options and state aggregation}},
  author={Ciosek, Kamil and Silver, David},
  journal={arXiv preprint arXiv:1501.03959},
  year={2015}
}

@article{frontier-math,
  title={\href{https://arxiv.org/pdf/2411.04872v1}{FrontierMath: A Benchmark for Evaluating Advanced Mathematical Reasoning in AI}},
  author={Glazer, Elliot and Erdil, Ege and Besiroglu, Tamay and Chicharro, Diego and Chen, Evan and Gunning, Alex and Falkman Olsson, Caroline and Denain, Jean-Stanislas and Ho, Anson and de Oliveira Santos, Emily and others},
  journal={arXiv e-prints},
  pages={arXiv--2411},
  year={2024}
}


@article{test-snell,
  title={\href{https://arxiv.org/abs/2408.03314}{Scaling llm test-time compute optimally can be more effective than scaling model parameters}},
  author={Snell, Charlie and Lee, Jaehoon and Xu, Kelvin and Kumar, Aviral},
  journal={arXiv preprint arXiv:2408.03314},
  year={2024}
}

@inproceedings{math-shepherd,
  title={\href{https://arxiv.org/pdf/2408.03314}{Math-shepherd: Verify and reinforce llms step-by-step without human annotations}},
  author={Wang, Peiyi and Li, Lei and Shao, Zhihong and Xu, Runxin and Dai, Damai and Li, Yifei and Chen, Deli and Wu, Yu and Sui, Zhifang},
  booktitle={Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  pages={9426--9439},
  year={2024}
}

@article{alphazero-llm,
  title={\href{https://arxiv.org/abs/2309.17179}{Alphazero-like tree-search can guide large language model decoding and training}},
  author={Feng, Xidong and Wan, Ziyu and Wen, Muning and McAleer, Stephen Marcus and Wen, Ying and Zhang, Weinan and Wang, Jun},
  journal={arXiv preprint arXiv:2309.17179},
  year={2023}
}

@inproceedings{rir,
    title={\href{https://openreview.net/pdf?id=H5hePMXKht}{Reasoning in Reasoning: A Hierarchical Framework for (Better and Faster) Neural Theorem Proving}},
    author={Ziyu Ye and Jiacheng Chen and Jonathan Light and Yifei Wang and Jiankai Sun and Guohao Li and Mac Schwager and Philip Torr and Yuxin Chen and Kaiyu Yang and Yisong Yue and Ziniu Hu},
    booktitle={The 4th Workshop on Mathematical Reasoning and AI at NeurIPS'24},
    year={2024},
}

@article{strategist,
  title={\href{https://arxiv.org/abs/2408.10635}{Strategist: Learning Strategic Skills by LLMs via Bi-Level Tree Search}},
  author={Light, Jonathan and Cai, Min and Chen, Weiqin and Wang, Guanzhi and Chen, Xiusi and Cheng, Wei and Yue, Yisong and Hu, Ziniu},
  journal={arXiv preprint arXiv:2408.10635},
  year={2024}
}


@article{leanstar,
  title={\href{https://arxiv.org/abs/2407.10040}{Lean-star: Learning to interleave thinking and proving}},
  author={Lin, Haohan and Sun, Zhiqing and Yang, Yiming and Welleck, Sean},
  journal={arXiv preprint arXiv:2407.10040},
  year={2024}
}

@article{rest-mcts,
  title={\href{https://arxiv.org/abs/2406.03816}{Rest-mcts*: Llm self-training via process reward guided tree search}},
  author={Zhang, Dan and Zhoubian, Sining and Hu, Ziniu and Yue, Yisong and Dong, Yuxiao and Tang, Jie},
  journal={arXiv preprint arXiv:2406.03816},
  year={2024}
}


@article{coderl,
  title={\href{https://arxiv.org/abs/2207.01780}{Coderl: Mastering code generation through pretrained models and deep reinforcement learning}},
  author={Le, Hung and Wang, Yue and Gotmare, Akhilesh Deepak and Savarese, Silvio and Hoi, Steven Chu Hong},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={21314--21328},
  year={2022}
}

@article{ppocoder,
  title={\href{https://arxiv.org/pdf/2301.13816}{Execution-based code generation using deep reinforcement learning}},
  author={Shojaee, Parshin and Jain, Aneesh and Tipirneni, Sindhu and Reddy, Chandan K},
  journal={arXiv preprint arXiv:2301.13816},
  year={2023}
}


@article{rltf,
  title={\href{https://arxiv.org/pdf/2307.04349}{RLTF: Reinforcement learning from unit test feedback}},
  author={Liu, Jiate and Zhu, Yiqin and Xiao, Kaiwen and Fu, Qiang and Han, Xiao and Yang, Wei and Ye, Deheng},
  journal={arXiv preprint arXiv:2307.04349},
  year={2023}
}

@article{rlef,
  title={\href{https://arxiv.org/pdf/2410.02089v1}{RLEF: Grounding code llms in execution feedback with reinforcement learning}},
  author={Gehring, Jonas and Zheng, Kunhao and Copet, Jade and Mella, Vegard and Cohen, Taco and Synnaeve, Gabriel},
  journal={arXiv preprint arXiv:2410.02089},
  year={2024}
}

@article{stepcoder,
  title={\href{https://arxiv.org/pdf/2402.01391}{Stepcoder: Improve code generation with reinforcement learning from compiler feedback}},
  author={Dou, Shihan and Liu, Yan and Jia, Haoxiang and Xiong, Limao and Zhou, Enyu and Shen, Wei and Shan, Junjie and Huang, Caishuang and Wang, Xiao and Fan, Xiaoran and others},
  journal={arXiv preprint arXiv:2402.01391},
  year={2024}
}



@article{r2q,
  title={\href{https://arxiv.org/abs/2404.12358}{From r to Q*: Your Language Model is Secretly a Q-Function}},
  author={Rafailov, Rafael and Hejna, Joey and Park, Ryan and Finn, Chelsea},
  journal={arXiv preprint arXiv:2404.12358},
  year={2024}
}

@article{xpo,
  title={\href{https://arxiv.org/pdf/2405.21046}{Exploratory Preference Optimization: Harnessing Implicit Q*-Approximation for Sample-Efficient RLHF}},
  author={Xie, Tengyang and Foster, Dylan J and Krishnamurthy, Akshay and Rosset, Corby and Awadallah, Ahmed and Rakhlin, Alexander},
  journal={arXiv preprint arXiv:2405.21046},
  year={2024}
}


@article{trice,
  title={\href{https://arxiv.org/pdf/2312.02179}{Training chain-of-thought via latent-variable inference}},
  author={Hoffman, Matthew Douglas and Phan, Du and Dohan, David and Douglas, Sholto and Le, Tuan Anh and Parisi, Aaron and Sountsov, Pavel and Sutton, Charles and Vikram, Sharad and A Saurous, Rif},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  year={2024}
}


@article{apps,
  title={\href{https://github.com/hendrycks/apps}{Measuring Coding Challenge Competence With APPS}},
  author={Dan Hendrycks and Steven Basart and Saurav Kadavath and Mantas Mazeika and Akul Arora and Ethan Guo and Collin Burns and Samir Puranik and Horace He and Dawn Song and Jacob Steinhardt},
  journal={NeurIPS},
  year={2021}
}

@article{prm-code,
  title={\href{https://arxiv.org/pdf/2410.17621}{Process supervision-guided policy optimization for code generation}},
  author={Dai, Ning and Wu, Zheng and Zheng, Renjie and Wei, Ziyun and Shi, Wenlei and Jin, Xing and Liu, Guanlin and Dun, Chen and Huang, Liang and Yan, Lin},
  journal={arXiv preprint arXiv:2410.17621},
  year={2024}
}


@article{alphacode,
  title={\href{https://arxiv.org/abs/2203.07814}{Competition-level code generation with alphacode}},
  author={Li, Yujia and Choi, David and Chung, Junyoung and Kushman, Nate and Schrittwieser, Julian and Leblond, R{\'e}mi and Eccles, Tom and Keeling, James and Gimeno, Felix and Dal Lago, Agustin and others},
  journal={Science},
  volume={378},
  number={6624},
  pages={1092--1097},
  year={2022},
  publisher={American Association for the Advancement of Science}
}


@article{deepseek-coder-v2,
  title={\href{https://arxiv.org/pdf/2406.11931}{DeepSeek-Coder-V2: Breaking the Barrier of Closed-Source Models in Code Intelligence}},
  author={Zhu, Qihao and Guo, Daya and Shao, Zhihong and Yang, Dejian and Wang, Peiyi and Xu, Runxin and Wu, Y and Li, Yukun and Gao, Huazuo and Ma, Shirong and others},
  journal={arXiv preprint arXiv:2406.11931},
  year={2024}
}

@article{infalign,
  title={\href{https://arxiv.org/pdf/2412.19792}{InfAlign: Inference-aware language model alignment}},
  author={Balashankar, Ananth and Sun, Ziteng and Berant, Jonathan and Eisenstein, Jacob and Collins, Michael and Hutter, Adrian and Lee, Jong and Nagpal, Chirag and Prost, Flavien and Sinha, Aradhana and others},
  journal={arXiv preprint arXiv:2412.19792},
  year={2024}
}

@article{qwen-coder,
      title={\href{https://arxiv.org/pdf/2409.12186}{Qwen2. 5-Coder Technical Report}},
      author={Hui, Binyuan and Yang, Jian and Cui, Zeyu and Yang, Jiaxi and Liu, Dayiheng and Zhang, Lei and Liu, Tianyu and Zhang, Jiajun and Yu, Bowen and Dang, Kai and others},
      journal={arXiv preprint arXiv:2409.12186},
      year={2024}
}


@article{codet5+,
  title={\href{https://arxiv.org/pdf/2305.07922}{Codet5+: Open code large language models for code understanding and generation}},
  author={Wang, Yue and Le, Hung and Gotmare, Akhilesh Deepak and Bui, Nghi DQ and Li, Junnan and Hoi, Steven CH},
  journal={arXiv preprint arXiv:2305.07922},
  year={2023}
}


@article{codegemma,
  title={\href{https://arxiv.org/pdf/2406.11409}{Codegemma: Open code models based on gemma}},
  author={Team, CodeGemma and Zhao, Heri and Hui, Jeffrey and Howland, Joshua and Nguyen, Nam and Zuo, Siqi and Hu, Andrea and Choquette-Choo, Christopher A and Shen, Jingyue and Kelley, Joe and others},
  journal={arXiv preprint arXiv:2406.11409},
  year={2024}
}


@article{starcoder-2,
  title={\href{https://arxiv.org/pdf/2402.19173}{Starcoder 2 and the stack v2: The next generation}},
  author={Lozhkov, Anton and Li, Raymond and Allal, Loubna Ben and Cassano, Federico and Lamy-Poirier, Joel and Tazi, Nouamane and Tang, Ao and Pykhtar, Dmytro and Liu, Jiawei and Wei, Yuxiang and others},
  journal={arXiv preprint arXiv:2402.19173},
  year={2024}
}

@misc{prm-implicit-1,
  title={\href{https://github.com/PRIME-RL/PRIME}{Process Reinforcement through Implicit Rewards}},
  author={Ganqu Cui and Lifan Yuan and Zefan Wang and Hanbin Wang and Wendi Li and Bingxiang He and Yuchen Fan and Tianyu Yu and Qixin Xu and Weize Chen and Jiarui Yuan and Huayu Chen and Kaiyan Zhang and Xingtai Lv and Shuo Wang and Yuan Yao and Hao Peng and Yu Cheng and Zhiyuan Liu and Maosong Sun and Bowen Zhou and Ning Ding},
  year={2025},
  howpublished={\url{https://curvy-check-498.notion.site/Process-Reinforcement-through-Implicit-Rewards-15f4fcb9c42180f1b498cc9b2eaf896f}},
  note={Notion Blog}
}

@article{prm-implicit-2,
  title={\href{https://github.com/PRIME-RL/PRIME}{Free Process Rewards without Process Labels}},
  author={Lifan Yuan and Wendi Li and Huayu Chen and Ganqu Cui and Ning Ding and Kaiyan Zhang and Bowen Zhou and Zhiyuan Liu and Hao Peng},
  journal={arXiv preprint arXiv:2412.01981},
  year={2024}
}

@article{cot,
  title={\href{https://arxiv.org/abs/2201.11903}{Chain-of-thought prompting elicits reasoning in large language models}},
  author={Wei, Jason and Wang, Xuezhi and Schuurmans, Dale and Bosma, Maarten and Xia, Fei and Chi, Ed and Le, Quoc V and Zhou, Denny and others},
  journal={Advances in neural information processing systems},
  volume={35},
  pages={24824--24837},
  year={2022}
}


@article{pause-token,
  title={Think before you speak: Training language models with pause tokens},
  author={Goyal, Sachin and Ji, Ziwei and Rawat, Ankit Singh and Menon, Aditya Krishna and Kumar, Sanjiv and Nagarajan, Vaishnavh},
  journal={arXiv preprint arXiv:2310.02226},
  year={2023}
}


@article{quiet-star,
  title={\href{https://arxiv.org/abs/2403.09629}{Quiet-star: Language models can teach themselves to think before speaking}},
  author={Zelikman, Eric and Harik, Georges and Shao, Yijia and Jayasiri, Varuna and Haber, Nick and Goodman, Noah D},
  journal={arXiv preprint arXiv:2403.09629},
  year={2024}
}

@article{filler-token,
  title={\href{https://arxiv.org/pdf/2404.15758}{Let's Think Dot by Dot: Hidden Computation in Transformer Language Models}},
  author={Pfau, Jacob and Merrill, William and Bowman, Samuel R},
  journal={arXiv preprint arXiv:2404.15758},
  year={2024}
}



@article{coconut,
  title={\href{https://arxiv.org/pdf/2412.06769v1}{Training large language models to reason in a continuous latent space}},
  author={Hao, Shibo and Sukhbaatar, Sainbayar and Su, DiJia and Li, Xian and Hu, Zhiting and Weston, Jason and Tian, Yuandong},
  journal={arXiv preprint arXiv:2412.06769},
  year={2024}
}



@article{ccot,
  title={\href{https://arxiv.org/pdf/2412.13171}{Compressed Chain of Thought: Efficient Reasoning Through Dense Representations}},
  author={Cheng, Jeffrey and Van Durme, Benjamin},
  journal={arXiv preprint arXiv:2412.13171},
  year={2024}
}


@article{thinking-tokens,
  title={\href{https://arxiv.org/pdf/2405.08644}{Thinking Tokens for Language Modeling}},
  author={Herel, David and Mikolov, Tomas},
  journal={arXiv preprint arXiv:2405.08644},
  year={2024}
}


@article{srpo,
  title={\href{https://arxiv.org/pdf/2406.01660v3}{Self-Improving Robust Preference Optimization}},
  author={Choi, Eugene and Ahmadian, Arash and Geist, Matthieu and Pietquin, Oilvier and Azar, Mohammad Gheshlaghi},
  journal={arXiv preprint arXiv:2406.01660},
  year={2024}
}

@article{score,
  title={\href{https://arxiv.org/pdf/2409.12917}{Training language models to self-correct via reinforcement learning}},
  author={Kumar, Aviral and Zhuang, Vincent and Agarwal, Rishabh and Su, Yi and Co-Reyes, John D and Singh, Avi and Baumli, Kate and Iqbal, Shariq and Bishop, Colton and Roelofs, Rebecca and others},
  journal={arXiv preprint arXiv:2409.12917},
  year={2024}
}

@article{scaleai-planning,
  title={\href{https://arxiv.org/pdf/2409.03733v2}{Planning in natural language improves llm search for code generation}},
  author={Wang, Evan and Cassano, Federico and Wu, Catherine and Bai, Yunfeng and Song, Will and Nath, Vaskar and Han, Ziwen and Hendryx, Sean and Yue, Summer and Zhang, Hugh},
  journal={arXiv preprint arXiv:2409.03733},
  year={2024}
}


@article{structured-cot,
  title={\href{https://arxiv.org/pdf/2305.06599}{Structured chain-of-thought prompting for code generation}},
  author={Li, Jia and Li, Ge and Li, Yongmin and Jin, Zhi},
  journal={ACM Transactions on Software Engineering and Methodology},
  year={2023},
  publisher={ACM New York, NY}
}


@article{prm800k,
      title={\href{https://github.com/openai/prm800k}{Let's Verify Step by Step}}, 
      author={Lightman, Hunter and Kosaraju, Vineet and Burda, Yura and Edwards, Harri and Baker, Bowen and Lee, Teddy and Leike, Jan and Schulman, John and Sutskever, Ilya and Cobbe, Karl},
      journal={arXiv preprint arXiv:2305.20050},
      year={2023}
}

@article{lcm,
  title={\href{https://ai.meta.com/research/publications/large-concept-models-language-modeling-in-a-sentence-representation-space/}{Large Concept Models: Language Modeling in a Sentence Representation Space}},
  author={The, LCM and Barrault, Lo{\"\i}c and Duquenne, Paul-Ambroise and Elbayad, Maha and Kozhevnikov, Artyom and Alastruey, Belen and Andrews, Pierre and Coria, Mariano and Couairon, Guillaume and Costa-juss{\`a}, Marta R and others},
  journal={arXiv preprint arXiv:2412.08821},
  year={2024}
}

@article{sean-self-correct,
  title={\href{https://arxiv.org/pdf/2211.00053}{Generating sequences by learning to self-correct}},
  author={Welleck, Sean and Lu, Ximing and West, Peter and Brahman, Faeze and Shen, Tianxiao and Khashabi, Daniel and Choi, Yejin},
  journal={arXiv preprint arXiv:2211.00053},
  year={2022}
}

@article{inference-bon,
  title={\href{https://arxiv.org/pdf/2412.15287}{Inference-Aware Fine-Tuning for Best-of-N Sampling in Large Language Models}},
  author={Chow, Yinlam and Tennenholtz, Guy and Gur, Izzeddin and Zhuang, Vincent and Dai, Bo and Thiagarajan, Sridhar and Boutilier, Craig and Agarwal, Rishabh and Kumar, Aviral and Faust, Aleksandra},
  journal={arXiv preprint arXiv:2412.15287},
  year={2024}
}

@article{rloo,
  title={\href{https://arxiv.org/pdf/2402.14740v1}{Back to basics: Revisiting reinforce style optimization for learning from human feedback in llms}},
  author={Ahmadian, Arash and Cremer, Chris and Gall{\'e}, Matthias and Fadaee, Marzieh and Kreutzer, Julia and Pietquin, Olivier and {\"U}st{\"u}n, Ahmet and Hooker, Sara},
  journal={arXiv preprint arXiv:2402.14740},
  year={2024}
}

@article{gae,
  title={\href{https://arxiv.org/pdf/1506.02438}{High-dimensional continuous control using generalized advantage estimation}},
  author={Schulman, John and Moritz, Philipp and Levine, Sergey and Jordan, Michael and Abbeel, Pieter},
  journal={arXiv preprint arXiv:1506.02438},
  year={2015}
}

@article{buy,
  title={\href{https://openreview.net/forum?id=r1lgTGL5DE}{Buy 4 reinforce samples, get a baseline for free!}},
  author={Kool, Wouter and van Hoof, Herke and Welling, Max},
  year={2019},
  journal={ICLR Workshop Deep Reinforcement Learning Meets Structured Prediction},
}

@article{star,
  title={\href{https://arxiv.org/abs/2203.14465}{Star: Bootstrapping reasoning with reasoning}},
  author={Zelikman, Eric and Wu, Yuhuai and Mu, Jesse and Goodman, Noah},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={15476--15488},
  year={2022}
}

@inproceedings{math-sheperd,
  title={\href{https://arxiv.org/abs/2312.08935}{Math-shepherd: Verify and reinforce llms step-by-step without human annotations}},
  author={Wang, Peiyi and Li, Lei and Shao, Zhihong and Xu, Runxin and Dai, Damai and Li, Yifei and Chen, Deli and Wu, Yu and Sui, Zhifang},
  booktitle={Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  pages={9426--9439},
  year={2024}
}

@inproceedings{minimax-bound,
  title={\href{https://proceedings.mlr.press/v70/azar17a/azar17a.pdf}{Minimax regret bounds for reinforcement learning}},
  author={Azar, Mohammad Gheshlaghi and Osband, Ian and Munos, R{\'e}mi},
  booktitle={International conference on machine learning},
  pages={263--272},
  year={2017},
  organization={PMLR}
}

@inproceedings{non,
  title={\href{https://proceedings.mlr.press/v108/yue20b/yue20b.pdf}{Why non-myopic Bayesian optimization is promising and how far should we look-ahead? A study via rollout}},
  author={Yue, Xubo and Kontar, Raed AL},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={2808--2818},
  year={2020},
  organization={PMLR}
}

@misc{mcallester-friendly-ai,
  author = {David McAllester},
  title = {\href{https://machinethoughts.wordpress.com/2014/08/10/friendly-ai-and-the-servant-mission/}{Friendly AI and the Servant Mission}},
  year = {2014},
  month = {August 10},
  url = {https://machinethoughts.wordpress.com/2014/08/10/friendly-ai-and-the-servant-mission/},
  note = {Accessed: 2024-12-12}
}

@misc{o1,
  author = {OpenAI},
  title = {\href{https://cdn.openai.com/o1-system-card-20241205.pdf}{OpenAI o1 System Card}},
  year = {2024},
  month = {December 5},
  url = {https://cdn.openai.com/o1-system-card-20241205.pdf},
}

@article{r1,
  title={\href{https://arxiv.org/abs/2501.12948}{Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning}},
  author={Guo, Daya and Yang, Dejian and Zhang, Haowei and Song, Junxiao and Zhang, Ruoyu and Xu, Runxin and Zhu, Qihao and Ma, Shirong and Wang, Peiyi and Bi, Xiao and others},
  journal={arXiv preprint arXiv:2501.12948},
  year={2025}
}

@inproceedings{lp-mdp,
  title={\href{https://proceedings.mlr.press/v238/wang24h/wang24h.pdf}{Don't Be Pessimistic Too Early: Look K Steps Ahead!}},
  author={Wang, Chaoqi and Ye, Ziyu and Murphy, Kevin and Chen, Yuxin},
  booktitle={AISTATS},
  pages={3313--3321},
  year={2024}
}

@article{explore-then-commit,
  title={\href{https://arxiv.org/abs/1605.08988}{On explore-then-commit strategies}},
  author={Garivier, Aur{\'e}lien and Lattimore, Tor and Kaufmann, Emilie},
  journal={Advances in Neural Information Processing Systems},
  volume={29},
  year={2016}
}

@article{ids,
  title={\href{https://arxiv.org/abs/1403.5556}{Learning to optimize via information-directed sampling}},
  author={Russo, Daniel and Van Roy, Benjamin},
  journal={Advances in neural information processing systems},
  volume={27},
  year={2014}
}


@inproceedings{pure-exploration,
  title={\href{https://arxiv.org/abs/0802.2655}{Pure exploration in multi-armed bandits problems}},
  author={Bubeck, S{\'e}bastien and Munos, R{\'e}mi and Stoltz, Gilles},
  booktitle={International conference on Algorithmic learning theory},
  pages={23--37},
  year={2009},
  organization={Springer}
}


@book{bandit-algs,
  title={\href{https://tor-lattimore.com/downloads/book/book.pdf}{Bandit algorithms}},
  author={Lattimore, Tor and Szepesv{\'a}ri, Csaba},
  year={2020},
  publisher={Cambridge University Press}
}


@inproceedings{lightman2023let,
  title={\href{https://arxiv.org/abs/2305.20050}{Let's verify step by step}},
  author={Lightman, Hunter and Kosaraju, Vineet and Burda, Yuri and Edwards, Harrison and Baker, Bowen and Lee, Teddy and Leike, Jan and Schulman, John and Sutskever, Ilya and Cobbe, Karl},
  booktitle={The Twelfth International Conference on Learning Representations},
  year={2023}
}


@article{all-roads,
  title={\href{https://arxiv.org/pdf/2503.01067}{All Roads Lead to Likelihood: The Value of Reinforcement Learning in Fine-Tuning}},
  author={Swamy, Gokul and Choudhury, Sanjiban and Sun, Wen and Wu, Zhiwei Steven and Bagnell, J Andrew},
  journal={arXiv preprint arXiv:2503.01067},
  year={2025}
}



%https://www.lesswrong.com/posts/2ZnvFDtSWc3Hteah6/non-myopia-stories