$ latexmk -pdf -interaction=nonstopmode -halt-on-error -file-line-error -outdir=/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench main.tex Rc files read: /etc/LatexMk Latexmk: This is Latexmk, John Collins, 31 Jan. 2024. Version 4.83. No existing .aux file, so I'll make a simple one, and require run of *latex. Latexmk: applying rule 'pdflatex'... Rule 'pdflatex': Reasons for rerun Category 'other': Rerun of 'pdflatex' forced or previously required: Reason or flag: 'Initial setup' ------------ Run number 1 of rule 'pdflatex' ------------ ------------ Running 'pdflatex -interaction=nonstopmode -halt-on-error -file-line-error -recorder -output-directory="/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench" "main.tex"' ------------ This is pdfTeX, Version 3.141592653-2.6-1.40.25 (TeX Live 2023/Debian) (preloaded format=pdflatex) restricted \write18 enabled. entering extended mode (./main.tex LaTeX2e <2023-11-01> patch level 1 L3 programming layer <2024-01-22> (/usr/share/texlive/texmf-dist/tex/latex/base/article.cls Document Class: article 2023/05/17 v1.4n Standard LaTeX document class (/usr/share/texlive/texmf-dist/tex/latex/base/size10.clo)) (./iclr2027_conference.sty (/usr/share/texlive/texmf-dist/tex/latex/eso-pic/eso-pic.sty (/usr/share/texlive/texmf-dist/tex/latex/graphics/keyval.sty) (/usr/share/texlive/texmf-dist/tex/latex/xcolor/xcolor.sty (/usr/share/texlive/texmf-dist/tex/latex/graphics-cfg/color.cfg) (/usr/share/texlive/texmf-dist/tex/latex/graphics-def/pdftex.def) (/usr/share/texlive/texmf-dist/tex/latex/graphics/mathcolor.ltx))) (./fancyhdr.sty) (./natbib.sty)) (/usr/share/texlive/texmf-dist/tex/latex/psnfss/times.sty) (/usr/share/texlive/texmf-dist/tex/latex/microtype/microtype.sty (/usr/share/texlive/texmf-dist/tex/latex/etoolbox/etoolbox.sty) (/usr/share/texlive/texmf-dist/tex/latex/microtype/microtype-pdftex.def) (/usr/share/texlive/texmf-dist/tex/latex/microtype/microtype.cfg)) (/usr/share/texlive/texmf-dist/tex/latex/graphics/graphicx.sty (/usr/share/texlive/texmf-dist/tex/latex/graphics/graphics.sty (/usr/share/texlive/texmf-dist/tex/latex/graphics/trig.sty) (/usr/share/texlive/texmf-dist/tex/latex/graphics-cfg/graphics.cfg))) (/usr/share/texlive/texmf-dist/tex/latex/float/float.sty) (/usr/share/texlive/texmf-dist/tex/latex/booktabs/booktabs.sty) (/usr/share/texlive/texmf-dist/tex/latex/tools/array.sty) (/usr/share/texlive/texmf-dist/tex/latex/enumitem/enumitem.sty) (/usr/share/texlive/texmf-dist/tex/latex/pgf/frontendlayer/tikz.sty (/usr/share/texlive/texmf-dist/tex/latex/pgf/basiclayer/pgf.sty (/usr/share/texlive/texmf-dist/tex/latex/pgf/utilities/pgfrcs.sty (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfutil-common.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfutil-latex.def) (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfrcs.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/pgf.revision.tex))) (/usr/share/texlive/texmf-dist/tex/latex/pgf/basiclayer/pgfcore.sty (/usr/share/texlive/texmf-dist/tex/latex/pgf/systemlayer/pgfsys.sty (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgfsys.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfkeyslibraryfiltered.code.tex)) (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgf.cfg) (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-pdftex.def (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-common-pdf.def))) (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgfsyssoftpath.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgfsysprotocol.code.tex)) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcore.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathutil.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathparser.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.basic.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.trigonometric.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.random.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.comparison.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.base.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.round.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.misc.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.integerarithmetics.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathcalc.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfloat.code.tex)) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfint.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepoints.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathconstruct.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathusage.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorescopes.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoregraphicstate.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretransformations.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorequick.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreobjects.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathprocessing.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorearrows.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreshade.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreimage.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreexternal.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorelayers.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretransparency.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepatterns.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorerdf.code.tex))) (/usr/share/texlive/texmf-dist/tex/generic/pgf/modules/pgfmoduleshapes.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/modules/pgfmoduleplot.code.tex) (/usr/share/texlive/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version-0-65.sty) (/usr/share/texlive/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version-1-18.sty)) (/usr/share/texlive/texmf-dist/tex/latex/pgf/utilities/pgffor.sty (/usr/share/texlive/texmf-dist/tex/latex/pgf/utilities/pgfkeys.sty (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex)) (/usr/share/texlive/texmf-dist/tex/latex/pgf/math/pgfmath.sty (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex)) (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgffor.code.tex)) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/tikz.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/libraries/pgflibraryplothandlers.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/modules/pgfmodulematrix.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarytopaths.code.tex))) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarypositioning.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryfit.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarycalc.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/libraries/pgflibraryarrows.meta.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarybackgrounds.code.tex) (./figures/tikz/styles.tex) (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsmath.sty For additional information on amsmath, use the `?' option. (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amstext.sty (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsgen.sty)) (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsbsy.sty) (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsopn.sty)) (/usr/share/texlive/texmf-dist/tex/latex/amsfonts/amssymb.sty (/usr/share/texlive/texmf-dist/tex/latex/amsfonts/amsfonts.sty)) (/usr/share/texlive/texmf-dist/tex/latex/psnfss/ot1ptm.fd) (/usr/share/texlive/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def) (/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.aux) (/usr/share/texlive/texmf-dist/tex/context/base/mkii/supp-pdf.mkii [Loading MPS to PDF converter (version 2006.09.02).] ) (/usr/share/texlive/texmf-dist/tex/latex/epstopdf-pkg/epstopdf-base.sty (/usr/share/texlive/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg)) (/usr/share/texlive/texmf-dist/tex/latex/microtype/mt-ptm.cfg) (/usr/share/texlive/texmf-dist/tex/latex/microtype/mt-cmr.cfg) (/usr/share/texlive/texmf-dist/tex/latex/amsfonts/umsa.fd) (/usr/share/texlive/texmf-dist/tex/latex/microtype/mt-msa.cfg) (/usr/share/texlive/texmf-dist/tex/latex/amsfonts/umsb.fd) (/usr/share/texlive/texmf-dist/tex/latex/microtype/mt-msb.cfg) (/usr/share/texlive/texmf-dist/tex/latex/psnfss/ot1pcr.fd) LaTeX Warning: Reference `fig:story' on page 1 undefined on input line 39. (./figures/tikz/fig-problem-gap.tex) Package natbib Warning: Citation `swebench2024' on page 1 undefined on input line 48. Package natbib Warning: Citation `programbench2026' on page 1 undefined on input line 48. Package natbib Warning: Citation `webarena2023' on page 1 undefined on input line 48. Package natbib Warning: Citation `osworld2024' on page 1 undefined on input line 48. Package natbib Warning: Citation `appworld2024' on page 1 undefined on input line 48. Package natbib Warning: Citation `webgenbench2025' on page 1 undefined on input line 48. Package natbib Warning: Citation `gaia2023' on page 1 undefined on input line 48. Package natbib Warning: Citation `agentslastexam2026' on page 1 undefined on input line 48. Package natbib Warning: Citation `programbench2026' on page 1 undefined on input line 48. Package natbib Warning: Citation `appworld2024' on page 1 undefined on input line 48. Package natbib Warning: Citation `agentslastexam2026' on page 1 undefined on input line 50. [1{/var/lib/texmf/fonts/map/pdftex/updmap/pdftex.map}{/usr/share/texlive/texmf-dist/fonts/enc/dvips/base/8r.enc}] (/usr/share/texlive/texmf-dist/tex/latex/psnfss/ts1ptm.fd) LaTeX Warning: Reference `sec:task' on page 2 undefined on input line 54. LaTeX Warning: Reference `sec:dataset' on page 2 undefined on input line 55. LaTeX Warning: Reference `sec:harness' on page 2 undefined on input line 56. LaTeX Warning: Reference `sec:experiments' on page 2 undefined on input line 57. [2] LaTeX Warning: Reference `tab:contracts' on page 3 undefined on input line 74. Underfull \hbox (badness 10000) in paragraph at lines 87--88 []|\OT1/ptm/m/n/9 (+20) hidden back-end mem-ory, Package natbib Warning: Citation `appworld2024' on page 3 undefined on input line 96. Package natbib Warning: Citation `taubench2024' on page 3 undefined on input line 96. Package natbib Warning: Citation `appworld2024' on page 3 undefined on input line 101. LaTeX Warning: Reference `tab:dataset-stats' on page 3 undefined on input line 104. LaTeX Warning: Reference `app:dataset-details' on page 3 undefined on input line 104. Package natbib Warning: Citation `livecodebench2024' on page 3 undefined on input line 107. Package natbib Warning: Citation `agentslastexam2026' on page 3 undefined on input line 107. [3] (./tables/dataset-stats-table.tex) Underfull \hbox (badness 2503) in paragraph at lines 113--113 []\OT1/ptm/m/n/10 (+20) Table 2: \OT1/ptm/m/sc/10 (+20) LiveGUI-Bench \OT1/ptm/m/n/10 (+20) statis-tics --- case, app-domain, episode, and deterministic- LaTeX Warning: Reference `fig:evaluator-architecture' on page 4 undefined on input line 133. LaTeX Warning: Reference `app:trajectories' on page 4 undefined on input line 133. [4] (./figures/tikz/fig-evaluator-architecture.tex) LaTeX Warning: Reference `fig:gui-behavior-gates' on page 5 undefined on input line 143. (./figures/tikz/fig-gui-behavior-gates.tex) LaTeX Warning: Reference `fig:pipeline' on page 5 undefined on input line 153. (./figures/tikz/fig-benchmark-pipeline.tex) LaTeX Warning: Reference `app:run-artifacts' on page 5 undefined on input line 165. [5] LaTeX Warning: Reference `tab:control-suite' on page 6 undefined on input line 168. LaTeX Warning: Reference `app:dataset-details' on page 6 undefined on input line 168. LaTeX Warning: Reference `tab:main-results' on page 6 undefined on input line 168. LaTeX Warning: Reference `tab:main-results' on page 6 undefined on input line 171. LaTeX Warning: Reference `app:run-artifacts' on page 6 undefined on input line 171. (./tables/appworld-bench-table.tex) LaTeX Font Warning: Font shape `OT1/ptm/m/scit' undefined (Font) using `OT1/ptm/m/sc' instead on input line 183. LaTeX Font Warning: Font shape `OT1/ptm/m/scit' undefined (Font) using `OT1/ptm/m/sc' instead on input line 183. LaTeX Warning: Reference `tab:main-results' on page 6 undefined on input line 183. LaTeX Warning: Reference `tab:main-results' on page 6 undefined on input line 183. [6] Package natbib Warning: Citation `miniwob2017' on page 7 undefined on input line 189. Package natbib Warning: Citation `mind2web2023' on page 7 undefined on input line 189. Package natbib Warning: Citation `webarena2023' on page 7 undefined on input line 189. Package natbib Warning: Citation `visualwebarena2024' on page 7 undefined on input line 189. Package natbib Warning: Citation `webshop2022' on page 7 undefined on input line 189. Package natbib Warning: Citation `webvoyager2024' on page 7 undefined on input line 189. Package natbib Warning: Citation `osworld2024' on page 7 undefined on input line 189. Package natbib Warning: Citation `androidworld2024' on page 7 undefined on input line 189. Package natbib Warning: Citation `aitw2023' on page 7 undefined on input line 189. Package natbib Warning: Citation `omniact2024' on page 7 undefined on input line 189. Package natbib Warning: Citation `workarena2024' on page 7 undefined on input line 189. Package natbib Warning: Citation `officebench2024' on page 7 undefined on input line 189. Package natbib Warning: Citation `odysseybench2025' on page 7 undefined on input line 189. Package natbib Warning: Citation `stwebagentbench2024' on page 7 undefined on input line 189. Package natbib Warning: Citation `realbenchmark2025' on page 7 undefined on input line 189. Package natbib Warning: Citation `browserarena2025' on page 7 undefined on input line 189. Package natbib Warning: Citation `appworld2024' on page 7 undefined on input line 189. Package natbib Warning: Citation `aguvis2024' on page 7 undefined on input line 189. Package natbib Warning: Citation `osgenesis2024' on page 7 undefined on input line 189. Package natbib Warning: Citation `humaneval2021' on page 7 undefined on input line 192. Package natbib Warning: Citation `mbpp2021' on page 7 undefined on input line 192. Package natbib Warning: Citation `apps2021' on page 7 undefined on input line 192. Package natbib Warning: Citation `ds1000' on page 7 undefined on input line 192. Package natbib Warning: Citation `codexglue2021' on page 7 undefined on input line 192. Package natbib Warning: Citation `multiple2023' on page 7 undefined on input line 192. Package natbib Warning: Citation `livecodebench2024' on page 7 undefined on input line 192. Package natbib Warning: Citation `bigcodebench2024' on page 7 undefined on input line 192. Package natbib Warning: Citation `codebleu2020' on page 7 undefined on input line 192. Package natbib Warning: Citation `defects4j2014' on page 7 undefined on input line 192. Package natbib Warning: Citation `quixbugs2017' on page 7 undefined on input line 192. Package natbib Warning: Citation `bugsinpy2020' on page 7 undefined on input line 192. Package natbib Warning: Citation `bears2019' on page 7 undefined on input line 192. Package natbib Warning: Citation `manysstubs4j2019' on page 7 undefined on input line 192. Package natbib Warning: Citation `introclass2015' on page 7 undefined on input line 192. Package natbib Warning: Citation `codeflaws2017' on page 7 undefined on input line 192. Package natbib Warning: Citation `swebench2024' on page 7 undefined on input line 192. Package natbib Warning: Citation `swebenchplus2024' on page 7 undefined on input line 192. Package natbib Warning: Citation `swebenchmobile2026' on page 7 undefined on input line 192. Package natbib Warning: Citation `programbench2026' on page 7 undefined on input line 192. Package natbib Warning: Citation `agentbench2023' on page 7 undefined on input line 195. Package natbib Warning: Citation `toolbench2023' on page 7 undefined on input line 195. Package natbib Warning: Citation `apibank2023' on page 7 undefined on input line 195. Package natbib Warning: Citation `taubench2024' on page 7 undefined on input line 195. Package natbib Warning: Citation `gaia2023' on page 7 undefined on input line 195. Package natbib Warning: Citation `assistantbench2024' on page 7 undefined on input line 195. Package natbib Warning: Citation `alfworld2021' on page 7 undefined on input line 195. Package natbib Warning: Citation `scienceworld2022' on page 7 undefined on input line 195. Package natbib Warning: Citation `agentslastexam2026' on page 7 undefined on input line 195. Package natbib Warning: Citation `generativeinterfaces2025' on page 7 undefined on input line 198. Package natbib Warning: Citation `malleableui2025' on page 7 undefined on input line 198. Package natbib Warning: Citation `dynavis2024' on page 7 undefined on input line 198. Package natbib Warning: Citation `generativeagents2023' on page 7 undefined on input line 198. Package natbib Warning: Citation `webgenbench2025' on page 7 undefined on input line 198. Package natbib Warning: Citation `auigym2025' on page 7 undefined on input line 198. [7] Package natbib Warning: Citation `dynabench2021' on page 8 undefined on input line 201. Package natbib Warning: Citation `bigbench2023' on page 8 undefined on input line 201. Package natbib Warning: Citation `helm2023' on page 8 undefined on input line 201. Package natbib Warning: Citation `checklist2020' on page 8 undefined on input line 201. Package natbib Warning: Citation `datasheets2018' on page 8 undefined on input line 201. Package natbib Warning: Citation `modelcards2019' on page 8 undefined on input line 201. Package natbib Warning: Citation `datastatements2018' on page 8 undefined on input line 201. Package natbib Warning: Citation `dedup2022' on page 8 undefined on input line 201. Package natbib Warning: Citation `shortcutlearning2020' on page 8 undefined on input line 201. LaTeX Warning: Reference `tab:positioning' on page 8 undefined on input line 201. [8] LaTeX Warning: Reference `app:run-artifacts' on page 9 undefined on input line 239. LaTeX Warning: Reference `sec:dataset' on page 9 undefined on input line 239. LaTeX Warning: Reference `sec:harness' on page 9 undefined on input line 239. Underfull \hbox (badness 3482) in paragraph at lines 239--240 \OT1/ptm/m/n/10 (+20) All em-pir-i-cal num-bers are gen-er-ated from archived run ar-ti-facts (\OT1/pcr/m/n/10 report.json \OT1/ptm/m/n/10 (+20) and LaTeX Warning: Reference `sec:dataset' on page 9 undefined on input line 242. LaTeX Warning: Reference `sec:harness' on page 9 undefined on input line 242. (./main.bbl Underfull \hbox (badness 1694) in paragraph at lines 10--14 []\OT1/ptm/m/n/10 (+20) Reem Alei-than, Hao-ran Xue, Mo-ham-mad Mahdi Mo-ha-jer, Eli-jah Nnorom, Gias Ud-din, [9] Underfull \hbox (badness 2293) in paragraph at lines 154--159 []\OT1/ptm/m/n/10 (+20) Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fan-jia Yan, Tian-jun Zhang, Sida I. Underfull \hbox (badness 1824) in paragraph at lines 162--167 []\OT1/ptm/m/n/10 (+20) Carlos E. Jimenez, John Yang, Alexan-der Wet-tig, Shunyu Yao, Kexin Pei, Ofir Press, Underfull \hbox (badness 3009) in paragraph at lines 162--167 \OT1/ptm/m/n/10 (+20) is-sues? In \OT1/ptm/m/it/10 (+20) In-ter-na-tional Con-fer-ence on Learn-ing Rep-re-sen-ta-tions\OT1/ptm/m/n/10 (+20) , 2024. URL Underfull \hbox (badness 1742) in paragraph at lines 169--176 []\OT1/ptm/m/n/10 (+20) Ren[]e Just, Dar-i-oush Jalali, and Michael D. Ernst. De-fects4J: A database of ex-ist-ing Underfull \hbox (badness 1019) in paragraph at lines 169--176 \OT1/ptm/m/n/10 (+20) faults to en-able con-trolled test-ing stud-ies for Java pro-grams. In \OT1/ptm/m/it/10 (+20) In-ter-na-tional Sym-po- Underfull \hbox (badness 3078) in paragraph at lines 179--184 \OT1/ptm/m/n/10 (+20) Al-shikh, and Rus-lan Salakhut-di-nov. Om-ni-ACT: A dataset and bench-mark for en- Underfull \hbox (badness 2213) in paragraph at lines 179--184 \OT1/ptm/m/n/10 (+20) abling mul-ti-modal gen-er-al-ist au-tonomous agents for desk-top and web, 2024. URL [10] Underfull \hbox (badness 2503) in paragraph at lines 209--214 []\OT1/ptm/m/n/10 (+20) Jing Yu Koh, Robert Lo, Lawrence Jang, Vikram Duvvur, Ming Chong Lim, Po-Yu Underfull \hbox (badness 8189) in paragraph at lines 237--244 []\OT1/ptm/m/n/10 (+20) Katherine Lee, Daphne Ip-polito, An-drew Nys-trom, Chiyuan Zhang, Dou-glas Eck, Underfull \hbox (badness 2229) in paragraph at lines 237--244 \OT1/ptm/m/n/10 (+20) Chris Callison-Burch, and Nicholas Car-lini. Dedu-pli-cat-ing train-ing data makes lan- Underfull \hbox (badness 4316) in paragraph at lines 237--244 \OT1/ptm/m/n/10 (+20) guage mod-els bet-ter. In \OT1/ptm/m/it/10 (+20) Pro-ceed-ings of the An-nual Meet-ing of the As-so-ci-a-tion Underfull \hbox (badness 3965) in paragraph at lines 237--244 \OT1/ptm/m/it/10 (+20) for Com-pu-ta-tional Lin-guis-tics\OT1/ptm/m/n/10 (+20) , 2022. doi: 10.18653/v1/2022.acl-long.577. URL [11] Underfull \hbox (badness 5771) in paragraph at lines 318--325 []\OT1/ptm/m/n/10 (+20) Fernanda Madeiral, Si-mon Urli, Marcelo Maia, and Mar-tin Mon-per-rus. BEARS: Underfull \hbox (badness 10000) in paragraph at lines 318--325 \OT1/ptm/m/n/10 (+20) An ex-ten-si-ble Java bug bench-mark for au-to-matic pro-gram re-pair stud-ies. In Underfull \hbox (badness 10000) in paragraph at lines 318--325 \OT1/ptm/m/it/10 (+20) Pro-ceed-ings of the In-ter-na-tional Con-fer-ence on Soft-ware Anal-y-sis, Evo-lu- Underfull \hbox (badness 10000) in paragraph at lines 318--325 \OT1/ptm/m/it/10 (+20) tion and Reengi-neer-ing\OT1/ptm/m/n/10 (+20) , 2019. doi: 10.1109/SANER.2019.8667991. URL Underfull \hbox (badness 3049) in paragraph at lines 328--332 []\OT1/ptm/m/n/10 (+20) Gr[]egoire Mi-alon, Cl[]ementine Four-rier, Craig Swift, Thomas Wolf, Yann Le-Cun, and Underfull \hbox (badness 3826) in paragraph at lines 328--332 \OT1/ptm/m/n/10 (+20) Thomas Scialom. GAIA: A bench-mark for gen-eral AI as-sis-tants, 2023. URL Underfull \hbox (badness 1810) in paragraph at lines 374--381 []\OT1/ptm/m/n/10 (+20) Christopher Rawles, Sarah Clinck-e-mail-lie, Yi-fan Chang, Jonathan Waltz, Gabrielle Lau, Underfull \hbox (badness 1924) in paragraph at lines 374--381 \OT1/ptm/m/n/10 (+20) Mary-beth Fair, Al-ice Li, William Bishop, Wei Li, Fo-lawiyo Campbell-Ajala, Daniel [12] Underfull \hbox (badness 1014) in paragraph at lines 438--444 \OT1/ptm/m/n/10 (+20) Yan, Bo Liu, Han-son Wen, Mingyang Xu, Xi-aoyuan Liu, Zi-meng Chen, Weiyan Shi, Underfull \hbox (badness 2293) in paragraph at lines 447--454 \OT1/ptm/m/n/10 (+20) flaws: A pro-gram-ming com-pe-ti-tion bench-mark for eval-u-at-ing au-to-mated pro-gram re- Underfull \hbox (badness 1430) in paragraph at lines 447--454 \OT1/ptm/m/n/10 (+20) pair tools. In \OT1/ptm/m/it/10 (+20) 2017 IEEE/ACM 39th In-ter-na-tional Con-fer-ence on Soft-ware En-gi-neer- Underfull \hbox (badness 1184) in paragraph at lines 447--454 \OT1/ptm/m/it/10 (+20) ing Com-pan-ion (ICSE-C)\OT1/ptm/m/n/10 (+20) , pp. 180--182, 2017. doi: 10.1109/ICSE-C.2017.76. URL Underfull \hbox (badness 4569) in paragraph at lines 457--462 \OT1/ptm/m/n/10 (+20) Li, Xinni Xie, Guangjing Wang, and Ji-ax-uan You. SWE-Bench Mo-bile: Can Underfull \hbox (badness 1221) in paragraph at lines 457--462 \OT1/ptm/m/n/10 (+20) large lan-guage model agents de-velop industry-level mo-bile ap-pli-ca-tions?, 2026. URL Underfull \hbox (badness 2050) in paragraph at lines 543--549 []\OT1/ptm/m/n/10 (+20) John Yang, Kil-ian Lieret, Jef-frey Ma, Parth Thakkar, Dmitrii Ped-chenko, Sten Sootla, Underfull \hbox (badness 1990) in paragraph at lines 543--549 \OT1/ptm/m/n/10 (+20) Pro-gram-Bench: Can lan-guage mod-els re-build pro-grams from scratch?, 2026. URL [13] Underfull \hbox (badness 3758) in paragraph at lines 558--562 []\OT1/ptm/m/n/10 (+20) Shunyu Yao, Noah Shinn, Pe-dram Razavi, and Karthik Narasimhan. $\OML/cmm/m/it/10 ^^\$\OT1/ptm/m/n/10 (+20) -bench: A Underfull \hbox (badness 10000) in paragraph at lines 558--562 \OT1/ptm/m/n/10 (+20) bench-mark for tool-agent-user in-ter-ac-tion in real-world do-mains, 2024. URL ) (./tables/livegui-bench-split-counts-table.tex) Underfull \vbox (badness 1371) has occurred while \output is active [14] (./tables/livegui-bench-app-domain-table.tex) (./tables/livegui-bench-assertion-counts-table.tex) (./tables/release-governance-table.tex Underfull \hbox (badness 10000) in paragraph at lines 8--8 []|\OT1/ptm/m/n/7 (+20) hashes, meta-data, schema, Underfull \hbox (badness 10000) in paragraph at lines 8--9 []|\OT1/ptm/m/n/7 (+20) sealed-test rank-ing Underfull \hbox (badness 3407) in paragraph at lines 8--9 \OT1/ptm/m/n/7 (+20) archived app pack- Underfull \hbox (badness 6364) in paragraph at lines 9--9 []|\OT1/ptm/m/n/7 (+20) periodic para-phrase and new- ) (./tables/release-gates-table.tex) [15] LaTeX Warning: Reference `fig:traj-pass' on page 16 undefined on input line 327. LaTeX Warning: Reference `fig:traj-fail' on page 16 undefined on input line 327. LaTeX Warning: Reference `fig:traj-xdomain' on page 16 undefined on input line 329. Underfull \vbox (badness 10000) has occurred while \output is active [16] Underfull \hbox (badness 1314) in paragraph at lines 354--354 \OT1/ptm/m/n/10 (+20) face opens, then a sub-mit routes into the app-scoped work-flow, which in-vokes the real Underfull \vbox (badness 10000) has occurred while \output is active [17 <./figures/trajectories/fitness-step-01.png (PNG copy)> <./figures/trajectories/fitness-step-04.png (PNG copy)> <./figures/trajectories/fitness-step-06.png (PNG copy)> <./figures/trajectories/amazon-step-01.png (PNG copy)> <./figures/trajectories/amazon-step-05.png (PNG copy)> <./figures/trajectories/amazon-step-09.png (PNG copy)>] LaTeX Warning: Reference `sec:dataset' on page 18 undefined on input line 361. [18 <./figures/trajectories/xdomain-amazon-open.png (PNG copy)> <./figures/trajectories/xdomain-amazon-success.png (PNG copy)> <./figures/trajectories/xdomain-venmo-fail.png (PNG copy)>] LaTeX Warning: Reference `app:case-camping' on page 19 undefined on input line 386. LaTeX Warning: Reference `app:case-camping' on page 19 undefined on input line 396. LaTeX Warning: Reference `app:case-newsletter' on page 19 undefined on input line 396. [19] Package natbib Warning: There were undefined citations. [20] (/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.aux Package natbib Warning: Citation(s) may have changed. (natbib) Rerun to get citations correct. ) LaTeX Warning: There were undefined references. LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right. ) (see the transcript file for additional information) Output written on /home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.pdf (20 pages, 2168399 bytes). Transcript written on /home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.log. Latexmk: Getting log file '/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.log' Latexmk: Examining '/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.fls' Latexmk: Examining '/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.log' Latexmk: Found input bbl file 'main.bbl' Latexmk: References changed. Latexmk: References changed. Latexmk: Log file says output to '/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.pdf' Latexmk: Found bibliography file(s): ./references.bib Latexmk: applying rule 'bibtex main'... Rule 'bibtex main': Reasons for rerun Category 'other': Rerun of 'bibtex main' forced or previously required: Reason or flag: 'Initial set up of rule' ------------ Run number 1 of rule 'bibtex main' ------------ ------------ Running 'bibtex "main.aux"' ------------ This is BibTeX, Version 0.99d (TeX Live 2023/Debian) The top-level auxiliary file: main.aux The style file: iclr2027_conference.bst Database file #1: references.bib Latexmk: applying rule 'pdflatex'... Rule 'pdflatex': Reasons for rerun Changed files or newly in use/created: /home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.aux main.bbl ------------ Run number 2 of rule 'pdflatex' ------------ ------------ Running 'pdflatex -interaction=nonstopmode -halt-on-error -file-line-error -recorder -output-directory="/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench" "main.tex"' ------------ This is pdfTeX, Version 3.141592653-2.6-1.40.25 (TeX Live 2023/Debian) (preloaded format=pdflatex) restricted \write18 enabled. entering extended mode (./main.tex LaTeX2e <2023-11-01> patch level 1 L3 programming layer <2024-01-22> (/usr/share/texlive/texmf-dist/tex/latex/base/article.cls Document Class: article 2023/05/17 v1.4n Standard LaTeX document class (/usr/share/texlive/texmf-dist/tex/latex/base/size10.clo)) (./iclr2027_conference.sty (/usr/share/texlive/texmf-dist/tex/latex/eso-pic/eso-pic.sty (/usr/share/texlive/texmf-dist/tex/latex/graphics/keyval.sty) (/usr/share/texlive/texmf-dist/tex/latex/xcolor/xcolor.sty (/usr/share/texlive/texmf-dist/tex/latex/graphics-cfg/color.cfg) (/usr/share/texlive/texmf-dist/tex/latex/graphics-def/pdftex.def) (/usr/share/texlive/texmf-dist/tex/latex/graphics/mathcolor.ltx))) (./fancyhdr.sty) (./natbib.sty)) (/usr/share/texlive/texmf-dist/tex/latex/psnfss/times.sty) (/usr/share/texlive/texmf-dist/tex/latex/microtype/microtype.sty (/usr/share/texlive/texmf-dist/tex/latex/etoolbox/etoolbox.sty) (/usr/share/texlive/texmf-dist/tex/latex/microtype/microtype-pdftex.def) (/usr/share/texlive/texmf-dist/tex/latex/microtype/microtype.cfg)) (/usr/share/texlive/texmf-dist/tex/latex/graphics/graphicx.sty (/usr/share/texlive/texmf-dist/tex/latex/graphics/graphics.sty (/usr/share/texlive/texmf-dist/tex/latex/graphics/trig.sty) (/usr/share/texlive/texmf-dist/tex/latex/graphics-cfg/graphics.cfg))) (/usr/share/texlive/texmf-dist/tex/latex/float/float.sty) (/usr/share/texlive/texmf-dist/tex/latex/booktabs/booktabs.sty) (/usr/share/texlive/texmf-dist/tex/latex/tools/array.sty) (/usr/share/texlive/texmf-dist/tex/latex/enumitem/enumitem.sty) (/usr/share/texlive/texmf-dist/tex/latex/pgf/frontendlayer/tikz.sty (/usr/share/texlive/texmf-dist/tex/latex/pgf/basiclayer/pgf.sty (/usr/share/texlive/texmf-dist/tex/latex/pgf/utilities/pgfrcs.sty (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfutil-common.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfutil-latex.def) (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfrcs.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/pgf.revision.tex))) (/usr/share/texlive/texmf-dist/tex/latex/pgf/basiclayer/pgfcore.sty (/usr/share/texlive/texmf-dist/tex/latex/pgf/systemlayer/pgfsys.sty (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgfsys.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfkeyslibraryfiltered.code.tex)) (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgf.cfg) (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-pdftex.def (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-common-pdf.def))) (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgfsyssoftpath.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgfsysprotocol.code.tex)) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcore.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathutil.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathparser.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.basic.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.trigonometric.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.random.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.comparison.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.base.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.round.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.misc.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.integerarithmetics.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathcalc.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfloat.code.tex)) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfint.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepoints.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathconstruct.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathusage.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorescopes.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoregraphicstate.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretransformations.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorequick.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreobjects.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathprocessing.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorearrows.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreshade.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreimage.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreexternal.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorelayers.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretransparency.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepatterns.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorerdf.code.tex))) (/usr/share/texlive/texmf-dist/tex/generic/pgf/modules/pgfmoduleshapes.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/modules/pgfmoduleplot.code.tex) (/usr/share/texlive/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version-0-65.sty) (/usr/share/texlive/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version-1-18.sty)) (/usr/share/texlive/texmf-dist/tex/latex/pgf/utilities/pgffor.sty (/usr/share/texlive/texmf-dist/tex/latex/pgf/utilities/pgfkeys.sty (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex)) (/usr/share/texlive/texmf-dist/tex/latex/pgf/math/pgfmath.sty (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex)) (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgffor.code.tex)) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/tikz.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/libraries/pgflibraryplothandlers.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/modules/pgfmodulematrix.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarytopaths.code.tex))) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarypositioning.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryfit.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarycalc.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/libraries/pgflibraryarrows.meta.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarybackgrounds.code.tex) (./figures/tikz/styles.tex) (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsmath.sty For additional information on amsmath, use the `?' option. (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amstext.sty (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsgen.sty)) (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsbsy.sty) (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsopn.sty)) (/usr/share/texlive/texmf-dist/tex/latex/amsfonts/amssymb.sty (/usr/share/texlive/texmf-dist/tex/latex/amsfonts/amsfonts.sty)) (/usr/share/texlive/texmf-dist/tex/latex/psnfss/ot1ptm.fd) (/usr/share/texlive/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def) (/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.aux) (/usr/share/texlive/texmf-dist/tex/context/base/mkii/supp-pdf.mkii [Loading MPS to PDF converter (version 2006.09.02).] ) (/usr/share/texlive/texmf-dist/tex/latex/epstopdf-pkg/epstopdf-base.sty (/usr/share/texlive/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg)) (/usr/share/texlive/texmf-dist/tex/latex/microtype/mt-ptm.cfg) (/usr/share/texlive/texmf-dist/tex/latex/microtype/mt-cmr.cfg) (/usr/share/texlive/texmf-dist/tex/latex/amsfonts/umsa.fd) (/usr/share/texlive/texmf-dist/tex/latex/microtype/mt-msa.cfg) (/usr/share/texlive/texmf-dist/tex/latex/amsfonts/umsb.fd) (/usr/share/texlive/texmf-dist/tex/latex/microtype/mt-msb.cfg) (/usr/share/texlive/texmf-dist/tex/latex/psnfss/ot1pcr.fd) (./figures/tikz/fig-problem-gap.tex) [1{/var/lib/texmf/fonts/map/pdftex/updmap/pdftex.map}{/usr/share/texlive/texmf-dist/fonts/enc/dvips/base/8r.enc}] (/usr/share/texlive/texmf-dist/tex/latex/psnfss/ts1ptm.fd) Underfull \vbox (badness 10000) has occurred while \output is active [2] Underfull \hbox (badness 10000) in paragraph at lines 87--88 []|\OT1/ptm/m/n/9 (+20) hidden back-end mem-ory, Underfull \hbox (badness 1400) in paragraph at lines 101--102 \OT1/ptm/m/sc/10 (+20) LiveGUI-Bench \OT1/ptm/m/n/10 (+20) tar-gets re-cur-ring per-sonal work-flows in which a per-sis-tent vis-i-ble sur- Underfull \hbox (badness 2818) in paragraph at lines 101--102 \OT1/ptm/m/n/10 (+20) face is more ap-pro-pri-ate than a one-off an-swer, and grounds ev-ery case in the ex-e- Underfull \hbox (badness 1527) in paragraph at lines 101--102 \OT1/ptm/m/n/10 (+20) cutable ap-pli-ca-tion APIs of Ap-p-World (Trivedi et al., 2024). The tracked case file is [3] (./tables/dataset-stats-table.tex) Underfull \hbox (badness 2503) in paragraph at lines 113--113 []\OT1/ptm/m/n/10 (+20) Table 2: \OT1/ptm/m/sc/10 (+20) LiveGUI-Bench \OT1/ptm/m/n/10 (+20) statis-tics --- case, app-domain, episode, and deterministic- [4] (./figures/tikz/fig-evaluator-architecture.tex) (./figures/tikz/fig-gui-behavior-gates.tex) (./figures/tikz/fig-benchmark-pipeline.tex) Underfull \vbox (badness 10000) has occurred while \output is active [5] (./tables/appworld-bench-table.tex) LaTeX Font Warning: Font shape `OT1/ptm/m/scit' undefined (Font) using `OT1/ptm/m/sc' instead on input line 183. LaTeX Font Warning: Font shape `OT1/ptm/m/scit' undefined (Font) using `OT1/ptm/m/sc' instead on input line 183. Underfull \vbox (badness 2495) has occurred while \output is active [6] Underfull \vbox (badness 1406) has occurred while \output is active [7] [8] Underfull \hbox (badness 3482) in paragraph at lines 239--240 \OT1/ptm/m/n/10 (+20) All em-pir-i-cal num-bers are gen-er-ated from archived run ar-ti-facts (\OT1/pcr/m/n/10 report.json \OT1/ptm/m/n/10 (+20) and (./main.bbl Underfull \hbox (badness 1694) in paragraph at lines 9--12 []\OT1/ptm/m/n/10 (+20) Reem Alei-than, Hao-ran Xue, Mo-ham-mad Mahdi Mo-ha-jer, Eli-jah Nnorom, Gias Ud-din, [9] Underfull \hbox (badness 2293) in paragraph at lines 93--96 []\OT1/ptm/m/n/10 (+20) Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fan-jia Yan, Tian-jun Zhang, Sida I. [10] Underfull \hbox (badness 1824) in paragraph at lines 98--102 []\OT1/ptm/m/n/10 (+20) Carlos E. Jimenez, John Yang, Alexan-der Wet-tig, Shunyu Yao, Kexin Pei, Ofir Press, Underfull \hbox (badness 3009) in paragraph at lines 98--102 \OT1/ptm/m/n/10 (+20) is-sues? In \OT1/ptm/m/it/10 (+20) In-ter-na-tional Con-fer-ence on Learn-ing Rep-re-sen-ta-tions\OT1/ptm/m/n/10 (+20) , 2024. URL Underfull \hbox (badness 1742) in paragraph at lines 104--109 []\OT1/ptm/m/n/10 (+20) Ren[]e Just, Dar-i-oush Jalali, and Michael D. Ernst. De-fects4J: A database of ex-ist-ing Underfull \hbox (badness 1019) in paragraph at lines 104--109 \OT1/ptm/m/n/10 (+20) faults to en-able con-trolled test-ing stud-ies for Java pro-grams. In \OT1/ptm/m/it/10 (+20) In-ter-na-tional Sym-po- Underfull \hbox (badness 3078) in paragraph at lines 111--114 \OT1/ptm/m/n/10 (+20) Al-shikh, and Rus-lan Salakhut-di-nov. Om-ni-ACT: A dataset and bench-mark for en- Underfull \hbox (badness 2213) in paragraph at lines 111--114 \OT1/ptm/m/n/10 (+20) abling mul-ti-modal gen-er-al-ist au-tonomous agents for desk-top and web, 2024. URL Underfull \hbox (badness 2503) in paragraph at lines 129--132 []\OT1/ptm/m/n/10 (+20) Jing Yu Koh, Robert Lo, Lawrence Jang, Vikram Duvvur, Ming Chong Lim, Po-Yu Underfull \hbox (badness 8189) in paragraph at lines 147--152 []\OT1/ptm/m/n/10 (+20) Katherine Lee, Daphne Ip-polito, An-drew Nys-trom, Chiyuan Zhang, Dou-glas Eck, Underfull \hbox (badness 2229) in paragraph at lines 147--152 \OT1/ptm/m/n/10 (+20) Chris Callison-Burch, and Nicholas Car-lini. Dedu-pli-cat-ing train-ing data makes lan- Underfull \hbox (badness 4316) in paragraph at lines 147--152 \OT1/ptm/m/n/10 (+20) guage mod-els bet-ter. In \OT1/ptm/m/it/10 (+20) Pro-ceed-ings of the An-nual Meet-ing of the As-so-ci-a-tion Underfull \hbox (badness 3965) in paragraph at lines 147--152 \OT1/ptm/m/it/10 (+20) for Com-pu-ta-tional Lin-guis-tics\OT1/ptm/m/n/10 (+20) , 2022. doi: 10.18653/v1/2022.acl-long.577. URL [11] Underfull \hbox (badness 5771) in paragraph at lines 198--203 []\OT1/ptm/m/n/10 (+20) Fernanda Madeiral, Si-mon Urli, Marcelo Maia, and Mar-tin Mon-per-rus. BEARS: Underfull \hbox (badness 10000) in paragraph at lines 198--203 \OT1/ptm/m/n/10 (+20) An ex-ten-si-ble Java bug bench-mark for au-to-matic pro-gram re-pair stud-ies. In Underfull \hbox (badness 10000) in paragraph at lines 198--203 \OT1/ptm/m/it/10 (+20) Pro-ceed-ings of the In-ter-na-tional Con-fer-ence on Soft-ware Anal-y-sis, Evo-lu- Underfull \hbox (badness 10000) in paragraph at lines 198--203 \OT1/ptm/m/it/10 (+20) tion and Reengi-neer-ing\OT1/ptm/m/n/10 (+20) , 2019. doi: 10.1109/SANER.2019.8667991. URL Underfull \hbox (badness 3049) in paragraph at lines 205--208 []\OT1/ptm/m/n/10 (+20) Gr[]egoire Mi-alon, Cl[]ementine Four-rier, Craig Swift, Thomas Wolf, Yann Le-Cun, and Underfull \hbox (badness 3826) in paragraph at lines 205--208 \OT1/ptm/m/n/10 (+20) Thomas Scialom. GAIA: A bench-mark for gen-eral AI as-sis-tants, 2023. URL Underfull \hbox (badness 1810) in paragraph at lines 235--238 []\OT1/ptm/m/n/10 (+20) Christopher Rawles, Sarah Clinck-e-mail-lie, Yi-fan Chang, Jonathan Waltz, Gabrielle Lau, Underfull \hbox (badness 1924) in paragraph at lines 235--238 \OT1/ptm/m/n/10 (+20) Mary-beth Fair, Al-ice Li, William Bishop, Wei Li, Fo-lawiyo Campbell-Ajala, Daniel [12] Underfull \hbox (badness 1014) in paragraph at lines 277--280 \OT1/ptm/m/n/10 (+20) Yan, Bo Liu, Han-son Wen, Mingyang Xu, Xi-aoyuan Liu, Zi-meng Chen, Weiyan Shi, Underfull \hbox (badness 2293) in paragraph at lines 282--287 \OT1/ptm/m/n/10 (+20) flaws: A pro-gram-ming com-pe-ti-tion bench-mark for eval-u-at-ing au-to-mated pro-gram re- Underfull \hbox (badness 1430) in paragraph at lines 282--287 \OT1/ptm/m/n/10 (+20) pair tools. In \OT1/ptm/m/it/10 (+20) 2017 IEEE/ACM 39th In-ter-na-tional Con-fer-ence on Soft-ware En-gi-neer- Underfull \hbox (badness 1184) in paragraph at lines 282--287 \OT1/ptm/m/it/10 (+20) ing Com-pan-ion (ICSE-C)\OT1/ptm/m/n/10 (+20) , pp. 180--182, 2017. doi: 10.1109/ICSE-C.2017.76. URL Underfull \hbox (badness 4569) in paragraph at lines 289--292 \OT1/ptm/m/n/10 (+20) Li, Xinni Xie, Guangjing Wang, and Ji-ax-uan You. SWE-Bench Mo-bile: Can Underfull \hbox (badness 1221) in paragraph at lines 289--292 \OT1/ptm/m/n/10 (+20) large lan-guage model agents de-velop industry-level mo-bile ap-pli-ca-tions?, 2026. URL [13] Underfull \hbox (badness 2050) in paragraph at lines 342--345 []\OT1/ptm/m/n/10 (+20) John Yang, Kil-ian Lieret, Jef-frey Ma, Parth Thakkar, Dmitrii Ped-chenko, Sten Sootla, Underfull \hbox (badness 1990) in paragraph at lines 342--345 \OT1/ptm/m/n/10 (+20) Pro-gram-Bench: Can lan-guage mod-els re-build pro-grams from scratch?, 2026. URL Underfull \hbox (badness 3758) in paragraph at lines 353--356 []\OT1/ptm/m/n/10 (+20) Shunyu Yao, Noah Shinn, Pe-dram Razavi, and Karthik Narasimhan. $\OML/cmm/m/it/10 ^^\$\OT1/ptm/m/n/10 (+20) -bench: A Underfull \hbox (badness 10000) in paragraph at lines 353--356 \OT1/ptm/m/n/10 (+20) bench-mark for tool-agent-user in-ter-ac-tion in real-world do-mains, 2024. URL ) [14] (./tables/livegui-bench-split-counts-table.tex) (./tables/livegui-bench-app-domain-table.tex) (./tables/livegui-bench-assertion-counts-table.tex) Underfull \vbox (badness 10000) has occurred while \output is active [15] (./tables/release-governance-table.tex Underfull \hbox (badness 10000) in paragraph at lines 8--8 []|\OT1/ptm/m/n/7 (+20) hashes, meta-data, schema, Underfull \hbox (badness 10000) in paragraph at lines 8--9 []|\OT1/ptm/m/n/7 (+20) sealed-test rank-ing Underfull \hbox (badness 3407) in paragraph at lines 8--9 \OT1/ptm/m/n/7 (+20) archived app pack- Underfull \hbox (badness 6364) in paragraph at lines 9--9 []|\OT1/ptm/m/n/7 (+20) periodic para-phrase and new- ) (./tables/release-gates-table.tex) [16] Underfull \vbox (badness 10000) has occurred while \output is active [17 <./figures/trajectories/fitness-step-01.png (PNG copy)> <./figures/trajectories/fitness-step-04.png (PNG copy)> <./figures/trajectories/fitness-step-06.png (PNG copy)>] Underfull \hbox (badness 1314) in paragraph at lines 354--354 \OT1/ptm/m/n/10 (+20) face opens, then a sub-mit routes into the app-scoped work-flow, which in-vokes the real Underfull \vbox (badness 10000) has occurred while \output is active [18 <./figures/trajectories/amazon-step-01.png (PNG copy)> <./figures/trajectories/amazon-step-05.png (PNG copy)> <./figures/trajectories/amazon-step-09.png (PNG copy)> <./figures/trajectories/xdomain-amazon-open.png (PNG copy)> <./figures/trajectories/xdomain-amazon-success.png (PNG copy)> <./figures/trajectories/xdomain-venmo-fail.png (PNG copy)>] [19] [20] (/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.aux) LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right. ) (see the transcript file for additional information) Output written on /home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.pdf (20 pages, 2168447 bytes). Transcript written on /home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.log. Latexmk: Getting log file '/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.log' Latexmk: Examining '/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.fls' Latexmk: Examining '/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.log' Latexmk: Found input bbl file 'main.bbl' Latexmk: References changed. Latexmk: Log file says output to '/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.pdf' Latexmk: Found bibliography file(s): ./references.bib Latexmk: applying rule 'pdflatex'... Rule 'pdflatex': Reasons for rerun Changed files or newly in use/created: /home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.aux ------------ Run number 3 of rule 'pdflatex' ------------ ------------ Running 'pdflatex -interaction=nonstopmode -halt-on-error -file-line-error -recorder -output-directory="/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench" "main.tex"' ------------ This is pdfTeX, Version 3.141592653-2.6-1.40.25 (TeX Live 2023/Debian) (preloaded format=pdflatex) restricted \write18 enabled. entering extended mode (./main.tex LaTeX2e <2023-11-01> patch level 1 L3 programming layer <2024-01-22> (/usr/share/texlive/texmf-dist/tex/latex/base/article.cls Document Class: article 2023/05/17 v1.4n Standard LaTeX document class (/usr/share/texlive/texmf-dist/tex/latex/base/size10.clo)) (./iclr2027_conference.sty (/usr/share/texlive/texmf-dist/tex/latex/eso-pic/eso-pic.sty (/usr/share/texlive/texmf-dist/tex/latex/graphics/keyval.sty) (/usr/share/texlive/texmf-dist/tex/latex/xcolor/xcolor.sty (/usr/share/texlive/texmf-dist/tex/latex/graphics-cfg/color.cfg) (/usr/share/texlive/texmf-dist/tex/latex/graphics-def/pdftex.def) (/usr/share/texlive/texmf-dist/tex/latex/graphics/mathcolor.ltx))) (./fancyhdr.sty) (./natbib.sty)) (/usr/share/texlive/texmf-dist/tex/latex/psnfss/times.sty) (/usr/share/texlive/texmf-dist/tex/latex/microtype/microtype.sty (/usr/share/texlive/texmf-dist/tex/latex/etoolbox/etoolbox.sty) (/usr/share/texlive/texmf-dist/tex/latex/microtype/microtype-pdftex.def) (/usr/share/texlive/texmf-dist/tex/latex/microtype/microtype.cfg)) (/usr/share/texlive/texmf-dist/tex/latex/graphics/graphicx.sty (/usr/share/texlive/texmf-dist/tex/latex/graphics/graphics.sty (/usr/share/texlive/texmf-dist/tex/latex/graphics/trig.sty) (/usr/share/texlive/texmf-dist/tex/latex/graphics-cfg/graphics.cfg))) (/usr/share/texlive/texmf-dist/tex/latex/float/float.sty) (/usr/share/texlive/texmf-dist/tex/latex/booktabs/booktabs.sty) (/usr/share/texlive/texmf-dist/tex/latex/tools/array.sty) (/usr/share/texlive/texmf-dist/tex/latex/enumitem/enumitem.sty) (/usr/share/texlive/texmf-dist/tex/latex/pgf/frontendlayer/tikz.sty (/usr/share/texlive/texmf-dist/tex/latex/pgf/basiclayer/pgf.sty (/usr/share/texlive/texmf-dist/tex/latex/pgf/utilities/pgfrcs.sty (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfutil-common.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfutil-latex.def) (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfrcs.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/pgf.revision.tex))) (/usr/share/texlive/texmf-dist/tex/latex/pgf/basiclayer/pgfcore.sty (/usr/share/texlive/texmf-dist/tex/latex/pgf/systemlayer/pgfsys.sty (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgfsys.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfkeyslibraryfiltered.code.tex)) (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgf.cfg) (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-pdftex.def (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-common-pdf.def))) (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgfsyssoftpath.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/systemlayer/pgfsysprotocol.code.tex)) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcore.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathutil.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathparser.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.basic.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.trigonometric.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.random.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.comparison.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.base.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.round.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.misc.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.integerarithmetics.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathcalc.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmathfloat.code.tex)) (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfint.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepoints.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathconstruct.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathusage.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorescopes.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoregraphicstate.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretransformations.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorequick.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreobjects.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathprocessing.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorearrows.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreshade.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreimage.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreexternal.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorelayers.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretransparency.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepatterns.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/basiclayer/pgfcorerdf.code.tex))) (/usr/share/texlive/texmf-dist/tex/generic/pgf/modules/pgfmoduleshapes.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/modules/pgfmoduleplot.code.tex) (/usr/share/texlive/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version-0-65.sty) (/usr/share/texlive/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version-1-18.sty)) (/usr/share/texlive/texmf-dist/tex/latex/pgf/utilities/pgffor.sty (/usr/share/texlive/texmf-dist/tex/latex/pgf/utilities/pgfkeys.sty (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex)) (/usr/share/texlive/texmf-dist/tex/latex/pgf/math/pgfmath.sty (/usr/share/texlive/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex)) (/usr/share/texlive/texmf-dist/tex/generic/pgf/utilities/pgffor.code.tex)) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/tikz.code.tex (/usr/share/texlive/texmf-dist/tex/generic/pgf/libraries/pgflibraryplothandlers.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/modules/pgfmodulematrix.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarytopaths.code.tex))) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarypositioning.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibraryfit.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarycalc.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/libraries/pgflibraryarrows.meta.code.tex) (/usr/share/texlive/texmf-dist/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarybackgrounds.code.tex) (./figures/tikz/styles.tex) (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsmath.sty For additional information on amsmath, use the `?' option. (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amstext.sty (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsgen.sty)) (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsbsy.sty) (/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsopn.sty)) (/usr/share/texlive/texmf-dist/tex/latex/amsfonts/amssymb.sty (/usr/share/texlive/texmf-dist/tex/latex/amsfonts/amsfonts.sty)) (/usr/share/texlive/texmf-dist/tex/latex/psnfss/ot1ptm.fd) (/usr/share/texlive/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def) (/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.aux) (/usr/share/texlive/texmf-dist/tex/context/base/mkii/supp-pdf.mkii [Loading MPS to PDF converter (version 2006.09.02).] ) (/usr/share/texlive/texmf-dist/tex/latex/epstopdf-pkg/epstopdf-base.sty (/usr/share/texlive/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg)) (/usr/share/texlive/texmf-dist/tex/latex/microtype/mt-ptm.cfg) (/usr/share/texlive/texmf-dist/tex/latex/microtype/mt-cmr.cfg) (/usr/share/texlive/texmf-dist/tex/latex/amsfonts/umsa.fd) (/usr/share/texlive/texmf-dist/tex/latex/microtype/mt-msa.cfg) (/usr/share/texlive/texmf-dist/tex/latex/amsfonts/umsb.fd) (/usr/share/texlive/texmf-dist/tex/latex/microtype/mt-msb.cfg) (/usr/share/texlive/texmf-dist/tex/latex/psnfss/ot1pcr.fd) (./figures/tikz/fig-problem-gap.tex) [1{/var/lib/texmf/fonts/map/pdftex/updmap/pdftex.map}{/usr/share/texlive/texmf-dist/fonts/enc/dvips/base/8r.enc}] (/usr/share/texlive/texmf-dist/tex/latex/psnfss/ts1ptm.fd) Underfull \vbox (badness 10000) has occurred while \output is active [2] Underfull \hbox (badness 10000) in paragraph at lines 87--88 []|\OT1/ptm/m/n/9 (+20) hidden back-end mem-ory, Underfull \hbox (badness 1400) in paragraph at lines 101--102 \OT1/ptm/m/sc/10 (+20) LiveGUI-Bench \OT1/ptm/m/n/10 (+20) tar-gets re-cur-ring per-sonal work-flows in which a per-sis-tent vis-i-ble sur- Underfull \hbox (badness 2818) in paragraph at lines 101--102 \OT1/ptm/m/n/10 (+20) face is more ap-pro-pri-ate than a one-off an-swer, and grounds ev-ery case in the ex-e- Underfull \hbox (badness 1527) in paragraph at lines 101--102 \OT1/ptm/m/n/10 (+20) cutable ap-pli-ca-tion APIs of Ap-p-World (Trivedi et al., 2024). The tracked case file is [3] (./tables/dataset-stats-table.tex) Underfull \hbox (badness 2503) in paragraph at lines 113--113 []\OT1/ptm/m/n/10 (+20) Table 2: \OT1/ptm/m/sc/10 (+20) LiveGUI-Bench \OT1/ptm/m/n/10 (+20) statis-tics --- case, app-domain, episode, and deterministic- [4] (./figures/tikz/fig-evaluator-architecture.tex) (./figures/tikz/fig-gui-behavior-gates.tex) (./figures/tikz/fig-benchmark-pipeline.tex) Underfull \vbox (badness 10000) has occurred while \output is active [5] (./tables/appworld-bench-table.tex) LaTeX Font Warning: Font shape `OT1/ptm/m/scit' undefined (Font) using `OT1/ptm/m/sc' instead on input line 183. LaTeX Font Warning: Font shape `OT1/ptm/m/scit' undefined (Font) using `OT1/ptm/m/sc' instead on input line 183. Underfull \vbox (badness 2495) has occurred while \output is active [6] Underfull \vbox (badness 1406) has occurred while \output is active [7] [8] Underfull \hbox (badness 3482) in paragraph at lines 239--240 \OT1/ptm/m/n/10 (+20) All em-pir-i-cal num-bers are gen-er-ated from archived run ar-ti-facts (\OT1/pcr/m/n/10 report.json \OT1/ptm/m/n/10 (+20) and (./main.bbl Underfull \hbox (badness 1694) in paragraph at lines 9--12 []\OT1/ptm/m/n/10 (+20) Reem Alei-than, Hao-ran Xue, Mo-ham-mad Mahdi Mo-ha-jer, Eli-jah Nnorom, Gias Ud-din, [9] Underfull \hbox (badness 2293) in paragraph at lines 93--96 []\OT1/ptm/m/n/10 (+20) Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fan-jia Yan, Tian-jun Zhang, Sida I. [10] Underfull \hbox (badness 1824) in paragraph at lines 98--102 []\OT1/ptm/m/n/10 (+20) Carlos E. Jimenez, John Yang, Alexan-der Wet-tig, Shunyu Yao, Kexin Pei, Ofir Press, Underfull \hbox (badness 3009) in paragraph at lines 98--102 \OT1/ptm/m/n/10 (+20) is-sues? In \OT1/ptm/m/it/10 (+20) In-ter-na-tional Con-fer-ence on Learn-ing Rep-re-sen-ta-tions\OT1/ptm/m/n/10 (+20) , 2024. URL Underfull \hbox (badness 1742) in paragraph at lines 104--109 []\OT1/ptm/m/n/10 (+20) Ren[]e Just, Dar-i-oush Jalali, and Michael D. Ernst. De-fects4J: A database of ex-ist-ing Underfull \hbox (badness 1019) in paragraph at lines 104--109 \OT1/ptm/m/n/10 (+20) faults to en-able con-trolled test-ing stud-ies for Java pro-grams. In \OT1/ptm/m/it/10 (+20) In-ter-na-tional Sym-po- Underfull \hbox (badness 3078) in paragraph at lines 111--114 \OT1/ptm/m/n/10 (+20) Al-shikh, and Rus-lan Salakhut-di-nov. Om-ni-ACT: A dataset and bench-mark for en- Underfull \hbox (badness 2213) in paragraph at lines 111--114 \OT1/ptm/m/n/10 (+20) abling mul-ti-modal gen-er-al-ist au-tonomous agents for desk-top and web, 2024. URL Underfull \hbox (badness 2503) in paragraph at lines 129--132 []\OT1/ptm/m/n/10 (+20) Jing Yu Koh, Robert Lo, Lawrence Jang, Vikram Duvvur, Ming Chong Lim, Po-Yu Underfull \hbox (badness 8189) in paragraph at lines 147--152 []\OT1/ptm/m/n/10 (+20) Katherine Lee, Daphne Ip-polito, An-drew Nys-trom, Chiyuan Zhang, Dou-glas Eck, Underfull \hbox (badness 2229) in paragraph at lines 147--152 \OT1/ptm/m/n/10 (+20) Chris Callison-Burch, and Nicholas Car-lini. Dedu-pli-cat-ing train-ing data makes lan- Underfull \hbox (badness 4316) in paragraph at lines 147--152 \OT1/ptm/m/n/10 (+20) guage mod-els bet-ter. In \OT1/ptm/m/it/10 (+20) Pro-ceed-ings of the An-nual Meet-ing of the As-so-ci-a-tion Underfull \hbox (badness 3965) in paragraph at lines 147--152 \OT1/ptm/m/it/10 (+20) for Com-pu-ta-tional Lin-guis-tics\OT1/ptm/m/n/10 (+20) , 2022. doi: 10.18653/v1/2022.acl-long.577. URL [11] Underfull \hbox (badness 5771) in paragraph at lines 198--203 []\OT1/ptm/m/n/10 (+20) Fernanda Madeiral, Si-mon Urli, Marcelo Maia, and Mar-tin Mon-per-rus. BEARS: Underfull \hbox (badness 10000) in paragraph at lines 198--203 \OT1/ptm/m/n/10 (+20) An ex-ten-si-ble Java bug bench-mark for au-to-matic pro-gram re-pair stud-ies. In Underfull \hbox (badness 10000) in paragraph at lines 198--203 \OT1/ptm/m/it/10 (+20) Pro-ceed-ings of the In-ter-na-tional Con-fer-ence on Soft-ware Anal-y-sis, Evo-lu- Underfull \hbox (badness 10000) in paragraph at lines 198--203 \OT1/ptm/m/it/10 (+20) tion and Reengi-neer-ing\OT1/ptm/m/n/10 (+20) , 2019. doi: 10.1109/SANER.2019.8667991. URL Underfull \hbox (badness 3049) in paragraph at lines 205--208 []\OT1/ptm/m/n/10 (+20) Gr[]egoire Mi-alon, Cl[]ementine Four-rier, Craig Swift, Thomas Wolf, Yann Le-Cun, and Underfull \hbox (badness 3826) in paragraph at lines 205--208 \OT1/ptm/m/n/10 (+20) Thomas Scialom. GAIA: A bench-mark for gen-eral AI as-sis-tants, 2023. URL Underfull \hbox (badness 1810) in paragraph at lines 235--238 []\OT1/ptm/m/n/10 (+20) Christopher Rawles, Sarah Clinck-e-mail-lie, Yi-fan Chang, Jonathan Waltz, Gabrielle Lau, Underfull \hbox (badness 1924) in paragraph at lines 235--238 \OT1/ptm/m/n/10 (+20) Mary-beth Fair, Al-ice Li, William Bishop, Wei Li, Fo-lawiyo Campbell-Ajala, Daniel [12] Underfull \hbox (badness 1014) in paragraph at lines 277--280 \OT1/ptm/m/n/10 (+20) Yan, Bo Liu, Han-son Wen, Mingyang Xu, Xi-aoyuan Liu, Zi-meng Chen, Weiyan Shi, Underfull \hbox (badness 2293) in paragraph at lines 282--287 \OT1/ptm/m/n/10 (+20) flaws: A pro-gram-ming com-pe-ti-tion bench-mark for eval-u-at-ing au-to-mated pro-gram re- Underfull \hbox (badness 1430) in paragraph at lines 282--287 \OT1/ptm/m/n/10 (+20) pair tools. In \OT1/ptm/m/it/10 (+20) 2017 IEEE/ACM 39th In-ter-na-tional Con-fer-ence on Soft-ware En-gi-neer- Underfull \hbox (badness 1184) in paragraph at lines 282--287 \OT1/ptm/m/it/10 (+20) ing Com-pan-ion (ICSE-C)\OT1/ptm/m/n/10 (+20) , pp. 180--182, 2017. doi: 10.1109/ICSE-C.2017.76. URL Underfull \hbox (badness 4569) in paragraph at lines 289--292 \OT1/ptm/m/n/10 (+20) Li, Xinni Xie, Guangjing Wang, and Ji-ax-uan You. SWE-Bench Mo-bile: Can Underfull \hbox (badness 1221) in paragraph at lines 289--292 \OT1/ptm/m/n/10 (+20) large lan-guage model agents de-velop industry-level mo-bile ap-pli-ca-tions?, 2026. URL [13] Underfull \hbox (badness 2050) in paragraph at lines 342--345 []\OT1/ptm/m/n/10 (+20) John Yang, Kil-ian Lieret, Jef-frey Ma, Parth Thakkar, Dmitrii Ped-chenko, Sten Sootla, Underfull \hbox (badness 1990) in paragraph at lines 342--345 \OT1/ptm/m/n/10 (+20) Pro-gram-Bench: Can lan-guage mod-els re-build pro-grams from scratch?, 2026. URL Underfull \hbox (badness 3758) in paragraph at lines 353--356 []\OT1/ptm/m/n/10 (+20) Shunyu Yao, Noah Shinn, Pe-dram Razavi, and Karthik Narasimhan. $\OML/cmm/m/it/10 ^^\$\OT1/ptm/m/n/10 (+20) -bench: A Underfull \hbox (badness 10000) in paragraph at lines 353--356 \OT1/ptm/m/n/10 (+20) bench-mark for tool-agent-user in-ter-ac-tion in real-world do-mains, 2024. URL ) [14] (./tables/livegui-bench-split-counts-table.tex) (./tables/livegui-bench-app-domain-table.tex) (./tables/livegui-bench-assertion-counts-table.tex) Underfull \vbox (badness 10000) has occurred while \output is active [15] (./tables/release-governance-table.tex Underfull \hbox (badness 10000) in paragraph at lines 8--8 []|\OT1/ptm/m/n/7 (+20) hashes, meta-data, schema, Underfull \hbox (badness 10000) in paragraph at lines 8--9 []|\OT1/ptm/m/n/7 (+20) sealed-test rank-ing Underfull \hbox (badness 3407) in paragraph at lines 8--9 \OT1/ptm/m/n/7 (+20) archived app pack- Underfull \hbox (badness 6364) in paragraph at lines 9--9 []|\OT1/ptm/m/n/7 (+20) periodic para-phrase and new- ) (./tables/release-gates-table.tex) [16] Underfull \vbox (badness 10000) has occurred while \output is active [17 <./figures/trajectories/fitness-step-01.png (PNG copy)> <./figures/trajectories/fitness-step-04.png (PNG copy)> <./figures/trajectories/fitness-step-06.png (PNG copy)>] Underfull \hbox (badness 1314) in paragraph at lines 354--354 \OT1/ptm/m/n/10 (+20) face opens, then a sub-mit routes into the app-scoped work-flow, which in-vokes the real Underfull \vbox (badness 10000) has occurred while \output is active [18 <./figures/trajectories/amazon-step-01.png (PNG copy)> <./figures/trajectories/amazon-step-05.png (PNG copy)> <./figures/trajectories/amazon-step-09.png (PNG copy)> <./figures/trajectories/xdomain-amazon-open.png (PNG copy)> <./figures/trajectories/xdomain-amazon-success.png (PNG copy)> <./figures/trajectories/xdomain-venmo-fail.png (PNG copy)>] [19] [20] (/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.aux) ) (see the transcript file for additional information) Output written on /home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.pdf (20 pages, 2168447 bytes). Transcript written on /home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.log. Latexmk: Getting log file '/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.log' Latexmk: Examining '/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.fls' Latexmk: Examining '/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.log' Latexmk: Found input bbl file 'main.bbl' Latexmk: Log file says output to '/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.pdf' Latexmk: Found bibliography file(s): ./references.bib Latexmk: All targets (/home/runner/work/tongLab/tongLab/arxiv/build/pdfs/liveguiBench/main.pdf) are up-to-date exit_code=0