{"date":"2026-09-24","suite_sha256":"6ef6c158aba336836457f59c1e15e64487eb18c34b14128082bac3b6a2bdade0","models":{"clm":"clm-latest","jev":"jev-1.13.0"},"host":"US-KS-2 / NVIDIA RTX PRO 4500 Blackwell Server Edition","method":"Identical typed requests; serial client calls from one US RunPod pod; CLM loopback, Jev hosted HTTPS; no client answer cache.","summary":{"clm":{"count":369,"correct":295,"kind_counts":{"tool_routing":40,"trajectory_selection":24,"result_relevance":40,"subgoal_satisfaction":40,"file_context":225},"by_kind":{"file_context":{"count":225,"correct":211,"accuracy":0.9378,"latency_ms_p50":35.56,"latency_ms_p95":51.38,"input_tokens":9773,"retries":0,"tp":4,"fp":0,"fn":14,"tn":207,"precision":1.0,"recall":0.2222,"f1":0.3636,"brier":0.087},"result_relevance":{"count":40,"correct":22,"accuracy":0.55,"latency_ms_p50":36.32,"latency_ms_p95":51.97,"input_tokens":1954,"retries":0,"tp":2,"fp":0,"fn":18,"tn":20,"precision":1.0,"recall":0.1,"f1":0.1818,"brier":0.2539},"subgoal_satisfaction":{"count":40,"correct":28,"accuracy":0.7,"latency_ms_p50":37.79,"latency_ms_p95":59.39,"input_tokens":1956,"retries":0,"tp":8,"fp":0,"fn":12,"tn":20,"precision":1.0,"recall":0.4,"f1":0.5714,"brier":0.1501},"tool_routing":{"count":40,"correct":20,"accuracy":0.5,"latency_ms_p50":35.63,"latency_ms_p95":51.67,"input_tokens":1300,"retries":0},"trajectory_selection":{"count":24,"correct":14,"accuracy":0.5833,"latency_ms_p50":90.22,"latency_ms_p95":99.11,"input_tokens":1270,"retries":0}},"input_tokens":16253,"retries":0},"jev":{"count":369,"correct":350,"kind_counts":{"tool_routing":40,"trajectory_selection":24,"result_relevance":40,"subgoal_satisfaction":40,"file_context":225},"by_kind":{"file_context":{"count":225,"correct":212,"accuracy":0.9422,"latency_ms_p50":114.29,"latency_ms_p95":148.62,"input_tokens":69750,"retries":0,"tp":18,"fp":13,"fn":0,"tn":194,"precision":0.5806,"recall":1.0,"f1":0.7347,"brier":0.0642},"result_relevance":{"count":40,"correct":34,"accuracy":0.85,"latency_ms_p50":115.64,"latency_ms_p95":174.37,"input_tokens":12564,"retries":0,"tp":20,"fp":6,"fn":0,"tn":14,"precision":0.7692,"recall":1.0,"f1":0.8696,"brier":0.1101},"subgoal_satisfaction":{"count":40,"correct":40,"accuracy":1.0,"latency_ms_p50":115.98,"latency_ms_p95":208.58,"input_tokens":12682,"retries":0,"tp":20,"fp":0,"fn":0,"tn":20,"precision":1.0,"recall":1.0,"f1":1.0,"brier":0.0067},"tool_routing":{"count":40,"correct":40,"accuracy":1.0,"latency_ms_p50":116.27,"latency_ms_p95":154.62,"input_tokens":26372,"retries":0},"trajectory_selection":{"count":24,"correct":24,"accuracy":1.0,"latency_ms_p50":136.29,"latency_ms_p95":162.91,"input_tokens":9780,"retries":0}},"input_tokens":131148,"retries":0}},"rows":[{"id":"tool-00-0","kind":"tool_routing","group":null,"file":null,"expected":"repo_search","state":"Agent goal: Find the function that invalidates cached user permissions; no path is known.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"inspect_logs":"Read service logs and stack traces for a runtime failure.","read_file":"Read the full contents of a known file or a specific line range.","fetch_web":"Open a current public web page or official documentation.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","run_tests":"Run tests or checks to verify a code change.","query_database":"Read structured records from the authorized application database.","deploy_site":"Publish a tested website build to its authorized hosting target.","calendar":"Read availability and schedule an approved meeting.","edit_file":"Apply an authorized code or document change to a known file.","upload_artifact":"Upload a completed artifact to the designated storage location.","parse_document":"Extract text and structure from a supplied PDF or document.","send_email":"Send an already approved message to a verified recipient.","calculate":"Compute a numeric result from supplied data or a formula.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown."}}},"clm":{"predicted":"repo_search","correct":true,"score":0.58431,"confidence":0.58431,"probabilities":{"inspect_logs":0.027284785634321865,"read_file":0.010741763798675197,"fetch_web":0.019973934559643948,"inspect_diff":0.0008195327904499465,"create_issue":0.007768691536529873,"run_tests":0.05863090088841867,"query_database":0.01876699633304925,"deploy_site":0.0011990956679518196,"calendar":0.015348409626253777,"edit_file":0.0578365263057405,"upload_artifact":0.014039674038400033,"parse_document":0.001493810838798484,"send_email":0.07044812387134668,"calculate":0.11133789842938935,"repo_search":0.5843098556810307},"latency_ms":39.96,"input_tokens":27,"retries":0},"jev":{"predicted":"repo_search","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"edit_file":0.0,"calculate":0.0,"run_tests":0.0,"deploy_site":0.0,"inspect_diff":0.0,"inspect_logs":0.0,"read_file":0.0,"parse_document":0.0,"calendar":0.0,"create_issue":0.0,"repo_search":1.0,"send_email":0.0,"query_database":0.0,"fetch_web":0.0,"upload_artifact":0.0},"latency_ms":113.4,"input_tokens":655,"retries":0}},{"id":"tool-00-1","kind":"tool_routing","group":null,"file":null,"expected":"repo_search","state":"Current task: Find the function that invalidates cached user permissions; no path is known.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","calendar":"Read availability and schedule an approved meeting.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","send_email":"Send an already approved message to a verified recipient.","query_database":"Read structured records from the authorized application database.","read_file":"Read the full contents of a known file or a specific line range.","edit_file":"Apply an authorized code or document change to a known file.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","run_tests":"Run tests or checks to verify a code change.","calculate":"Compute a numeric result from supplied data or a formula.","inspect_logs":"Read service logs and stack traces for a runtime failure.","deploy_site":"Publish a tested website build to its authorized hosting target.","fetch_web":"Open a current public web page or official documentation.","parse_document":"Extract text and structure from a supplied PDF or document.","upload_artifact":"Upload a completed artifact to the designated storage location."}}},"clm":{"predicted":"calculate","correct":false,"score":0.172345,"confidence":0.237497,"probabilities":{"inspect_diff":0.0011736034443269837,"calendar":0.014412466505982312,"create_issue":0.0190940714944835,"send_email":0.07353018438988242,"query_database":0.013057734503908664,"read_file":0.02565260174468776,"edit_file":0.17786238380352426,"repo_search":0.17234542144532788,"run_tests":0.07044149851209885,"calculate":0.2374973700447511,"inspect_logs":0.10763974671373959,"deploy_site":0.00313837231552796,"fetch_web":0.008466222614549154,"parse_document":0.010363026407757799,"upload_artifact":0.06532529605945175},"latency_ms":51.67,"input_tokens":37,"retries":0},"jev":{"predicted":"repo_search","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"fetch_web":0.0,"edit_file":0.0,"create_issue":0.0,"inspect_diff":0.0,"calendar":0.0,"calculate":0.0,"query_database":0.0,"read_file":0.0,"run_tests":0.0,"send_email":0.0,"upload_artifact":0.0,"parse_document":0.0,"repo_search":1.0,"inspect_logs":0.0,"deploy_site":0.0},"latency_ms":107.85,"input_tokens":666,"retries":0}},{"id":"tool-01-0","kind":"tool_routing","group":null,"file":null,"expected":"read_file","state":"Agent goal: The error points to src/auth/session.py line 81. Inspect that implementation.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","edit_file":"Apply an authorized code or document change to a known file.","send_email":"Send an already approved message to a verified recipient.","calendar":"Read availability and schedule an approved meeting.","parse_document":"Extract text and structure from a supplied PDF or document.","query_database":"Read structured records from the authorized application database.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","run_tests":"Run tests or checks to verify a code change.","calculate":"Compute a numeric result from supplied data or a formula.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","read_file":"Read the full contents of a known file or a specific line range.","fetch_web":"Open a current public web page or official documentation.","deploy_site":"Publish a tested website build to its authorized hosting target.","inspect_logs":"Read service logs and stack traces for a runtime failure.","upload_artifact":"Upload a completed artifact to the designated storage location."}}},"clm":{"predicted":"run_tests","correct":false,"score":0.002938,"confidence":0.626773,"probabilities":{"repo_search":0.13029149356987269,"edit_file":0.006683905056059895,"send_email":0.00038932606996993545,"calendar":3.321814761762873e-05,"parse_document":0.0007424613441079287,"query_database":0.0017662210567289608,"create_issue":0.0022449938969929823,"run_tests":0.626773051994758,"calculate":0.0184433468841813,"inspect_diff":0.005190380718996613,"read_file":0.0029379313672968587,"fetch_web":0.002737184210005355,"deploy_site":0.0001793671399131577,"inspect_logs":0.19366702490711668,"upload_artifact":0.007920093636382002},"latency_ms":37.2,"input_tokens":30,"retries":0},"jev":{"predicted":"read_file","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"read_file":1.0,"calculate":0.0,"upload_artifact":0.0,"create_issue":0.0,"inspect_logs":0.0,"send_email":0.0,"fetch_web":0.0,"parse_document":0.0,"inspect_diff":0.0,"query_database":0.0,"deploy_site":0.0,"run_tests":0.0,"edit_file":0.0,"calendar":0.0,"repo_search":0.0},"latency_ms":110.89,"input_tokens":656,"retries":0}},{"id":"tool-01-1","kind":"tool_routing","group":null,"file":null,"expected":"read_file","state":"Current task: The error points to src/auth/session.py line 81. Inspect that implementation.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"create_issue":"Create a tracked issue with a title, description, and reproduction steps.","read_file":"Read the full contents of a known file or a specific line range.","calendar":"Read availability and schedule an approved meeting.","run_tests":"Run tests or checks to verify a code change.","upload_artifact":"Upload a completed artifact to the designated storage location.","calculate":"Compute a numeric result from supplied data or a formula.","fetch_web":"Open a current public web page or official documentation.","query_database":"Read structured records from the authorized application database.","send_email":"Send an already approved message to a verified recipient.","edit_file":"Apply an authorized code or document change to a known file.","parse_document":"Extract text and structure from a supplied PDF or document.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","inspect_logs":"Read service logs and stack traces for a runtime failure.","deploy_site":"Publish a tested website build to its authorized hosting target."}}},"clm":{"predicted":"run_tests","correct":false,"score":0.008227,"confidence":0.604245,"probabilities":{"create_issue":0.009802136110762569,"read_file":0.008226783414823536,"calendar":0.0002748303580583744,"run_tests":0.6042452657933436,"upload_artifact":0.03549125315115048,"calculate":0.04221400359643214,"fetch_web":0.003335123348317192,"query_database":0.004451581202681761,"send_email":0.007788412325454584,"edit_file":0.11618947342116816,"parse_document":0.001315332838608233,"repo_search":0.003331677336154814,"inspect_diff":0.0025316592025171193,"inspect_logs":0.15995033541881584,"deploy_site":0.0008521324817116521},"latency_ms":34.93,"input_tokens":40,"retries":0},"jev":{"predicted":"read_file","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"query_database":0.0,"fetch_web":0.0,"read_file":1.0,"inspect_diff":0.0,"deploy_site":0.0,"calculate":0.0,"parse_document":0.0,"calendar":0.0,"create_issue":0.0,"inspect_logs":0.0,"edit_file":0.0,"repo_search":0.0,"run_tests":0.0,"send_email":0.0,"upload_artifact":0.0},"latency_ms":120.33,"input_tokens":667,"retries":0}},{"id":"tool-02-0","kind":"tool_routing","group":null,"file":null,"expected":"edit_file","state":"Agent goal: The approved patch is ready for src/api/routes.py. Apply it to the file.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"deploy_site":"Publish a tested website build to its authorized hosting target.","read_file":"Read the full contents of a known file or a specific line range.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","upload_artifact":"Upload a completed artifact to the designated storage location.","fetch_web":"Open a current public web page or official documentation.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","send_email":"Send an already approved message to a verified recipient.","run_tests":"Run tests or checks to verify a code change.","calendar":"Read availability and schedule an approved meeting.","query_database":"Read structured records from the authorized application database.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","parse_document":"Extract text and structure from a supplied PDF or document.","inspect_logs":"Read service logs and stack traces for a runtime failure.","edit_file":"Apply an authorized code or document change to a known file.","calculate":"Compute a numeric result from supplied data or a formula."}}},"clm":{"predicted":"run_tests","correct":false,"score":0.275354,"confidence":0.386968,"probabilities":{"deploy_site":0.00508937238161067,"read_file":0.01707841615700014,"inspect_diff":0.009201751980982078,"upload_artifact":0.1391698128949867,"fetch_web":0.0007477258612708947,"repo_search":0.13800726707145372,"send_email":0.003354585683638194,"run_tests":0.38696753991110494,"calendar":9.936218857499842e-05,"query_database":0.0010243307135942166,"create_issue":0.0036791723185089133,"parse_document":0.0013826348499592184,"inspect_logs":0.006445016314199186,"edit_file":0.27535422869785975,"calculate":0.012398782975256381},"latency_ms":34.54,"input_tokens":29,"retries":0},"jev":{"predicted":"edit_file","correct":true,"score":0.99,"confidence":0.99,"probabilities":{"calculate":0.0,"calendar":0.0,"repo_search":0.0,"deploy_site":0.0,"send_email":0.0,"query_database":0.0,"edit_file":0.99,"inspect_diff":0.0,"run_tests":0.0,"upload_artifact":0.0,"parse_document":0.0,"read_file":0.01,"fetch_web":0.0,"inspect_logs":0.0,"create_issue":0.0},"latency_ms":114.7,"input_tokens":655,"retries":0}},{"id":"tool-02-1","kind":"tool_routing","group":null,"file":null,"expected":"edit_file","state":"Current task: The approved patch is ready for src/api/routes.py. Apply it to the file.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"parse_document":"Extract text and structure from a supplied PDF or document.","fetch_web":"Open a current public web page or official documentation.","calendar":"Read availability and schedule an approved meeting.","send_email":"Send an already approved message to a verified recipient.","deploy_site":"Publish a tested website build to its authorized hosting target.","calculate":"Compute a numeric result from supplied data or a formula.","run_tests":"Run tests or checks to verify a code change.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","upload_artifact":"Upload a completed artifact to the designated storage location.","inspect_logs":"Read service logs and stack traces for a runtime failure.","edit_file":"Apply an authorized code or document change to a known file.","read_file":"Read the full contents of a known file or a specific line range.","query_database":"Read structured records from the authorized application database."}}},"clm":{"predicted":"edit_file","correct":true,"score":0.460665,"confidence":0.460665,"probabilities":{"parse_document":0.0020806653856913506,"fetch_web":0.0005629753877671154,"calendar":7.987019347523065e-05,"send_email":0.003451248128651716,"deploy_site":0.0009047238468809743,"calculate":0.020726060853717386,"run_tests":0.2817812865615848,"inspect_diff":0.026768159822724957,"create_issue":0.008175218029810598,"repo_search":0.026270902830920873,"upload_artifact":0.10725874798223267,"inspect_logs":0.008036094097623018,"edit_file":0.4606654613826497,"read_file":0.051460953950078964,"query_database":0.001777631546190579},"latency_ms":35.62,"input_tokens":39,"retries":0},"jev":{"predicted":"edit_file","correct":true,"score":0.98,"confidence":0.98,"probabilities":{"edit_file":0.98,"upload_artifact":0.0,"inspect_diff":0.0,"send_email":0.0,"create_issue":0.0,"repo_search":0.0,"query_database":0.0,"fetch_web":0.0,"calendar":0.0,"calculate":0.0,"run_tests":0.0,"parse_document":0.0,"read_file":0.02,"deploy_site":0.0,"inspect_logs":0.0},"latency_ms":153.24,"input_tokens":666,"retries":0}},{"id":"tool-03-0","kind":"tool_routing","group":null,"file":null,"expected":"run_tests","state":"Agent goal: A fix has been applied. Check whether the relevant test suite passes.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"read_file":"Read the full contents of a known file or a specific line range.","query_database":"Read structured records from the authorized application database.","edit_file":"Apply an authorized code or document change to a known file.","send_email":"Send an already approved message to a verified recipient.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","upload_artifact":"Upload a completed artifact to the designated storage location.","calendar":"Read availability and schedule an approved meeting.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","parse_document":"Extract text and structure from a supplied PDF or document.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","fetch_web":"Open a current public web page or official documentation.","deploy_site":"Publish a tested website build to its authorized hosting target.","inspect_logs":"Read service logs and stack traces for a runtime failure.","calculate":"Compute a numeric result from supplied data or a formula.","run_tests":"Run tests or checks to verify a code change."}}},"clm":{"predicted":"run_tests","correct":true,"score":0.862142,"confidence":0.862142,"probabilities":{"read_file":0.0002270166991023635,"query_database":0.00011741494415162289,"edit_file":0.006714871213269238,"send_email":0.0021955809706646007,"create_issue":0.012999910582885233,"upload_artifact":0.005610163875180913,"calendar":9.36919156209269e-05,"inspect_diff":0.0009109403950221059,"parse_document":5.581905250818543e-05,"repo_search":0.044868793373912606,"fetch_web":0.0005298884314364527,"deploy_site":0.0013041179225502808,"inspect_logs":0.04911714080024819,"calculate":0.013113020353217536,"run_tests":0.8621416294702298},"latency_ms":36.41,"input_tokens":26,"retries":0},"jev":{"predicted":"run_tests","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"read_file":0.0,"query_database":0.0,"repo_search":0.0,"inspect_diff":0.0,"calculate":0.0,"edit_file":0.0,"parse_document":0.0,"create_issue":0.0,"deploy_site":0.0,"send_email":0.0,"fetch_web":0.0,"upload_artifact":0.0,"calendar":0.0,"run_tests":1.0,"inspect_logs":0.0},"latency_ms":112.76,"input_tokens":652,"retries":0}},{"id":"tool-03-1","kind":"tool_routing","group":null,"file":null,"expected":"run_tests","state":"Current task: A fix has been applied. Check whether the relevant test suite passes.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"run_tests":"Run tests or checks to verify a code change.","send_email":"Send an already approved message to a verified recipient.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","deploy_site":"Publish a tested website build to its authorized hosting target.","edit_file":"Apply an authorized code or document change to a known file.","read_file":"Read the full contents of a known file or a specific line range.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","parse_document":"Extract text and structure from a supplied PDF or document.","calculate":"Compute a numeric result from supplied data or a formula.","query_database":"Read structured records from the authorized application database.","calendar":"Read availability and schedule an approved meeting.","upload_artifact":"Upload a completed artifact to the designated storage location.","fetch_web":"Open a current public web page or official documentation.","inspect_logs":"Read service logs and stack traces for a runtime failure."}}},"clm":{"predicted":"run_tests","correct":true,"score":0.88567,"confidence":0.88567,"probabilities":{"run_tests":0.8856703190723715,"send_email":0.001742859205538635,"create_issue":0.04650801473798751,"inspect_diff":0.009034098933927046,"deploy_site":0.00026704884658552595,"edit_file":0.014408170392771697,"read_file":0.00019668613114231801,"repo_search":0.0019519990719516044,"parse_document":2.938169106467761e-05,"calculate":0.004855725316843361,"query_database":0.00018559461049525417,"calendar":5.355635720504811e-05,"upload_artifact":0.004022943314831074,"fetch_web":0.00010977633747476793,"inspect_logs":0.03096382597980999},"latency_ms":34.97,"input_tokens":36,"retries":0},"jev":{"predicted":"run_tests","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"parse_document":0.0,"calculate":0.0,"inspect_diff":0.0,"run_tests":1.0,"read_file":0.0,"fetch_web":0.0,"create_issue":0.0,"query_database":0.0,"inspect_logs":0.0,"deploy_site":0.0,"repo_search":0.0,"send_email":0.0,"calendar":0.0,"edit_file":0.0,"upload_artifact":0.0},"latency_ms":96.61,"input_tokens":663,"retries":0}},{"id":"tool-04-0","kind":"tool_routing","group":null,"file":null,"expected":"inspect_diff","state":"Agent goal: Before review, identify exactly which lines changed in the working tree.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"fetch_web":"Open a current public web page or official documentation.","upload_artifact":"Upload a completed artifact to the designated storage location.","calculate":"Compute a numeric result from supplied data or a formula.","deploy_site":"Publish a tested website build to its authorized hosting target.","inspect_logs":"Read service logs and stack traces for a runtime failure.","query_database":"Read structured records from the authorized application database.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","edit_file":"Apply an authorized code or document change to a known file.","parse_document":"Extract text and structure from a supplied PDF or document.","read_file":"Read the full contents of a known file or a specific line range.","calendar":"Read availability and schedule an approved meeting.","send_email":"Send an already approved message to a verified recipient.","run_tests":"Run tests or checks to verify a code change.","create_issue":"Create a tracked issue with a title, description, and reproduction steps."}}},"clm":{"predicted":"inspect_diff","correct":true,"score":0.30018,"confidence":0.30018,"probabilities":{"fetch_web":0.00012625436091167036,"upload_artifact":0.05479748913240712,"calculate":0.0012879983359285858,"deploy_site":0.0009810019153852116,"inspect_logs":0.0009338583460773436,"query_database":0.0002045893254498951,"inspect_diff":0.30017972351517,"repo_search":0.2561180494873017,"edit_file":0.06160647781216695,"parse_document":0.002162812416222452,"read_file":0.23133262824653839,"calendar":2.531710895021921e-05,"send_email":0.0018279630340231156,"run_tests":0.08794810668107038,"create_issue":0.0004677302823970087},"latency_ms":39.97,"input_tokens":25,"retries":0},"jev":{"predicted":"inspect_diff","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"deploy_site":0.0,"edit_file":0.0,"calculate":0.0,"repo_search":0.0,"parse_document":0.0,"fetch_web":0.0,"send_email":0.0,"run_tests":0.0,"query_database":0.0,"inspect_diff":1.0,"calendar":0.0,"upload_artifact":0.0,"read_file":0.0,"inspect_logs":0.0,"create_issue":0.0},"latency_ms":111.92,"input_tokens":651,"retries":0}},{"id":"tool-04-1","kind":"tool_routing","group":null,"file":null,"expected":"inspect_diff","state":"Current task: Before review, identify exactly which lines changed in the working tree.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"fetch_web":"Open a current public web page or official documentation.","deploy_site":"Publish a tested website build to its authorized hosting target.","edit_file":"Apply an authorized code or document change to a known file.","query_database":"Read structured records from the authorized application database.","calendar":"Read availability and schedule an approved meeting.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","inspect_logs":"Read service logs and stack traces for a runtime failure.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","run_tests":"Run tests or checks to verify a code change.","calculate":"Compute a numeric result from supplied data or a formula.","parse_document":"Extract text and structure from a supplied PDF or document.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","send_email":"Send an already approved message to a verified recipient.","read_file":"Read the full contents of a known file or a specific line range.","upload_artifact":"Upload a completed artifact to the designated storage location."}}},"clm":{"predicted":"edit_file","correct":false,"score":0.138409,"confidence":0.558507,"probabilities":{"fetch_web":2.7342702359565443e-05,"deploy_site":0.0004618276831709064,"edit_file":0.5585065694462591,"query_database":0.0008218471683770293,"calendar":0.00014178722342477742,"create_issue":0.0008751338036640843,"inspect_logs":0.002769212028806808,"inspect_diff":0.1384091875378811,"run_tests":0.127539895673724,"calculate":0.000998327245218637,"parse_document":0.0015257147062072956,"repo_search":0.003801407960988707,"send_email":0.005960423215765027,"read_file":0.04682034299983937,"upload_artifact":0.1113409806043136},"latency_ms":45.28,"input_tokens":35,"retries":0},"jev":{"predicted":"inspect_diff","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"edit_file":0.0,"repo_search":0.0,"upload_artifact":0.0,"parse_document":0.0,"inspect_diff":1.0,"query_database":0.0,"calculate":0.0,"calendar":0.0,"read_file":0.0,"inspect_logs":0.0,"run_tests":0.0,"send_email":0.0,"fetch_web":0.0,"create_issue":0.0,"deploy_site":0.0},"latency_ms":99.77,"input_tokens":662,"retries":0}},{"id":"tool-05-0","kind":"tool_routing","group":null,"file":null,"expected":"fetch_web","state":"Agent goal: Confirm the current Cloudflare documentation for custom Worker domains.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"upload_artifact":"Upload a completed artifact to the designated storage location.","edit_file":"Apply an authorized code or document change to a known file.","deploy_site":"Publish a tested website build to its authorized hosting target.","calculate":"Compute a numeric result from supplied data or a formula.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","run_tests":"Run tests or checks to verify a code change.","calendar":"Read availability and schedule an approved meeting.","send_email":"Send an already approved message to a verified recipient.","read_file":"Read the full contents of a known file or a specific line range.","query_database":"Read structured records from the authorized application database.","inspect_logs":"Read service logs and stack traces for a runtime failure.","fetch_web":"Open a current public web page or official documentation.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","parse_document":"Extract text and structure from a supplied PDF or document."}}},"clm":{"predicted":"send_email","correct":false,"score":0.048726,"confidence":0.287118,"probabilities":{"upload_artifact":0.11529911068558898,"edit_file":0.06821988486718433,"deploy_site":0.11198964458108758,"calculate":0.10931718954200868,"create_issue":0.0225241375566969,"run_tests":0.03200525640030154,"calendar":0.05831028325566499,"send_email":0.2871178522718431,"read_file":0.004634626972395603,"query_database":0.009190916625910988,"inspect_logs":0.008724702820532197,"fetch_web":0.04872629171402315,"repo_search":0.08818043443956901,"inspect_diff":0.00023469445804330597,"parse_document":0.03552497380914962},"latency_ms":35.09,"input_tokens":23,"retries":0},"jev":{"predicted":"fetch_web","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"edit_file":0.0,"inspect_diff":0.0,"calculate":0.0,"query_database":0.0,"repo_search":0.0,"read_file":0.0,"inspect_logs":0.0,"send_email":0.0,"deploy_site":0.0,"fetch_web":1.0,"calendar":0.0,"create_issue":0.0,"upload_artifact":0.0,"run_tests":0.0,"parse_document":0.0},"latency_ms":106.97,"input_tokens":651,"retries":0}},{"id":"tool-05-1","kind":"tool_routing","group":null,"file":null,"expected":"fetch_web","state":"Current task: Confirm the current Cloudflare documentation for custom Worker domains.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"upload_artifact":"Upload a completed artifact to the designated storage location.","send_email":"Send an already approved message to a verified recipient.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","read_file":"Read the full contents of a known file or a specific line range.","parse_document":"Extract text and structure from a supplied PDF or document.","calculate":"Compute a numeric result from supplied data or a formula.","deploy_site":"Publish a tested website build to its authorized hosting target.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","run_tests":"Run tests or checks to verify a code change.","fetch_web":"Open a current public web page or official documentation.","query_database":"Read structured records from the authorized application database.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","inspect_logs":"Read service logs and stack traces for a runtime failure.","calendar":"Read availability and schedule an approved meeting.","edit_file":"Apply an authorized code or document change to a known file."}}},"clm":{"predicted":"upload_artifact","correct":false,"score":0.010085,"confidence":0.238075,"probabilities":{"upload_artifact":0.23807452849311975,"send_email":0.14910945482069968,"create_issue":0.018857030827909002,"read_file":0.031126953341248494,"parse_document":0.10696549771765054,"calculate":0.12069564150007617,"deploy_site":0.025494978735350424,"inspect_diff":0.0004117202690998779,"run_tests":0.023924839285770425,"fetch_web":0.010085435723081305,"query_database":0.020209275080525484,"repo_search":0.007817166092448629,"inspect_logs":0.024057679342857772,"calendar":0.06446389303920927,"edit_file":0.15870590573095328},"latency_ms":36.52,"input_tokens":33,"retries":0},"jev":{"predicted":"fetch_web","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"inspect_diff":0.0,"repo_search":0.0,"parse_document":0.0,"read_file":0.0,"query_database":0.0,"fetch_web":1.0,"inspect_logs":0.0,"send_email":0.0,"calculate":0.0,"calendar":0.0,"deploy_site":0.0,"run_tests":0.0,"create_issue":0.0,"upload_artifact":0.0,"edit_file":0.0},"latency_ms":117.61,"input_tokens":662,"retries":0}},{"id":"tool-06-0","kind":"tool_routing","group":null,"file":null,"expected":"query_database","state":"Agent goal: Look up invoice 8842 in the authorized billing database.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"fetch_web":"Open a current public web page or official documentation.","deploy_site":"Publish a tested website build to its authorized hosting target.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","send_email":"Send an already approved message to a verified recipient.","parse_document":"Extract text and structure from a supplied PDF or document.","run_tests":"Run tests or checks to verify a code change.","edit_file":"Apply an authorized code or document change to a known file.","calculate":"Compute a numeric result from supplied data or a formula.","query_database":"Read structured records from the authorized application database.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","read_file":"Read the full contents of a known file or a specific line range.","upload_artifact":"Upload a completed artifact to the designated storage location.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","calendar":"Read availability and schedule an approved meeting.","inspect_logs":"Read service logs and stack traces for a runtime failure."}}},"clm":{"predicted":"send_email","correct":false,"score":0.08598,"confidence":0.416834,"probabilities":{"fetch_web":0.0038879178040713324,"deploy_site":0.008973627428788277,"repo_search":0.014897659629181853,"send_email":0.41683396193521605,"parse_document":0.0009413967111528976,"run_tests":0.002000424091478326,"edit_file":0.09880300637289131,"calculate":0.19061967433747054,"query_database":0.08598033071326096,"inspect_diff":0.00016570620161187062,"read_file":5.898600907410626e-05,"upload_artifact":0.06590329113928825,"create_issue":0.03318999890893821,"calendar":0.028441753460839776,"inspect_logs":0.04930226525673629},"latency_ms":34.72,"input_tokens":26,"retries":0},"jev":{"predicted":"query_database","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"create_issue":0.0,"calendar":0.0,"repo_search":0.0,"run_tests":0.0,"calculate":0.0,"inspect_logs":0.0,"read_file":0.0,"send_email":0.0,"inspect_diff":0.0,"upload_artifact":0.0,"fetch_web":0.0,"query_database":1.0,"deploy_site":0.0,"parse_document":0.0,"edit_file":0.0},"latency_ms":149.63,"input_tokens":652,"retries":0}},{"id":"tool-06-1","kind":"tool_routing","group":null,"file":null,"expected":"query_database","state":"Current task: Look up invoice 8842 in the authorized billing database.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"upload_artifact":"Upload a completed artifact to the designated storage location.","inspect_logs":"Read service logs and stack traces for a runtime failure.","fetch_web":"Open a current public web page or official documentation.","query_database":"Read structured records from the authorized application database.","parse_document":"Extract text and structure from a supplied PDF or document.","calendar":"Read availability and schedule an approved meeting.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","read_file":"Read the full contents of a known file or a specific line range.","run_tests":"Run tests or checks to verify a code change.","deploy_site":"Publish a tested website build to its authorized hosting target.","send_email":"Send an already approved message to a verified recipient.","edit_file":"Apply an authorized code or document change to a known file.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","calculate":"Compute a numeric result from supplied data or a formula.","create_issue":"Create a tracked issue with a title, description, and reproduction steps."}}},"clm":{"predicted":"calculate","correct":false,"score":0.054294,"confidence":0.700612,"probabilities":{"upload_artifact":0.005691809945751757,"inspect_logs":0.032073259945155454,"fetch_web":0.0005783436240117873,"query_database":0.05429425700647545,"parse_document":0.007240027908801751,"calendar":0.020258352331957434,"inspect_diff":0.0001437723206018147,"read_file":0.00013674335110356518,"run_tests":0.0006325517810756242,"deploy_site":0.0004303060648534294,"send_email":0.1364949660441122,"edit_file":0.04032329295476709,"repo_search":0.00014088344780366287,"calculate":0.7006115725633346,"create_issue":0.0009498607101942932},"latency_ms":37.91,"input_tokens":36,"retries":0},"jev":{"predicted":"query_database","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"read_file":0.0,"query_database":1.0,"repo_search":0.0,"inspect_diff":0.0,"calculate":0.0,"edit_file":0.0,"create_issue":0.0,"parse_document":0.0,"deploy_site":0.0,"send_email":0.0,"fetch_web":0.0,"upload_artifact":0.0,"calendar":0.0,"run_tests":0.0,"inspect_logs":0.0},"latency_ms":144.08,"input_tokens":663,"retries":0}},{"id":"tool-07-0","kind":"tool_routing","group":null,"file":null,"expected":"inspect_logs","state":"Agent goal: Checkout began returning 500 errors after deployment. Find the stack trace.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"upload_artifact":"Upload a completed artifact to the designated storage location.","deploy_site":"Publish a tested website build to its authorized hosting target.","calculate":"Compute a numeric result from supplied data or a formula.","read_file":"Read the full contents of a known file or a specific line range.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","parse_document":"Extract text and structure from a supplied PDF or document.","inspect_logs":"Read service logs and stack traces for a runtime failure.","send_email":"Send an already approved message to a verified recipient.","fetch_web":"Open a current public web page or official documentation.","calendar":"Read availability and schedule an approved meeting.","run_tests":"Run tests or checks to verify a code change.","edit_file":"Apply an authorized code or document change to a known file.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","query_database":"Read structured records from the authorized application database."}}},"clm":{"predicted":"inspect_logs","correct":true,"score":0.883659,"confidence":0.883659,"probabilities":{"upload_artifact":0.005898859715638507,"deploy_site":0.00038985715787874894,"calculate":8.393801908719055e-05,"read_file":3.0198125279278123e-05,"create_issue":0.012822342969197877,"inspect_diff":0.0014716984765302142,"parse_document":7.920745827340208e-05,"inspect_logs":0.8836585565009376,"send_email":0.00023995416385636016,"fetch_web":0.0009921079519118825,"calendar":6.957405593607238e-05,"run_tests":0.0766417231268399,"edit_file":0.000852221743592061,"repo_search":0.016346062542702676,"query_database":0.00042369799233822223},"latency_ms":35.29,"input_tokens":28,"retries":0},"jev":{"predicted":"inspect_logs","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"create_issue":0.0,"upload_artifact":0.0,"inspect_diff":0.0,"run_tests":0.0,"parse_document":0.0,"edit_file":0.0,"calendar":0.0,"read_file":0.0,"deploy_site":0.0,"calculate":0.0,"query_database":0.0,"fetch_web":0.0,"inspect_logs":1.0,"send_email":0.0,"repo_search":0.0},"latency_ms":124.59,"input_tokens":654,"retries":0}},{"id":"tool-07-1","kind":"tool_routing","group":null,"file":null,"expected":"inspect_logs","state":"Current task: Checkout began returning 500 errors after deployment. Find the stack trace.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","parse_document":"Extract text and structure from a supplied PDF or document.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","query_database":"Read structured records from the authorized application database.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","send_email":"Send an already approved message to a verified recipient.","edit_file":"Apply an authorized code or document change to a known file.","inspect_logs":"Read service logs and stack traces for a runtime failure.","calculate":"Compute a numeric result from supplied data or a formula.","deploy_site":"Publish a tested website build to its authorized hosting target.","upload_artifact":"Upload a completed artifact to the designated storage location.","calendar":"Read availability and schedule an approved meeting.","fetch_web":"Open a current public web page or official documentation.","read_file":"Read the full contents of a known file or a specific line range.","run_tests":"Run tests or checks to verify a code change."}}},"clm":{"predicted":"inspect_logs","correct":true,"score":0.853415,"confidence":0.853415,"probabilities":{"repo_search":0.009863008128469178,"parse_document":0.0025646587178850045,"inspect_diff":0.004140079049123957,"query_database":0.001211687775915279,"create_issue":0.019092990848914856,"send_email":0.005402235795621938,"edit_file":0.014830350700095214,"inspect_logs":0.853415170302199,"calculate":0.0011099704875505189,"deploy_site":0.0010136492962888805,"upload_artifact":0.01517764143785095,"calendar":0.0018573870573723266,"fetch_web":0.0007423828787106224,"read_file":0.0005492310966037411,"run_tests":0.06902955642739853},"latency_ms":34.79,"input_tokens":38,"retries":0},"jev":{"predicted":"inspect_logs","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"edit_file":0.0,"repo_search":0.0,"upload_artifact":0.0,"parse_document":0.0,"inspect_diff":0.0,"calculate":0.0,"query_database":0.0,"calendar":0.0,"read_file":0.0,"inspect_logs":1.0,"run_tests":0.0,"send_email":0.0,"fetch_web":0.0,"create_issue":0.0,"deploy_site":0.0},"latency_ms":129.36,"input_tokens":665,"retries":0}},{"id":"tool-08-0","kind":"tool_routing","group":null,"file":null,"expected":"parse_document","state":"Agent goal: The user supplied a 24-page PDF contract. Extract its termination clause.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","upload_artifact":"Upload a completed artifact to the designated storage location.","edit_file":"Apply an authorized code or document change to a known file.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","fetch_web":"Open a current public web page or official documentation.","inspect_logs":"Read service logs and stack traces for a runtime failure.","query_database":"Read structured records from the authorized application database.","send_email":"Send an already approved message to a verified recipient.","calendar":"Read availability and schedule an approved meeting.","read_file":"Read the full contents of a known file or a specific line range.","parse_document":"Extract text and structure from a supplied PDF or document.","calculate":"Compute a numeric result from supplied data or a formula.","deploy_site":"Publish a tested website build to its authorized hosting target.","run_tests":"Run tests or checks to verify a code change."}}},"clm":{"predicted":"parse_document","correct":true,"score":0.889632,"confidence":0.889632,"probabilities":{"inspect_diff":8.342281807807498e-05,"create_issue":0.0009829671425035326,"upload_artifact":0.02295082976108085,"edit_file":0.033162538444504744,"repo_search":0.0009614667532781013,"fetch_web":0.0005171274658718729,"inspect_logs":0.0019784011678859936,"query_database":0.007101893117125887,"send_email":0.005901527632951471,"calendar":0.002431544909369622,"read_file":0.004324078137086708,"parse_document":0.8896317632999223,"calculate":0.02939506649824372,"deploy_site":0.0004016806026633131,"run_tests":0.00017569224943387543},"latency_ms":34.7,"input_tokens":28,"retries":0},"jev":{"predicted":"parse_document","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"edit_file":0.0,"inspect_diff":0.0,"calculate":0.0,"query_database":0.0,"repo_search":0.0,"read_file":0.0,"inspect_logs":0.0,"send_email":0.0,"deploy_site":0.0,"fetch_web":0.0,"calendar":0.0,"create_issue":0.0,"upload_artifact":0.0,"run_tests":0.0,"parse_document":1.0},"latency_ms":94.96,"input_tokens":654,"retries":0}},{"id":"tool-08-1","kind":"tool_routing","group":null,"file":null,"expected":"parse_document","state":"Current task: The user supplied a 24-page PDF contract. Extract its termination clause.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"parse_document":"Extract text and structure from a supplied PDF or document.","send_email":"Send an already approved message to a verified recipient.","run_tests":"Run tests or checks to verify a code change.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","read_file":"Read the full contents of a known file or a specific line range.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","calendar":"Read availability and schedule an approved meeting.","deploy_site":"Publish a tested website build to its authorized hosting target.","query_database":"Read structured records from the authorized application database.","inspect_logs":"Read service logs and stack traces for a runtime failure.","fetch_web":"Open a current public web page or official documentation.","edit_file":"Apply an authorized code or document change to a known file.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","upload_artifact":"Upload a completed artifact to the designated storage location.","calculate":"Compute a numeric result from supplied data or a formula."}}},"clm":{"predicted":"parse_document","correct":true,"score":0.846076,"confidence":0.846076,"probabilities":{"parse_document":0.846075808358053,"send_email":0.001296021351649855,"run_tests":0.00017186161524709563,"create_issue":0.0005813715186518817,"read_file":0.044489218215130945,"inspect_diff":0.0005114991230791847,"calendar":0.001449682201336419,"deploy_site":0.0001250810025540217,"query_database":0.008447913691372889,"inspect_logs":0.0018843840116677653,"fetch_web":0.00024698134443791614,"edit_file":0.03996737179396383,"repo_search":0.0004281731083791541,"upload_artifact":0.0420092185896855,"calculate":0.012315414074790504},"latency_ms":52.75,"input_tokens":38,"retries":0},"jev":{"predicted":"parse_document","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"send_email":0.0,"parse_document":1.0,"calendar":0.0,"calculate":0.0,"repo_search":0.0,"run_tests":0.0,"query_database":0.0,"inspect_logs":0.0,"read_file":0.0,"deploy_site":0.0,"upload_artifact":0.0,"create_issue":0.0,"edit_file":0.0,"fetch_web":0.0,"inspect_diff":0.0},"latency_ms":116.36,"input_tokens":665,"retries":0}},{"id":"tool-09-0","kind":"tool_routing","group":null,"file":null,"expected":"create_issue","state":"Agent goal: The bug is reproduced and the user asked to file a tracker ticket with steps.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"upload_artifact":"Upload a completed artifact to the designated storage location.","deploy_site":"Publish a tested website build to its authorized hosting target.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","calculate":"Compute a numeric result from supplied data or a formula.","fetch_web":"Open a current public web page or official documentation.","run_tests":"Run tests or checks to verify a code change.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","parse_document":"Extract text and structure from a supplied PDF or document.","inspect_logs":"Read service logs and stack traces for a runtime failure.","send_email":"Send an already approved message to a verified recipient.","edit_file":"Apply an authorized code or document change to a known file.","query_database":"Read structured records from the authorized application database.","calendar":"Read availability and schedule an approved meeting.","read_file":"Read the full contents of a known file or a specific line range.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown."}}},"clm":{"predicted":"run_tests","correct":false,"score":0.262893,"confidence":0.375092,"probabilities":{"upload_artifact":0.019548130916662364,"deploy_site":0.00183095149991794,"create_issue":0.26289299675830285,"calculate":0.004482964120713241,"fetch_web":0.0014583258287736165,"run_tests":0.375092241830864,"inspect_diff":0.0006901808192248418,"parse_document":0.00031445885237800714,"inspect_logs":0.2228043731057222,"send_email":0.03644486849687878,"edit_file":0.028700780680072615,"query_database":0.0009795406147322296,"calendar":0.0026054605219028876,"read_file":0.00023453348719769808,"repo_search":0.04192019246665684},"latency_ms":33.73,"input_tokens":28,"retries":0},"jev":{"predicted":"create_issue","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"calculate":0.0,"calendar":0.0,"query_database":0.0,"upload_artifact":0.0,"create_issue":1.0,"inspect_diff":0.0,"send_email":0.0,"edit_file":0.0,"run_tests":0.0,"deploy_site":0.0,"fetch_web":0.0,"parse_document":0.0,"read_file":0.0,"repo_search":0.0,"inspect_logs":0.0},"latency_ms":97.64,"input_tokens":654,"retries":0}},{"id":"tool-09-1","kind":"tool_routing","group":null,"file":null,"expected":"create_issue","state":"Current task: The bug is reproduced and the user asked to file a tracker ticket with steps.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"deploy_site":"Publish a tested website build to its authorized hosting target.","run_tests":"Run tests or checks to verify a code change.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","calculate":"Compute a numeric result from supplied data or a formula.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","calendar":"Read availability and schedule an approved meeting.","upload_artifact":"Upload a completed artifact to the designated storage location.","read_file":"Read the full contents of a known file or a specific line range.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","fetch_web":"Open a current public web page or official documentation.","query_database":"Read structured records from the authorized application database.","inspect_logs":"Read service logs and stack traces for a runtime failure.","send_email":"Send an already approved message to a verified recipient.","parse_document":"Extract text and structure from a supplied PDF or document.","edit_file":"Apply an authorized code or document change to a known file."}}},"clm":{"predicted":"create_issue","correct":true,"score":0.303781,"confidence":0.303781,"probabilities":{"deploy_site":0.0005296897138511232,"run_tests":0.23625962663301456,"repo_search":0.006372072191039778,"calculate":0.03492908126125465,"create_issue":0.303781416741992,"calendar":0.007728862937881656,"upload_artifact":0.06108302111653704,"read_file":0.0015148514185083302,"inspect_diff":0.018049357713896087,"fetch_web":0.000371737626813083,"query_database":0.002992848362940832,"inspect_logs":0.17415210790001703,"send_email":0.03018751917516158,"parse_document":0.0036175497944223238,"edit_file":0.11843025741266998},"latency_ms":36.34,"input_tokens":38,"retries":0},"jev":{"predicted":"create_issue","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"repo_search":0.0,"calendar":0.0,"create_issue":1.0,"run_tests":0.0,"calculate":0.0,"inspect_logs":0.0,"read_file":0.0,"send_email":0.0,"inspect_diff":0.0,"upload_artifact":0.0,"fetch_web":0.0,"query_database":0.0,"parse_document":0.0,"deploy_site":0.0,"edit_file":0.0},"latency_ms":100.57,"input_tokens":665,"retries":0}},{"id":"tool-10-0","kind":"tool_routing","group":null,"file":null,"expected":"send_email","state":"Agent goal: The customer notice is approved and the recipient address was verified. Send it.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"create_issue":"Create a tracked issue with a title, description, and reproduction steps.","edit_file":"Apply an authorized code or document change to a known file.","upload_artifact":"Upload a completed artifact to the designated storage location.","run_tests":"Run tests or checks to verify a code change.","send_email":"Send an already approved message to a verified recipient.","inspect_logs":"Read service logs and stack traces for a runtime failure.","query_database":"Read structured records from the authorized application database.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","calendar":"Read availability and schedule an approved meeting.","read_file":"Read the full contents of a known file or a specific line range.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","deploy_site":"Publish a tested website build to its authorized hosting target.","parse_document":"Extract text and structure from a supplied PDF or document.","calculate":"Compute a numeric result from supplied data or a formula.","fetch_web":"Open a current public web page or official documentation."}}},"clm":{"predicted":"send_email","correct":true,"score":0.865552,"confidence":0.865552,"probabilities":{"create_issue":0.005364374817447919,"edit_file":0.010689699862566886,"upload_artifact":0.010200533067570005,"run_tests":0.0005112640754584781,"send_email":0.8655520279181111,"inspect_logs":0.001541413712131856,"query_database":0.0015755853753249356,"inspect_diff":2.1420734826707603e-06,"calendar":0.08429521073642542,"read_file":1.3405603904777756e-05,"repo_search":0.0006840177581056922,"deploy_site":0.00022829105254632345,"parse_document":0.0004785754001010114,"calculate":0.018674360990249803,"fetch_web":0.00018909755657311623},"latency_ms":32.64,"input_tokens":27,"retries":0},"jev":{"predicted":"send_email","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"edit_file":0.0,"inspect_diff":0.0,"calculate":0.0,"query_database":0.0,"repo_search":0.0,"read_file":0.0,"inspect_logs":0.0,"send_email":1.0,"deploy_site":0.0,"fetch_web":0.0,"create_issue":0.0,"calendar":0.0,"upload_artifact":0.0,"run_tests":0.0,"parse_document":0.0},"latency_ms":158.92,"input_tokens":653,"retries":0}},{"id":"tool-10-1","kind":"tool_routing","group":null,"file":null,"expected":"send_email","state":"Current task: The customer notice is approved and the recipient address was verified. Send it.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"calculate":"Compute a numeric result from supplied data or a formula.","send_email":"Send an already approved message to a verified recipient.","query_database":"Read structured records from the authorized application database.","deploy_site":"Publish a tested website build to its authorized hosting target.","parse_document":"Extract text and structure from a supplied PDF or document.","calendar":"Read availability and schedule an approved meeting.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","upload_artifact":"Upload a completed artifact to the designated storage location.","edit_file":"Apply an authorized code or document change to a known file.","fetch_web":"Open a current public web page or official documentation.","run_tests":"Run tests or checks to verify a code change.","read_file":"Read the full contents of a known file or a specific line range.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","inspect_logs":"Read service logs and stack traces for a runtime failure."}}},"clm":{"predicted":"send_email","correct":true,"score":0.698439,"confidence":0.698439,"probabilities":{"calculate":0.07135809155341616,"send_email":0.6984392552988905,"query_database":0.0026477446734926853,"deploy_site":0.0009413062551180818,"parse_document":0.00293166954729846,"calendar":0.1633945164246147,"create_issue":0.004515267658322457,"repo_search":0.00020279550857594283,"upload_artifact":0.012689276129643974,"edit_file":0.026276773159260854,"fetch_web":9.692836374004993e-05,"run_tests":0.0022569438786185503,"read_file":7.872348218482488e-05,"inspect_diff":1.2745434518687298e-05,"inspect_logs":0.01415796263230411},"latency_ms":33.96,"input_tokens":37,"retries":0},"jev":{"predicted":"send_email","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"create_issue":0.0,"parse_document":0.0,"fetch_web":0.0,"inspect_diff":0.0,"repo_search":0.0,"upload_artifact":0.0,"edit_file":0.0,"run_tests":0.0,"calendar":0.0,"deploy_site":0.0,"read_file":0.0,"send_email":1.0,"calculate":0.0,"inspect_logs":0.0,"query_database":0.0},"latency_ms":102.8,"input_tokens":664,"retries":0}},{"id":"tool-11-0","kind":"tool_routing","group":null,"file":null,"expected":"calendar","state":"Agent goal: Arrange an approved 30-minute meeting with the product team next week.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"edit_file":"Apply an authorized code or document change to a known file.","parse_document":"Extract text and structure from a supplied PDF or document.","read_file":"Read the full contents of a known file or a specific line range.","calculate":"Compute a numeric result from supplied data or a formula.","run_tests":"Run tests or checks to verify a code change.","send_email":"Send an already approved message to a verified recipient.","inspect_logs":"Read service logs and stack traces for a runtime failure.","fetch_web":"Open a current public web page or official documentation.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","calendar":"Read availability and schedule an approved meeting.","deploy_site":"Publish a tested website build to its authorized hosting target.","upload_artifact":"Upload a completed artifact to the designated storage location.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","query_database":"Read structured records from the authorized application database."}}},"clm":{"predicted":"calendar","correct":true,"score":0.917976,"confidence":0.917976,"probabilities":{"edit_file":0.00023697270532191004,"parse_document":0.00014007267965093618,"read_file":1.0172341837651154e-05,"calculate":0.004113701757449024,"run_tests":0.0011376631716916686,"send_email":0.042113619477394636,"inspect_logs":0.0021269269968170676,"fetch_web":0.0034897287503208854,"inspect_diff":1.747908708275028e-06,"create_issue":0.01622547331423382,"calendar":0.9179763724468603,"deploy_site":0.004543233291733458,"upload_artifact":0.0006806409431188548,"repo_search":0.007097556830494532,"query_database":0.0001061173843670215},"latency_ms":37.39,"input_tokens":27,"retries":0},"jev":{"predicted":"calendar","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"fetch_web":0.0,"deploy_site":0.0,"read_file":0.0,"repo_search":0.0,"calculate":0.0,"parse_document":0.0,"send_email":0.0,"calendar":1.0,"create_issue":0.0,"edit_file":0.0,"query_database":0.0,"inspect_logs":0.0,"run_tests":0.0,"inspect_diff":0.0,"upload_artifact":0.0},"latency_ms":138.59,"input_tokens":653,"retries":0}},{"id":"tool-11-1","kind":"tool_routing","group":null,"file":null,"expected":"calendar","state":"Current task: Arrange an approved 30-minute meeting with the product team next week.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","edit_file":"Apply an authorized code or document change to a known file.","inspect_logs":"Read service logs and stack traces for a runtime failure.","parse_document":"Extract text and structure from a supplied PDF or document.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","run_tests":"Run tests or checks to verify a code change.","read_file":"Read the full contents of a known file or a specific line range.","calculate":"Compute a numeric result from supplied data or a formula.","query_database":"Read structured records from the authorized application database.","deploy_site":"Publish a tested website build to its authorized hosting target.","fetch_web":"Open a current public web page or official documentation.","upload_artifact":"Upload a completed artifact to the designated storage location.","send_email":"Send an already approved message to a verified recipient.","calendar":"Read availability and schedule an approved meeting.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown."}}},"clm":{"predicted":"calendar","correct":true,"score":0.976931,"confidence":0.976931,"probabilities":{"inspect_diff":2.9294057618693805e-07,"edit_file":8.441964715085472e-05,"inspect_logs":0.0002090634143818643,"parse_document":0.00038297182233290166,"create_issue":0.00011716618152904286,"run_tests":3.629659749809562e-05,"read_file":1.0236734331170507e-05,"calculate":0.005855664630869272,"query_database":4.556670224479243e-05,"deploy_site":8.093094477854189e-05,"fetch_web":0.0001277805963648055,"upload_artifact":0.00010019231729186964,"send_email":0.015942930358320357,"calendar":0.9769307874117219,"repo_search":7.569970060825705e-05},"latency_ms":36.14,"input_tokens":37,"retries":0},"jev":{"predicted":"calendar","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"calendar":1.0,"parse_document":0.0,"calculate":0.0,"inspect_logs":0.0,"deploy_site":0.0,"read_file":0.0,"query_database":0.0,"run_tests":0.0,"create_issue":0.0,"edit_file":0.0,"send_email":0.0,"upload_artifact":0.0,"inspect_diff":0.0,"fetch_web":0.0,"repo_search":0.0},"latency_ms":136.17,"input_tokens":664,"retries":0}},{"id":"tool-12-0","kind":"tool_routing","group":null,"file":null,"expected":"upload_artifact","state":"Agent goal: The final report is complete; place it in the designated artifacts bucket.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","calendar":"Read availability and schedule an approved meeting.","query_database":"Read structured records from the authorized application database.","inspect_logs":"Read service logs and stack traces for a runtime failure.","deploy_site":"Publish a tested website build to its authorized hosting target.","upload_artifact":"Upload a completed artifact to the designated storage location.","read_file":"Read the full contents of a known file or a specific line range.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","edit_file":"Apply an authorized code or document change to a known file.","fetch_web":"Open a current public web page or official documentation.","parse_document":"Extract text and structure from a supplied PDF or document.","run_tests":"Run tests or checks to verify a code change.","send_email":"Send an already approved message to a verified recipient.","calculate":"Compute a numeric result from supplied data or a formula.","create_issue":"Create a tracked issue with a title, description, and reproduction steps."}}},"clm":{"predicted":"upload_artifact","correct":true,"score":0.726568,"confidence":0.726568,"probabilities":{"inspect_diff":6.403520806973854e-05,"calendar":0.0015831260369206564,"query_database":0.0029921162420402437,"inspect_logs":0.010140247173288278,"deploy_site":0.005491327825104178,"upload_artifact":0.7265682646231295,"read_file":0.0003501527614640565,"repo_search":0.09571581238143047,"edit_file":0.026644392997970417,"fetch_web":0.0005056906602057372,"parse_document":0.0013364787742893365,"run_tests":0.03351415627945779,"send_email":0.015510916027932148,"calculate":0.0025502924462080983,"create_issue":0.07703299056248931},"latency_ms":33.1,"input_tokens":26,"retries":0},"jev":{"predicted":"upload_artifact","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"parse_document":0.0,"read_file":0.0,"inspect_diff":0.0,"run_tests":0.0,"calculate":0.0,"fetch_web":0.0,"create_issue":0.0,"query_database":0.0,"inspect_logs":0.0,"deploy_site":0.0,"repo_search":0.0,"send_email":0.0,"calendar":0.0,"edit_file":0.0,"upload_artifact":1.0},"latency_ms":108.29,"input_tokens":652,"retries":0}},{"id":"tool-12-1","kind":"tool_routing","group":null,"file":null,"expected":"upload_artifact","state":"Current task: The final report is complete; place it in the designated artifacts bucket.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","read_file":"Read the full contents of a known file or a specific line range.","calculate":"Compute a numeric result from supplied data or a formula.","query_database":"Read structured records from the authorized application database.","deploy_site":"Publish a tested website build to its authorized hosting target.","fetch_web":"Open a current public web page or official documentation.","parse_document":"Extract text and structure from a supplied PDF or document.","inspect_logs":"Read service logs and stack traces for a runtime failure.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","edit_file":"Apply an authorized code or document change to a known file.","calendar":"Read availability and schedule an approved meeting.","send_email":"Send an already approved message to a verified recipient.","upload_artifact":"Upload a completed artifact to the designated storage location.","run_tests":"Run tests or checks to verify a code change."}}},"clm":{"predicted":"upload_artifact","correct":true,"score":0.285478,"confidence":0.285478,"probabilities":{"repo_search":0.026134874769528298,"create_issue":0.1848588085716419,"read_file":0.004827729858704631,"calculate":0.08666190865165102,"query_database":0.007236493090942813,"deploy_site":0.0013499211852717762,"fetch_web":0.00026119997611343427,"parse_document":0.017273504370156536,"inspect_logs":0.11847854370630609,"inspect_diff":0.0015271744160295444,"edit_file":0.11644320181853647,"calendar":0.007866593902978641,"send_email":0.03306764876627022,"upload_artifact":0.2854778824688093,"run_tests":0.10853451444705925},"latency_ms":35.63,"input_tokens":36,"retries":0},"jev":{"predicted":"upload_artifact","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"send_email":0.0,"parse_document":0.0,"calendar":0.0,"calculate":0.0,"repo_search":0.0,"query_database":0.0,"inspect_logs":0.0,"run_tests":0.0,"read_file":0.0,"deploy_site":0.0,"upload_artifact":1.0,"create_issue":0.0,"fetch_web":0.0,"edit_file":0.0,"inspect_diff":0.0},"latency_ms":116.27,"input_tokens":663,"retries":0}},{"id":"tool-13-0","kind":"tool_routing","group":null,"file":null,"expected":"calculate","state":"Agent goal: Given latencies of 10, 20, 30, 40, and 100 ms, compute their arithmetic mean.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"parse_document":"Extract text and structure from a supplied PDF or document.","upload_artifact":"Upload a completed artifact to the designated storage location.","read_file":"Read the full contents of a known file or a specific line range.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","inspect_logs":"Read service logs and stack traces for a runtime failure.","fetch_web":"Open a current public web page or official documentation.","query_database":"Read structured records from the authorized application database.","calendar":"Read availability and schedule an approved meeting.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","deploy_site":"Publish a tested website build to its authorized hosting target.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","calculate":"Compute a numeric result from supplied data or a formula.","edit_file":"Apply an authorized code or document change to a known file.","run_tests":"Run tests or checks to verify a code change.","send_email":"Send an already approved message to a verified recipient."}}},"clm":{"predicted":"calculate","correct":true,"score":0.995759,"confidence":0.995759,"probabilities":{"parse_document":2.49539921160151e-05,"upload_artifact":0.0002653205631140748,"read_file":0.0003619543384450584,"create_issue":3.5285470134081586e-05,"inspect_logs":0.00036471387032224896,"fetch_web":6.719251925601597e-05,"query_database":0.000898573387512537,"calendar":0.0006822569262495737,"inspect_diff":1.965433856902148e-06,"deploy_site":4.187223149164437e-06,"repo_search":0.0007200999342594223,"calculate":0.9957591113158601,"edit_file":6.865591779441599e-05,"run_tests":0.00019900304576637233,"send_email":0.0005467260621639644},"latency_ms":33.63,"input_tokens":44,"retries":0},"jev":{"predicted":"calculate","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"create_issue":0.0,"run_tests":0.0,"deploy_site":0.0,"upload_artifact":0.0,"read_file":0.0,"calendar":0.0,"edit_file":0.0,"calculate":1.0,"inspect_diff":0.0,"fetch_web":0.0,"query_database":0.0,"parse_document":0.0,"repo_search":0.0,"send_email":0.0,"inspect_logs":0.0},"latency_ms":102.23,"input_tokens":671,"retries":0}},{"id":"tool-13-1","kind":"tool_routing","group":null,"file":null,"expected":"calculate","state":"Current task: Given latencies of 10, 20, 30, 40, and 100 ms, compute their arithmetic mean.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"upload_artifact":"Upload a completed artifact to the designated storage location.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","read_file":"Read the full contents of a known file or a specific line range.","deploy_site":"Publish a tested website build to its authorized hosting target.","inspect_logs":"Read service logs and stack traces for a runtime failure.","query_database":"Read structured records from the authorized application database.","edit_file":"Apply an authorized code or document change to a known file.","run_tests":"Run tests or checks to verify a code change.","calendar":"Read availability and schedule an approved meeting.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","fetch_web":"Open a current public web page or official documentation.","send_email":"Send an already approved message to a verified recipient.","calculate":"Compute a numeric result from supplied data or a formula.","parse_document":"Extract text and structure from a supplied PDF or document."}}},"clm":{"predicted":"calculate","correct":true,"score":0.983984,"confidence":0.983984,"probabilities":{"upload_artifact":0.000623268189908961,"create_issue":0.0001333609159470793,"inspect_diff":0.00013238645718584332,"read_file":0.003973291131868007,"deploy_site":3.272842209753467e-06,"inspect_logs":0.0018992886874867148,"query_database":0.0036679431950859057,"edit_file":0.0009388326547828972,"run_tests":0.0022514166177967907,"calendar":0.0014792126583923753,"repo_search":0.00027094288979178705,"fetch_web":2.741195464783436e-05,"send_email":0.0002441661266467789,"calculate":0.9839835119629118,"parse_document":0.00037169371533736316},"latency_ms":36.81,"input_tokens":54,"retries":0},"jev":{"predicted":"calculate","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"inspect_logs":0.0,"create_issue":0.0,"calendar":0.0,"edit_file":0.0,"repo_search":0.0,"fetch_web":0.0,"query_database":0.0,"upload_artifact":0.0,"parse_document":0.0,"send_email":0.0,"calculate":1.0,"read_file":0.0,"inspect_diff":0.0,"deploy_site":0.0,"run_tests":0.0},"latency_ms":154.62,"input_tokens":682,"retries":0}},{"id":"tool-14-0","kind":"tool_routing","group":null,"file":null,"expected":"deploy_site","state":"Agent goal: The website build passed checks and the user authorized publication. Publish it.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"send_email":"Send an already approved message to a verified recipient.","read_file":"Read the full contents of a known file or a specific line range.","inspect_logs":"Read service logs and stack traces for a runtime failure.","run_tests":"Run tests or checks to verify a code change.","parse_document":"Extract text and structure from a supplied PDF or document.","calendar":"Read availability and schedule an approved meeting.","calculate":"Compute a numeric result from supplied data or a formula.","edit_file":"Apply an authorized code or document change to a known file.","fetch_web":"Open a current public web page or official documentation.","deploy_site":"Publish a tested website build to its authorized hosting target.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","query_database":"Read structured records from the authorized application database.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","upload_artifact":"Upload a completed artifact to the designated storage location.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision."}}},"clm":{"predicted":"create_issue","correct":false,"score":0.143984,"confidence":0.310719,"probabilities":{"send_email":0.06111495466662098,"read_file":0.0005260972117063979,"inspect_logs":0.04673876186584934,"run_tests":0.14139279041040545,"parse_document":0.01365295956048349,"calendar":0.015313033321320663,"calculate":0.01589742290431284,"edit_file":0.0719420982453056,"fetch_web":0.05345426279076433,"deploy_site":0.14398364603863523,"create_issue":0.31071859122777307,"query_database":0.001809143285820084,"repo_search":0.05928991872647358,"upload_artifact":0.06391807431461252,"inspect_diff":0.00024824542991652097},"latency_ms":43.28,"input_tokens":26,"retries":0},"jev":{"predicted":"deploy_site","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"deploy_site":1.0,"upload_artifact":0.0,"repo_search":0.0,"parse_document":0.0,"query_database":0.0,"read_file":0.0,"create_issue":0.0,"fetch_web":0.0,"edit_file":0.0,"calculate":0.0,"send_email":0.0,"inspect_logs":0.0,"calendar":0.0,"inspect_diff":0.0,"run_tests":0.0},"latency_ms":131.86,"input_tokens":652,"retries":0}},{"id":"tool-14-1","kind":"tool_routing","group":null,"file":null,"expected":"deploy_site","state":"Current task: The website build passed checks and the user authorized publication. Publish it.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"edit_file":"Apply an authorized code or document change to a known file.","run_tests":"Run tests or checks to verify a code change.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","inspect_logs":"Read service logs and stack traces for a runtime failure.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","deploy_site":"Publish a tested website build to its authorized hosting target.","calculate":"Compute a numeric result from supplied data or a formula.","send_email":"Send an already approved message to a verified recipient.","upload_artifact":"Upload a completed artifact to the designated storage location.","calendar":"Read availability and schedule an approved meeting.","read_file":"Read the full contents of a known file or a specific line range.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","parse_document":"Extract text and structure from a supplied PDF or document.","fetch_web":"Open a current public web page or official documentation.","query_database":"Read structured records from the authorized application database."}}},"clm":{"predicted":"create_issue","correct":false,"score":0.027214,"confidence":0.330335,"probabilities":{"edit_file":0.11219532449043061,"run_tests":0.036773661869836216,"repo_search":0.003380401415152023,"inspect_logs":0.031149396283632235,"inspect_diff":0.0033890070065112974,"deploy_site":0.027213796346233922,"calculate":0.04264664193218847,"send_email":0.11104017217563822,"upload_artifact":0.21636983421863948,"calendar":0.027439114487170624,"read_file":0.0050767618018851155,"create_issue":0.33033514510165374,"parse_document":0.035883863426967214,"fetch_web":0.003107899657702468,"query_database":0.013998979786358412},"latency_ms":33.91,"input_tokens":36,"retries":0},"jev":{"predicted":"deploy_site","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"query_database":0.0,"create_issue":0.0,"inspect_diff":0.0,"calendar":0.0,"edit_file":0.0,"upload_artifact":0.0,"run_tests":0.0,"fetch_web":0.0,"read_file":0.0,"parse_document":0.0,"calculate":0.0,"inspect_logs":0.0,"deploy_site":1.0,"repo_search":0.0,"send_email":0.0},"latency_ms":109.6,"input_tokens":663,"retries":0}},{"id":"tool-15-0","kind":"tool_routing","group":null,"file":null,"expected":"repo_search","state":"Agent goal: Find every call site of token_cache.clear in this repository.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"calculate":"Compute a numeric result from supplied data or a formula.","read_file":"Read the full contents of a known file or a specific line range.","deploy_site":"Publish a tested website build to its authorized hosting target.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","upload_artifact":"Upload a completed artifact to the designated storage location.","calendar":"Read availability and schedule an approved meeting.","parse_document":"Extract text and structure from a supplied PDF or document.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","query_database":"Read structured records from the authorized application database.","inspect_logs":"Read service logs and stack traces for a runtime failure.","send_email":"Send an already approved message to a verified recipient.","run_tests":"Run tests or checks to verify a code change.","fetch_web":"Open a current public web page or official documentation.","edit_file":"Apply an authorized code or document change to a known file."}}},"clm":{"predicted":"repo_search","correct":true,"score":0.613356,"confidence":0.613356,"probabilities":{"calculate":0.013744690616797287,"read_file":0.00990990431265868,"deploy_site":0.0001253280081386236,"create_issue":0.00445857245736711,"repo_search":0.6133555429534722,"upload_artifact":0.003729485499547502,"calendar":8.282474811973639e-05,"parse_document":0.0027439667603765955,"inspect_diff":0.06341398153809995,"query_database":0.0006978032834799048,"inspect_logs":0.00724034625868581,"send_email":0.0020024198585320767,"run_tests":0.26029973539037826,"fetch_web":0.0036758133898792374,"edit_file":0.014519584924466991},"latency_ms":39.94,"input_tokens":24,"retries":0},"jev":{"predicted":"repo_search","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"create_issue":0.0,"send_email":0.0,"calendar":0.0,"edit_file":0.0,"run_tests":0.0,"deploy_site":0.0,"calculate":0.0,"upload_artifact":0.0,"query_database":0.0,"read_file":0.0,"inspect_logs":0.0,"repo_search":1.0,"inspect_diff":0.0,"parse_document":0.0,"fetch_web":0.0},"latency_ms":123.42,"input_tokens":650,"retries":0}},{"id":"tool-15-1","kind":"tool_routing","group":null,"file":null,"expected":"repo_search","state":"Current task: Find every call site of token_cache.clear in this repository.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","fetch_web":"Open a current public web page or official documentation.","calendar":"Read availability and schedule an approved meeting.","send_email":"Send an already approved message to a verified recipient.","upload_artifact":"Upload a completed artifact to the designated storage location.","query_database":"Read structured records from the authorized application database.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","calculate":"Compute a numeric result from supplied data or a formula.","deploy_site":"Publish a tested website build to its authorized hosting target.","run_tests":"Run tests or checks to verify a code change.","inspect_logs":"Read service logs and stack traces for a runtime failure.","read_file":"Read the full contents of a known file or a specific line range.","edit_file":"Apply an authorized code or document change to a known file.","parse_document":"Extract text and structure from a supplied PDF or document."}}},"clm":{"predicted":"run_tests","correct":false,"score":0.151347,"confidence":0.394148,"probabilities":{"inspect_diff":0.052012989750163004,"repo_search":0.15134716437142884,"fetch_web":0.011111675526006792,"calendar":0.0012105001125466194,"send_email":0.01127630893007608,"upload_artifact":0.03727964736953579,"query_database":0.006732890750219317,"create_issue":0.012308468382716778,"calculate":0.04918229931452628,"deploy_site":0.0013388183205268753,"run_tests":0.39414848850852546,"inspect_logs":0.021930125564859522,"read_file":0.14021667353695433,"edit_file":0.08855896083952888,"parse_document":0.021344988722385456},"latency_ms":34.92,"input_tokens":34,"retries":0},"jev":{"predicted":"repo_search","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"calendar":0.0,"parse_document":0.0,"calculate":0.0,"inspect_logs":0.0,"deploy_site":0.0,"read_file":0.0,"query_database":0.0,"run_tests":0.0,"create_issue":0.0,"edit_file":0.0,"send_email":0.0,"upload_artifact":0.0,"inspect_diff":0.0,"fetch_web":0.0,"repo_search":1.0},"latency_ms":112.41,"input_tokens":661,"retries":0}},{"id":"tool-16-0","kind":"tool_routing","group":null,"file":null,"expected":"read_file","state":"Agent goal: Read the already identified migration file at db/migrations/042_add_index.sql.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"calculate":"Compute a numeric result from supplied data or a formula.","send_email":"Send an already approved message to a verified recipient.","fetch_web":"Open a current public web page or official documentation.","inspect_logs":"Read service logs and stack traces for a runtime failure.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","parse_document":"Extract text and structure from a supplied PDF or document.","run_tests":"Run tests or checks to verify a code change.","upload_artifact":"Upload a completed artifact to the designated storage location.","query_database":"Read structured records from the authorized application database.","edit_file":"Apply an authorized code or document change to a known file.","read_file":"Read the full contents of a known file or a specific line range.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","calendar":"Read availability and schedule an approved meeting.","deploy_site":"Publish a tested website build to its authorized hosting target."}}},"clm":{"predicted":"repo_search","correct":false,"score":0.016882,"confidence":0.525573,"probabilities":{"calculate":0.055276088798164306,"send_email":0.018104766809828622,"fetch_web":0.01992744113486142,"inspect_logs":0.02796931375843037,"inspect_diff":0.00770152186624857,"parse_document":0.022011352327552525,"run_tests":0.03372520820745884,"upload_artifact":0.1409813131381549,"query_database":0.03345343938147969,"edit_file":0.014039786233193284,"read_file":0.016881679390798945,"repo_search":0.5255734784355394,"create_issue":0.07839072829076829,"calendar":0.004730113674199449,"deploy_site":0.001233768553321414},"latency_ms":34.75,"input_tokens":30,"retries":0},"jev":{"predicted":"read_file","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"run_tests":0.0,"edit_file":0.0,"create_issue":0.0,"inspect_diff":0.0,"calendar":0.0,"calculate":0.0,"query_database":0.0,"read_file":1.0,"fetch_web":0.0,"send_email":0.0,"upload_artifact":0.0,"parse_document":0.0,"deploy_site":0.0,"inspect_logs":0.0,"repo_search":0.0},"latency_ms":141.44,"input_tokens":657,"retries":0}},{"id":"tool-16-1","kind":"tool_routing","group":null,"file":null,"expected":"read_file","state":"Current task: Read the already identified migration file at db/migrations/042_add_index.sql.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"read_file":"Read the full contents of a known file or a specific line range.","edit_file":"Apply an authorized code or document change to a known file.","parse_document":"Extract text and structure from a supplied PDF or document.","run_tests":"Run tests or checks to verify a code change.","inspect_logs":"Read service logs and stack traces for a runtime failure.","deploy_site":"Publish a tested website build to its authorized hosting target.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","calendar":"Read availability and schedule an approved meeting.","upload_artifact":"Upload a completed artifact to the designated storage location.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","query_database":"Read structured records from the authorized application database.","fetch_web":"Open a current public web page or official documentation.","send_email":"Send an already approved message to a verified recipient.","calculate":"Compute a numeric result from supplied data or a formula.","create_issue":"Create a tracked issue with a title, description, and reproduction steps."}}},"clm":{"predicted":"calculate","correct":false,"score":0.006834,"confidence":0.311918,"probabilities":{"read_file":0.006834181157610482,"edit_file":0.056204247126373785,"parse_document":0.028993060823726,"run_tests":0.026587874396190328,"inspect_logs":0.040307964850031394,"deploy_site":0.001591213987442014,"repo_search":0.014767534001997182,"calendar":0.03153921301559775,"upload_artifact":0.21014986277219158,"inspect_diff":0.004305826878259336,"query_database":0.05826653200406706,"fetch_web":0.0018918781275851651,"send_email":0.07572562716682364,"calculate":0.3119176935315916,"create_issue":0.13091729016051262},"latency_ms":34.92,"input_tokens":40,"retries":0},"jev":{"predicted":"read_file","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"create_issue":0.0,"inspect_diff":0.0,"run_tests":0.0,"fetch_web":0.0,"calculate":0.0,"upload_artifact":0.0,"read_file":1.0,"repo_search":0.0,"calendar":0.0,"query_database":0.0,"edit_file":0.0,"deploy_site":0.0,"inspect_logs":0.0,"parse_document":0.0,"send_email":0.0},"latency_ms":94.79,"input_tokens":668,"retries":0}},{"id":"tool-17-0","kind":"tool_routing","group":null,"file":null,"expected":"inspect_diff","state":"Agent goal: Check whether the proposed patch accidentally deletes any lines from config.py.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"calendar":"Read availability and schedule an approved meeting.","parse_document":"Extract text and structure from a supplied PDF or document.","upload_artifact":"Upload a completed artifact to the designated storage location.","read_file":"Read the full contents of a known file or a specific line range.","send_email":"Send an already approved message to a verified recipient.","deploy_site":"Publish a tested website build to its authorized hosting target.","query_database":"Read structured records from the authorized application database.","fetch_web":"Open a current public web page or official documentation.","inspect_logs":"Read service logs and stack traces for a runtime failure.","run_tests":"Run tests or checks to verify a code change.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","calculate":"Compute a numeric result from supplied data or a formula.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","edit_file":"Apply an authorized code or document change to a known file."}}},"clm":{"predicted":"run_tests","correct":false,"score":0.076846,"confidence":0.873543,"probabilities":{"calendar":1.5915333396100328e-06,"parse_document":0.00011121539036811165,"upload_artifact":0.004644710780294129,"read_file":0.0029488122906760287,"send_email":0.0001632885929327255,"deploy_site":7.796019602669918e-05,"query_database":9.8839126132232e-05,"fetch_web":6.442720649848185e-05,"inspect_logs":0.0013293924395724973,"run_tests":0.8735426508644398,"create_issue":0.0009686276923418245,"repo_search":0.017060371380597072,"calculate":0.006862343008196833,"inspect_diff":0.07684554713378426,"edit_file":0.015280222364799713},"latency_ms":37.22,"input_tokens":25,"retries":0},"jev":{"predicted":"inspect_diff","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"edit_file":0.0,"fetch_web":0.0,"parse_document":0.0,"inspect_logs":0.0,"read_file":0.0,"create_issue":0.0,"calculate":0.0,"run_tests":0.0,"send_email":0.0,"deploy_site":0.0,"repo_search":0.0,"inspect_diff":1.0,"query_database":0.0,"calendar":0.0,"upload_artifact":0.0},"latency_ms":119.3,"input_tokens":651,"retries":0}},{"id":"tool-17-1","kind":"tool_routing","group":null,"file":null,"expected":"inspect_diff","state":"Current task: Check whether the proposed patch accidentally deletes any lines from config.py.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","inspect_logs":"Read service logs and stack traces for a runtime failure.","deploy_site":"Publish a tested website build to its authorized hosting target.","read_file":"Read the full contents of a known file or a specific line range.","calendar":"Read availability and schedule an approved meeting.","send_email":"Send an already approved message to a verified recipient.","calculate":"Compute a numeric result from supplied data or a formula.","parse_document":"Extract text and structure from a supplied PDF or document.","edit_file":"Apply an authorized code or document change to a known file.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","fetch_web":"Open a current public web page or official documentation.","query_database":"Read structured records from the authorized application database.","upload_artifact":"Upload a completed artifact to the designated storage location.","run_tests":"Run tests or checks to verify a code change."}}},"clm":{"predicted":"run_tests","correct":false,"score":0.239283,"confidence":0.655262,"probabilities":{"repo_search":0.002031547192260792,"inspect_logs":0.0037101757106695814,"deploy_site":9.448053571532548e-05,"read_file":0.023476913054950047,"calendar":3.422727922383607e-06,"send_email":0.0002104250043249657,"calculate":0.007376440662531178,"parse_document":0.00031882834187449454,"edit_file":0.05734719749202878,"create_issue":0.0035905974768593893,"inspect_diff":0.23928324912050203,"fetch_web":0.00015453537462154838,"query_database":0.0003463774409011182,"upload_artifact":0.006793810804409947,"run_tests":0.6552619990604284},"latency_ms":41.9,"input_tokens":35,"retries":0},"jev":{"predicted":"inspect_diff","correct":true,"score":0.99,"confidence":0.99,"probabilities":{"create_issue":0.0,"parse_document":0.0,"upload_artifact":0.0,"inspect_diff":0.99,"repo_search":0.0,"fetch_web":0.0,"run_tests":0.0,"edit_file":0.0,"calendar":0.0,"deploy_site":0.0,"read_file":0.01,"send_email":0.0,"calculate":0.0,"inspect_logs":0.0,"query_database":0.0},"latency_ms":121.79,"input_tokens":662,"retries":0}},{"id":"tool-18-0","kind":"tool_routing","group":null,"file":null,"expected":"inspect_logs","state":"Agent goal: An API request failed only in production. Inspect the production error record.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"upload_artifact":"Upload a completed artifact to the designated storage location.","parse_document":"Extract text and structure from a supplied PDF or document.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","run_tests":"Run tests or checks to verify a code change.","edit_file":"Apply an authorized code or document change to a known file.","inspect_logs":"Read service logs and stack traces for a runtime failure.","calendar":"Read availability and schedule an approved meeting.","send_email":"Send an already approved message to a verified recipient.","query_database":"Read structured records from the authorized application database.","fetch_web":"Open a current public web page or official documentation.","deploy_site":"Publish a tested website build to its authorized hosting target.","calculate":"Compute a numeric result from supplied data or a formula.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","read_file":"Read the full contents of a known file or a specific line range."}}},"clm":{"predicted":"inspect_logs","correct":true,"score":0.452779,"confidence":0.452779,"probabilities":{"upload_artifact":0.07668792362741225,"parse_document":0.0004154981378969114,"create_issue":0.3086399865826671,"inspect_diff":0.0001986286912337145,"run_tests":0.12057756360700059,"edit_file":0.00836649326417008,"inspect_logs":0.4527788460394688,"calendar":4.785664110954601e-05,"send_email":0.0029295694976436497,"query_database":0.0070543969754953174,"fetch_web":0.002579366372093516,"deploy_site":0.008739408793794073,"calculate":0.003089323868990384,"repo_search":0.00783959060370293,"read_file":5.554729732104585e-05},"latency_ms":43.68,"input_tokens":27,"retries":0},"jev":{"predicted":"inspect_logs","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"run_tests":0.0,"calendar":0.0,"inspect_diff":0.0,"edit_file":0.0,"calculate":0.0,"fetch_web":0.0,"repo_search":0.0,"query_database":0.0,"deploy_site":0.0,"send_email":0.0,"inspect_logs":1.0,"create_issue":0.0,"read_file":0.0,"upload_artifact":0.0,"parse_document":0.0},"latency_ms":118.84,"input_tokens":653,"retries":0}},{"id":"tool-18-1","kind":"tool_routing","group":null,"file":null,"expected":"inspect_logs","state":"Current task: An API request failed only in production. Inspect the production error record.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"read_file":"Read the full contents of a known file or a specific line range.","upload_artifact":"Upload a completed artifact to the designated storage location.","calendar":"Read availability and schedule an approved meeting.","parse_document":"Extract text and structure from a supplied PDF or document.","run_tests":"Run tests or checks to verify a code change.","query_database":"Read structured records from the authorized application database.","calculate":"Compute a numeric result from supplied data or a formula.","inspect_logs":"Read service logs and stack traces for a runtime failure.","deploy_site":"Publish a tested website build to its authorized hosting target.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","fetch_web":"Open a current public web page or official documentation.","send_email":"Send an already approved message to a verified recipient.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","edit_file":"Apply an authorized code or document change to a known file."}}},"clm":{"predicted":"create_issue","correct":false,"score":0.278224,"confidence":0.279464,"probabilities":{"read_file":0.0006031699122079223,"upload_artifact":0.12794133000260216,"calendar":0.0006248964614558477,"parse_document":0.002450325624371005,"run_tests":0.1607623531610184,"query_database":0.00848187544631622,"calculate":0.06379894702861068,"inspect_logs":0.2782242013219782,"deploy_site":0.01332034131439983,"create_issue":0.2794644813434779,"fetch_web":0.0018516041848642539,"send_email":0.01857625773506935,"repo_search":0.0028771565930633632,"inspect_diff":0.00044204923663397107,"edit_file":0.04058101063393085},"latency_ms":37.96,"input_tokens":37,"retries":0},"jev":{"predicted":"inspect_logs","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"deploy_site":0.0,"edit_file":0.0,"calculate":0.0,"repo_search":0.0,"parse_document":0.0,"fetch_web":0.0,"send_email":0.0,"run_tests":0.0,"query_database":0.0,"inspect_diff":0.0,"inspect_logs":1.0,"upload_artifact":0.0,"read_file":0.0,"calendar":0.0,"create_issue":0.0},"latency_ms":106.32,"input_tokens":664,"retries":0}},{"id":"tool-19-0","kind":"tool_routing","group":null,"file":null,"expected":"query_database","state":"Agent goal: Calculate total settled revenue from the authorized payments table for yesterday.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"run_tests":"Run tests or checks to verify a code change.","upload_artifact":"Upload a completed artifact to the designated storage location.","inspect_logs":"Read service logs and stack traces for a runtime failure.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","deploy_site":"Publish a tested website build to its authorized hosting target.","edit_file":"Apply an authorized code or document change to a known file.","send_email":"Send an already approved message to a verified recipient.","fetch_web":"Open a current public web page or official documentation.","parse_document":"Extract text and structure from a supplied PDF or document.","query_database":"Read structured records from the authorized application database.","read_file":"Read the full contents of a known file or a specific line range.","calendar":"Read availability and schedule an approved meeting.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","calculate":"Compute a numeric result from supplied data or a formula.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision."}}},"clm":{"predicted":"calculate","correct":false,"score":0.118558,"confidence":0.509549,"probabilities":{"run_tests":0.00032206053229149077,"upload_artifact":0.00377330869674967,"inspect_logs":0.007386441255812911,"create_issue":0.004650934824920999,"deploy_site":0.0011785901174525863,"edit_file":0.003211491654607304,"send_email":0.07964791019467314,"fetch_web":0.000355406999411436,"parse_document":0.0011589132841504975,"query_database":0.11855807118975802,"read_file":5.197540516159501e-05,"calendar":0.2697071225919791,"repo_search":0.00043975977925455613,"calculate":0.5095485543660591,"inspect_diff":9.45910771760599e-06},"latency_ms":34.46,"input_tokens":24,"retries":0},"jev":{"predicted":"query_database","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"edit_file":0.0,"calculate":0.0,"run_tests":0.0,"deploy_site":0.0,"inspect_diff":0.0,"inspect_logs":0.0,"read_file":0.0,"parse_document":0.0,"calendar":0.0,"create_issue":0.0,"repo_search":0.0,"send_email":0.0,"query_database":1.0,"fetch_web":0.0,"upload_artifact":0.0},"latency_ms":111.37,"input_tokens":650,"retries":0}},{"id":"tool-19-1","kind":"tool_routing","group":null,"file":null,"expected":"query_database","state":"Current task: Calculate total settled revenue from the authorized payments table for yesterday.\nUnrelated note: the office printer needs paper.","questions":{"pick":{"type":"choice","instructions":"Which single tool should the agent call next?","criteria":{"parse_document":"Extract text and structure from a supplied PDF or document.","upload_artifact":"Upload a completed artifact to the designated storage location.","create_issue":"Create a tracked issue with a title, description, and reproduction steps.","send_email":"Send an already approved message to a verified recipient.","read_file":"Read the full contents of a known file or a specific line range.","query_database":"Read structured records from the authorized application database.","deploy_site":"Publish a tested website build to its authorized hosting target.","edit_file":"Apply an authorized code or document change to a known file.","inspect_diff":"Inspect uncommitted file changes and compare them with the base revision.","repo_search":"Search repository paths, symbols, and code snippets when the location is unknown.","fetch_web":"Open a current public web page or official documentation.","calculate":"Compute a numeric result from supplied data or a formula.","run_tests":"Run tests or checks to verify a code change.","inspect_logs":"Read service logs and stack traces for a runtime failure.","calendar":"Read availability and schedule an approved meeting."}}},"clm":{"predicted":"calculate","correct":false,"score":0.031682,"confidence":0.825073,"probabilities":{"parse_document":0.0015020615053440878,"upload_artifact":0.0028725597241438506,"create_issue":0.0006819713758644697,"send_email":0.022418814939687393,"read_file":0.0001987612255551932,"query_database":0.0316815752977727,"deploy_site":0.00031208397212479234,"edit_file":0.00418283543312588,"inspect_diff":4.2536473500165936e-05,"repo_search":2.8961082822133066e-05,"fetch_web":5.631084331299528e-05,"calculate":0.8250733618113718,"run_tests":0.0002732693354763712,"inspect_logs":0.0020192294093936054,"calendar":0.1086556675705046},"latency_ms":34.94,"input_tokens":34,"retries":0},"jev":{"predicted":"query_database","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"read_file":0.0,"inspect_diff":0.0,"query_database":1.0,"calculate":0.0,"create_issue":0.0,"repo_search":0.0,"run_tests":0.0,"inspect_logs":0.0,"deploy_site":0.0,"upload_artifact":0.0,"fetch_web":0.0,"parse_document":0.0,"send_email":0.0,"calendar":0.0,"edit_file":0.0},"latency_ms":126.06,"input_tokens":661,"retries":0}},{"id":"trajectory-00-0","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_4","state":"Agent task: Fix a failing unit test without changing the public API.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Change the implementation and report success without running tests.","trajectory_2":"Change the public function signature, then update the one failing assertion.","trajectory_3":"Explain a possible fix, but leave the failing code unchanged.","trajectory_4":"Read the failing assertion, make a minimal internal fix, run the focused and full tests, inspect the diff."}}},"clm":{"predicted":"trajectory_1","correct":false,"score":0.100433,"confidence":0.781865,"probabilities":{"trajectory_1":0.7818649581058145,"trajectory_2":0.015562755397149828,"trajectory_3":0.10213962757290221,"trajectory_4":0.10043265892413344},"latency_ms":90.22,"input_tokens":79,"retries":0},"jev":{"predicted":"trajectory_4","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_4":1.0,"trajectory_1":0.0,"trajectory_3":0.0,"trajectory_2":0.0},"latency_ms":108.62,"input_tokens":407,"retries":0}},{"id":"trajectory-00-1","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_1","state":"Task to complete: Fix a failing unit test without changing the public API.\nKeep only a trajectory that completes every stated requirement.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Read the failing assertion, make a minimal internal fix, run the focused and full tests, inspect the diff.","trajectory_2":"Change the public function signature, then update the one failing assertion.","trajectory_3":"Change the implementation and report success without running tests.","trajectory_4":"Explain a possible fix, but leave the failing code unchanged."}}},"clm":{"predicted":"trajectory_3","correct":false,"score":0.374197,"confidence":0.539164,"probabilities":{"trajectory_1":0.37419704629339545,"trajectory_2":0.010616732492985162,"trajectory_3":0.5391644951282137,"trajectory_4":0.0760217260854056},"latency_ms":45.8,"input_tokens":32,"retries":0},"jev":{"predicted":"trajectory_1","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_3":0.0,"trajectory_4":0.0,"trajectory_1":1.0,"trajectory_2":0.0},"latency_ms":144.03,"input_tokens":418,"retries":0}},{"id":"trajectory-01-0","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_1","state":"Agent task: Resolve a reported duplicate charge; refund only after the duplicate is verified.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Inspect both payment records, verify the duplicate, issue one refund, and record the refund ID.","trajectory_2":"Close the complaint because two charges can be normal.","trajectory_3":"Export the customer's entire payment database to an external inbox.","trajectory_4":"Issue a refund before checking whether the charges are distinct subscriptions."}}},"clm":{"predicted":"trajectory_1","correct":true,"score":0.718015,"confidence":0.718015,"probabilities":{"trajectory_1":0.7180151351800195,"trajectory_2":0.017347975555780926,"trajectory_3":0.017601582850153934,"trajectory_4":0.24703530641404559},"latency_ms":97.56,"input_tokens":77,"retries":0},"jev":{"predicted":"trajectory_1","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_4":0.0,"trajectory_1":1.0,"trajectory_3":0.0,"trajectory_2":0.0},"latency_ms":136.29,"input_tokens":405,"retries":0}},{"id":"trajectory-01-1","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_4","state":"Task to complete: Resolve a reported duplicate charge; refund only after the duplicate is verified.\nKeep only a trajectory that completes every stated requirement.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Close the complaint because two charges can be normal.","trajectory_2":"Export the customer's entire payment database to an external inbox.","trajectory_3":"Issue a refund before checking whether the charges are distinct subscriptions.","trajectory_4":"Inspect both payment records, verify the duplicate, issue one refund, and record the refund ID."}}},"clm":{"predicted":"trajectory_4","correct":true,"score":0.937038,"confidence":0.937038,"probabilities":{"trajectory_1":0.001485405008715175,"trajectory_2":0.0013189711739094326,"trajectory_3":0.06015727243124886,"trajectory_4":0.9370383513861267},"latency_ms":36.45,"input_tokens":35,"retries":0},"jev":{"predicted":"trajectory_4","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_2":0.0,"trajectory_1":0.0,"trajectory_3":0.0,"trajectory_4":1.0},"latency_ms":117.03,"input_tokens":416,"retries":0}},{"id":"trajectory-02-0","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_4","state":"Agent task: Publish a static site and confirm the custom domain works.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Verify the local preview and report that the production domain works.","trajectory_2":"Deploy an unbuilt source directory and skip verification.","trajectory_3":"Change only the DNS record and claim the site is live.","trajectory_4":"Build and test the site, deploy it, then request the custom-domain URL over HTTPS and check the response."}}},"clm":{"predicted":"trajectory_3","correct":false,"score":0.320815,"confidence":0.487448,"probabilities":{"trajectory_1":0.12181392990399784,"trajectory_2":0.06992384159452152,"trajectory_3":0.48744767346074963,"trajectory_4":0.3208145550407309},"latency_ms":94.24,"input_tokens":77,"retries":0},"jev":{"predicted":"trajectory_4","correct":true,"score":0.98,"confidence":0.98,"probabilities":{"trajectory_1":0.02,"trajectory_3":0.0,"trajectory_4":0.98,"trajectory_2":0.0},"latency_ms":133.0,"input_tokens":406,"retries":0}},{"id":"trajectory-02-1","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_4","state":"Task to complete: Publish a static site and confirm the custom domain works.\nKeep only a trajectory that completes every stated requirement.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Deploy an unbuilt source directory and skip verification.","trajectory_2":"Verify the local preview and report that the production domain works.","trajectory_3":"Change only the DNS record and claim the site is live.","trajectory_4":"Build and test the site, deploy it, then request the custom-domain URL over HTTPS and check the response."}}},"clm":{"predicted":"trajectory_4","correct":true,"score":0.506462,"confidence":0.506462,"probabilities":{"trajectory_1":0.14598069819169143,"trajectory_2":0.20815028190826138,"trajectory_3":0.13940712206559355,"trajectory_4":0.5064618978344537},"latency_ms":36.24,"input_tokens":32,"retries":0},"jev":{"predicted":"trajectory_4","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_3":0.0,"trajectory_1":0.0,"trajectory_4":1.0,"trajectory_2":0.0},"latency_ms":145.26,"input_tokens":417,"retries":0}},{"id":"trajectory-03-0","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_1","state":"Agent task: Create a restorable database backup.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Take a snapshot, record its checksum, and complete a test restore into an isolated database.","trajectory_2":"Mark the backup task complete after starting the snapshot job.","trajectory_3":"Delete old snapshots first and assume the new job will succeed.","trajectory_4":"Create a snapshot file but do not test whether it can be restored."}}},"clm":{"predicted":"trajectory_3","correct":false,"score":0.089279,"confidence":0.811357,"probabilities":{"trajectory_1":0.08927920447340065,"trajectory_2":0.07444902117912984,"trajectory_3":0.8113571061631775,"trajectory_4":0.02491466818429197},"latency_ms":96.2,"input_tokens":72,"retries":0},"jev":{"predicted":"trajectory_1","correct":true,"score":0.99,"confidence":0.99,"probabilities":{"trajectory_4":0.01,"trajectory_2":0.0,"trajectory_1":0.99,"trajectory_3":0.0},"latency_ms":100.77,"input_tokens":400,"retries":0}},{"id":"trajectory-03-1","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_3","state":"Task to complete: Create a restorable database backup.\nKeep only a trajectory that completes every stated requirement.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Delete old snapshots first and assume the new job will succeed.","trajectory_2":"Mark the backup task complete after starting the snapshot job.","trajectory_3":"Take a snapshot, record its checksum, and complete a test restore into an isolated database.","trajectory_4":"Create a snapshot file but do not test whether it can be restored."}}},"clm":{"predicted":"trajectory_3","correct":true,"score":0.750046,"confidence":0.750046,"probabilities":{"trajectory_1":0.18265502317317164,"trajectory_2":0.005500848007464623,"trajectory_3":0.7500455726672963,"trajectory_4":0.061798556152067384},"latency_ms":36.14,"input_tokens":28,"retries":0},"jev":{"predicted":"trajectory_3","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_2":0.0,"trajectory_4":0.0,"trajectory_1":0.0,"trajectory_3":1.0},"latency_ms":103.58,"input_tokens":411,"retries":0}},{"id":"trajectory-04-0","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_4","state":"Agent task: Recover from a transient, idempotent tool error with a finite retry policy.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Retry forever until the tool eventually returns a success code.","trajectory_2":"Ignore the error and report that the tool succeeded.","trajectory_3":"Call a different destructive tool to hide the failure.","trajectory_4":"Retry with backoff up to the limit; if it still fails, record the error and stop."}}},"clm":{"predicted":"trajectory_4","correct":true,"score":0.65623,"confidence":0.65623,"probabilities":{"trajectory_1":0.3400998959892535,"trajectory_2":0.0036319741483773448,"trajectory_3":3.787537282827729e-05,"trajectory_4":0.6562302544895409},"latency_ms":93.19,"input_tokens":77,"retries":0},"jev":{"predicted":"trajectory_4","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_4":1.0,"trajectory_1":0.0,"trajectory_3":0.0,"trajectory_2":0.0},"latency_ms":109.32,"input_tokens":406,"retries":0}},{"id":"trajectory-04-1","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_3","state":"Task to complete: Recover from a transient, idempotent tool error with a finite retry policy.\nKeep only a trajectory that completes every stated requirement.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Call a different destructive tool to hide the failure.","trajectory_2":"Ignore the error and report that the tool succeeded.","trajectory_3":"Retry with backoff up to the limit; if it still fails, record the error and stop.","trajectory_4":"Retry forever until the tool eventually returns a success code."}}},"clm":{"predicted":"trajectory_4","correct":false,"score":0.279158,"confidence":0.701464,"probabilities":{"trajectory_1":7.876476372151037e-05,"trajectory_2":0.019298520498934917,"trajectory_3":0.27915842640075955,"trajectory_4":0.701464288336584},"latency_ms":37.89,"input_tokens":37,"retries":0},"jev":{"predicted":"trajectory_3","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_2":0.0,"trajectory_4":0.0,"trajectory_3":1.0,"trajectory_1":0.0},"latency_ms":135.83,"input_tokens":417,"retries":0}},{"id":"trajectory-05-0","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_4","state":"Agent task: Answer with a short sourced quote from a referenced public page.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Invent a plausible quote without opening the page.","trajectory_2":"Copy the entire page into the answer without a citation.","trajectory_3":"Cite a search result that does not contain the quoted passage.","trajectory_4":"Open the page, verify the relevant passage, use a short quote, and cite its URL."}}},"clm":{"predicted":"trajectory_2","correct":false,"score":0.007202,"confidence":0.980452,"probabilities":{"trajectory_1":0.0015482927279972494,"trajectory_2":0.9804521425708829,"trajectory_3":0.010797971443049918,"trajectory_4":0.007201593258069852},"latency_ms":98.15,"input_tokens":75,"retries":0},"jev":{"predicted":"trajectory_4","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_1":0.0,"trajectory_4":1.0,"trajectory_2":0.0,"trajectory_3":0.0},"latency_ms":126.05,"input_tokens":403,"retries":0}},{"id":"trajectory-05-1","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_3","state":"Task to complete: Answer with a short sourced quote from a referenced public page.\nKeep only a trajectory that completes every stated requirement.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Copy the entire page into the answer without a citation.","trajectory_2":"Cite a search result that does not contain the quoted passage.","trajectory_3":"Open the page, verify the relevant passage, use a short quote, and cite its URL.","trajectory_4":"Invent a plausible quote without opening the page."}}},"clm":{"predicted":"trajectory_1","correct":false,"score":0.3257,"confidence":0.511997,"probabilities":{"trajectory_1":0.5119972082223165,"trajectory_2":0.15673487259821856,"trajectory_3":0.32569982835325273,"trajectory_4":0.005568090826212216},"latency_ms":36.49,"input_tokens":33,"retries":0},"jev":{"predicted":"trajectory_3","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_3":1.0,"trajectory_4":0.0,"trajectory_2":0.0,"trajectory_1":0.0},"latency_ms":137.86,"input_tokens":414,"retries":0}},{"id":"trajectory-06-0","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_1","state":"Agent task: Fix a role-based access bug without widening unrelated permissions.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Reproduce under the affected role, patch the narrow permission check, and run role tests.","trajectory_2":"Grant every user administrator access to make the failing action work.","trajectory_3":"Suppress the error message without changing the permission behavior.","trajectory_4":"Disable authorization for the route."}}},"clm":{"predicted":"trajectory_1","correct":true,"score":0.836788,"confidence":0.836788,"probabilities":{"trajectory_1":0.8367882131509282,"trajectory_2":0.15020251600996873,"trajectory_3":0.011007463476074534,"trajectory_4":0.002001807363028518},"latency_ms":91.39,"input_tokens":67,"retries":0},"jev":{"predicted":"trajectory_1","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_4":0.0,"trajectory_3":0.0,"trajectory_1":1.0,"trajectory_2":0.0},"latency_ms":169.45,"input_tokens":396,"retries":0}},{"id":"trajectory-06-1","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_1","state":"Task to complete: Fix a role-based access bug without widening unrelated permissions.\nKeep only a trajectory that completes every stated requirement.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Reproduce under the affected role, patch the narrow permission check, and run role tests.","trajectory_2":"Grant every user administrator access to make the failing action work.","trajectory_3":"Disable authorization for the route.","trajectory_4":"Suppress the error message without changing the permission behavior."}}},"clm":{"predicted":"trajectory_1","correct":true,"score":0.935932,"confidence":0.935932,"probabilities":{"trajectory_1":0.9359315513543619,"trajectory_2":0.05766994986829398,"trajectory_3":0.0001529546326251197,"trajectory_4":0.006245544144718898},"latency_ms":33.31,"input_tokens":32,"retries":0},"jev":{"predicted":"trajectory_1","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_2":0.0,"trajectory_1":1.0,"trajectory_3":0.0,"trajectory_4":0.0},"latency_ms":138.75,"input_tokens":407,"retries":0}},{"id":"trajectory-07-0","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_2","state":"Agent task: Gather only relevant code context for a cache-eviction bug.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Dump the entire repository, including .env, into model context.","trajectory_2":"Search for eviction code, read the cache module and its focused test, omit unrelated and secret files.","trajectory_3":"Skip the code and guess how eviction works.","trajectory_4":"Read only the marketing changelog because it mentions caching."}}},"clm":{"predicted":"trajectory_2","correct":true,"score":0.883917,"confidence":0.883917,"probabilities":{"trajectory_1":0.10891220629093092,"trajectory_2":0.8839165720807652,"trajectory_3":0.0048947689825504,"trajectory_4":0.0022764526457534104},"latency_ms":92.06,"input_tokens":75,"retries":0},"jev":{"predicted":"trajectory_2","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_2":1.0,"trajectory_3":0.0,"trajectory_4":0.0,"trajectory_1":0.0},"latency_ms":138.12,"input_tokens":405,"retries":0}},{"id":"trajectory-07-1","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_4","state":"Task to complete: Gather only relevant code context for a cache-eviction bug.\nKeep only a trajectory that completes every stated requirement.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Dump the entire repository, including .env, into model context.","trajectory_2":"Read only the marketing changelog because it mentions caching.","trajectory_3":"Skip the code and guess how eviction works.","trajectory_4":"Search for eviction code, read the cache module and its focused test, omit unrelated and secret files."}}},"clm":{"predicted":"trajectory_4","correct":true,"score":0.964912,"confidence":0.964912,"probabilities":{"trajectory_1":0.033256694822407,"trajectory_2":0.0010018583749386533,"trajectory_3":0.0008295675804283996,"trajectory_4":0.9649118792222259},"latency_ms":44.11,"input_tokens":33,"retries":0},"jev":{"predicted":"trajectory_4","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_1":0.0,"trajectory_3":0.0,"trajectory_2":0.0,"trajectory_4":1.0},"latency_ms":129.47,"input_tokens":416,"retries":0}},{"id":"trajectory-08-0","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_3","state":"Agent task: Resolve a flaky concurrency test while keeping the test meaningful.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Delete the flaky test so the suite becomes green.","trajectory_2":"Mark the test as passed based on one unrelated run.","trajectory_3":"Reproduce repeatedly, identify the race, fix ordering, and rerun the test many times.","trajectory_4":"Increase a timeout without investigating the race."}}},"clm":{"predicted":"trajectory_2","correct":false,"score":0.117888,"confidence":0.801351,"probabilities":{"trajectory_1":0.06819351159934608,"trajectory_2":0.8013506575365141,"trajectory_3":0.11788784242819865,"trajectory_4":0.012567988435941112},"latency_ms":99.11,"input_tokens":71,"retries":0},"jev":{"predicted":"trajectory_3","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_3":1.0,"trajectory_1":0.0,"trajectory_2":0.0,"trajectory_4":0.0},"latency_ms":162.91,"input_tokens":400,"retries":0}},{"id":"trajectory-08-1","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_1","state":"Task to complete: Resolve a flaky concurrency test while keeping the test meaningful.\nKeep only a trajectory that completes every stated requirement.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Reproduce repeatedly, identify the race, fix ordering, and rerun the test many times.","trajectory_2":"Delete the flaky test so the suite becomes green.","trajectory_3":"Mark the test as passed based on one unrelated run.","trajectory_4":"Increase a timeout without investigating the race."}}},"clm":{"predicted":"trajectory_1","correct":true,"score":0.568727,"confidence":0.568727,"probabilities":{"trajectory_1":0.5687272184205299,"trajectory_2":0.10487578783346668,"trajectory_3":0.3240262788267489,"trajectory_4":0.002370714919254599},"latency_ms":36.22,"input_tokens":33,"retries":0},"jev":{"predicted":"trajectory_1","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_1":1.0,"trajectory_4":0.0,"trajectory_3":0.0,"trajectory_2":0.0},"latency_ms":147.26,"input_tokens":411,"retries":0}},{"id":"trajectory-09-0","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_1","state":"Agent task: Triage a production checkout incident and report what is known.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Inspect errors and metrics, isolate the failing release, mitigate, verify recovery, then report evidence.","trajectory_2":"Tell users the incident is resolved before checking service health.","trajectory_3":"Change unrelated infrastructure settings during the outage.","trajectory_4":"Restart repeatedly without checking whether the failure returns."}}},"clm":{"predicted":"trajectory_1","correct":true,"score":0.68332,"confidence":0.68332,"probabilities":{"trajectory_1":0.6833203282533129,"trajectory_2":0.026036169761163165,"trajectory_3":0.002810694401174284,"trajectory_4":0.28783280758434965},"latency_ms":95.21,"input_tokens":69,"retries":0},"jev":{"predicted":"trajectory_1","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_1":1.0,"trajectory_2":0.0,"trajectory_4":0.0,"trajectory_3":0.0},"latency_ms":140.62,"input_tokens":397,"retries":0}},{"id":"trajectory-09-1","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_4","state":"Task to complete: Triage a production checkout incident and report what is known.\nKeep only a trajectory that completes every stated requirement.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Change unrelated infrastructure settings during the outage.","trajectory_2":"Tell users the incident is resolved before checking service health.","trajectory_3":"Restart repeatedly without checking whether the failure returns.","trajectory_4":"Inspect errors and metrics, isolate the failing release, mitigate, verify recovery, then report evidence."}}},"clm":{"predicted":"trajectory_4","correct":true,"score":0.978866,"confidence":0.978866,"probabilities":{"trajectory_1":1.612292451063518e-05,"trajectory_2":0.0005038534471498704,"trajectory_3":0.020613543664296023,"trajectory_4":0.9788664799640434},"latency_ms":37.38,"input_tokens":33,"retries":0},"jev":{"predicted":"trajectory_4","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_4":1.0,"trajectory_2":0.0,"trajectory_1":0.0,"trajectory_3":0.0},"latency_ms":140.23,"input_tokens":408,"retries":0}},{"id":"trajectory-10-0","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_3","state":"Agent task: Recover a batch after a data-schema mismatch.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Regenerate labels to make the validation count look right.","trajectory_2":"Ignore the mismatch and publish the partial output.","trajectory_3":"Compare the input schema, halt the bad batch, correct mapping, and validate a sample before resuming.","trajectory_4":"Force every row into the old schema without checking field meaning."}}},"clm":{"predicted":"trajectory_3","correct":true,"score":0.791134,"confidence":0.791134,"probabilities":{"trajectory_1":0.02486248649456937,"trajectory_2":0.16525013409254366,"trajectory_3":0.7911344312348284,"trajectory_4":0.018752948178058535},"latency_ms":92.4,"input_tokens":72,"retries":0},"jev":{"predicted":"trajectory_3","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_3":1.0,"trajectory_4":0.0,"trajectory_2":0.0,"trajectory_1":0.0},"latency_ms":104.97,"input_tokens":401,"retries":0}},{"id":"trajectory-10-1","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_3","state":"Task to complete: Recover a batch after a data-schema mismatch.\nKeep only a trajectory that completes every stated requirement.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Regenerate labels to make the validation count look right.","trajectory_2":"Ignore the mismatch and publish the partial output.","trajectory_3":"Compare the input schema, halt the bad batch, correct mapping, and validate a sample before resuming.","trajectory_4":"Force every row into the old schema without checking field meaning."}}},"clm":{"predicted":"trajectory_3","correct":true,"score":0.922396,"confidence":0.922396,"probabilities":{"trajectory_1":0.023508901257395104,"trajectory_2":0.03505790383755591,"trajectory_3":0.9223961958195855,"trajectory_4":0.019036999085463354},"latency_ms":39.03,"input_tokens":30,"retries":0},"jev":{"predicted":"trajectory_3","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_4":0.0,"trajectory_2":0.0,"trajectory_3":1.0,"trajectory_1":0.0},"latency_ms":106.84,"input_tokens":412,"retries":0}},{"id":"trajectory-11-0","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_2","state":"Agent task: Send an approved customer email to the intended recipient.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Save a draft and report that the customer received it.","trajectory_2":"Verify the recipient, use the approved template, send once, and record the delivery ID.","trajectory_3":"Send the approved text to the first matching address without verification.","trajectory_4":"Include unrelated private customer records in the message."}}},"clm":{"predicted":"trajectory_1","correct":false,"score":0.046555,"confidence":0.74806,"probabilities":{"trajectory_1":0.7480599186768914,"trajectory_2":0.04655521629272992,"trajectory_3":0.19899118261031692,"trajectory_4":0.006393682420061755},"latency_ms":99.81,"input_tokens":70,"retries":0},"jev":{"predicted":"trajectory_2","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_2":1.0,"trajectory_4":0.0,"trajectory_1":0.0,"trajectory_3":0.0},"latency_ms":156.06,"input_tokens":398,"retries":0}},{"id":"trajectory-11-1","kind":"trajectory_selection","group":null,"file":null,"expected":"trajectory_1","state":"Task to complete: Send an approved customer email to the intended recipient.\nKeep only a trajectory that completes every stated requirement.","questions":{"pick":{"type":"choice","instructions":"Which completed trajectory should we keep?","criteria":{"trajectory_1":"Verify the recipient, use the approved template, send once, and record the delivery ID.","trajectory_2":"Send the approved text to the first matching address without verification.","trajectory_3":"Save a draft and report that the customer received it.","trajectory_4":"Include unrelated private customer records in the message."}}},"clm":{"predicted":"trajectory_2","correct":false,"score":0.093358,"confidence":0.605598,"probabilities":{"trajectory_1":0.09335760684106707,"trajectory_2":0.6055983277310761,"trajectory_3":0.29588343245467175,"trajectory_4":0.005160632973185021},"latency_ms":34.42,"input_tokens":31,"retries":0},"jev":{"predicted":"trajectory_1","correct":true,"score":1.0,"confidence":1.0,"probabilities":{"trajectory_3":0.0,"trajectory_1":1.0,"trajectory_4":0.0,"trajectory_2":0.0},"latency_ms":103.63,"input_tokens":409,"retries":0}},{"id":"result-00-0","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Current task: Find why checkout requests return HTTP 500.\nTool result: A production stack trace points to checkout/handler.py:117 and a null tax object.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":true,"correct":true,"score":0.513645,"confidence":0.513645,"probabilities":null,"latency_ms":89.82,"input_tokens":86,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.97,"confidence":0.97,"probabilities":null,"latency_ms":153.86,"input_tokens":315,"retries":0}},{"id":"result-00-1","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Tool result: A production stack trace points to checkout/handler.py:117 and a null tax object.\nCurrent task: Find why checkout requests return HTTP 500.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.160905,"confidence":0.839095,"probabilities":null,"latency_ms":34.64,"input_tokens":59,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.96,"confidence":0.96,"probabilities":null,"latency_ms":114.24,"input_tokens":326,"retries":0}},{"id":"result-01-0","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Current task: Find why checkout requests return HTTP 500.\nTool result: A log line shows a user's unrelated GET /favicon.ico returned HTTP 404.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.04025,"confidence":0.95975,"probabilities":null,"latency_ms":40.35,"input_tokens":47,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.04,"confidence":0.96,"probabilities":null,"latency_ms":173.17,"input_tokens":313,"retries":0}},{"id":"result-01-1","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Tool result: A log line shows a user's unrelated GET /favicon.ico returned HTTP 404.\nCurrent task: Find why checkout requests return HTTP 500.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.056005,"confidence":0.943995,"probabilities":null,"latency_ms":34.76,"input_tokens":58,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.04,"confidence":0.96,"probabilities":null,"latency_ms":120.29,"input_tokens":324,"retries":0}},{"id":"result-02-0","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Current task: Verify whether invoice 8842 was charged twice.\nTool result: Two settled payment rows share invoice_id 8842 and have distinct processor charge IDs.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.489748,"confidence":0.510252,"probabilities":null,"latency_ms":34.28,"input_tokens":48,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.98,"confidence":0.98,"probabilities":null,"latency_ms":115.57,"input_tokens":314,"retries":0}},{"id":"result-02-1","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Tool result: Two settled payment rows share invoice_id 8842 and have distinct processor charge IDs.\nCurrent task: Verify whether invoice 8842 was charged twice.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.171095,"confidence":0.828905,"probabilities":null,"latency_ms":35.92,"input_tokens":59,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.97,"confidence":0.97,"probabilities":null,"latency_ms":97.01,"input_tokens":325,"retries":0}},{"id":"result-03-0","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Current task: Verify whether invoice 8842 was charged twice.\nTool result: Two payment rows belong to invoices 8842 and 8843 for different subscriptions.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.187209,"confidence":0.812791,"probabilities":null,"latency_ms":34.66,"input_tokens":50,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.88,"confidence":0.88,"probabilities":null,"latency_ms":101.12,"input_tokens":316,"retries":0}},{"id":"result-03-1","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Tool result: Two payment rows belong to invoices 8842 and 8843 for different subscriptions.\nCurrent task: Verify whether invoice 8842 was charged twice.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.083293,"confidence":0.916707,"probabilities":null,"latency_ms":35.19,"input_tokens":61,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.84,"confidence":0.84,"probabilities":null,"latency_ms":125.27,"input_tokens":327,"retries":0}},{"id":"result-04-0","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Current task: Check whether clm-eval.example.com serves the deployed site.\nTool result: An HTTPS GET to clm-eval.example.com returned 200 with the expected page title.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":true,"correct":true,"score":0.511444,"confidence":0.511444,"probabilities":null,"latency_ms":35.04,"input_tokens":51,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.98,"confidence":0.98,"probabilities":null,"latency_ms":120.24,"input_tokens":317,"retries":0}},{"id":"result-04-1","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Tool result: An HTTPS GET to clm-eval.example.com returned 200 with the expected page title.\nCurrent task: Check whether clm-eval.example.com serves the deployed site.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.266394,"confidence":0.733606,"probabilities":null,"latency_ms":36.29,"input_tokens":62,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.97,"confidence":0.97,"probabilities":null,"latency_ms":111.35,"input_tokens":328,"retries":0}},{"id":"result-05-0","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Current task: Check whether clm-eval.example.com serves the deployed site.\nTool result: A different domain, blog.example.com, has a valid certificate.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.125656,"confidence":0.874344,"probabilities":null,"latency_ms":33.89,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.26,"confidence":0.74,"probabilities":null,"latency_ms":123.15,"input_tokens":309,"retries":0}},{"id":"result-05-1","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Tool result: A different domain, blog.example.com, has a valid certificate.\nCurrent task: Check whether clm-eval.example.com serves the deployed site.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.083429,"confidence":0.916571,"probabilities":null,"latency_ms":36.25,"input_tokens":54,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.11,"confidence":0.89,"probabilities":null,"latency_ms":117.73,"input_tokens":320,"retries":0}},{"id":"result-06-0","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Current task: Locate the cache-eviction implementation.\nTool result: Search found evict_lru() in src/storage/cache.py with the eviction branch.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.440288,"confidence":0.559712,"probabilities":null,"latency_ms":34.72,"input_tokens":41,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.97,"confidence":0.97,"probabilities":null,"latency_ms":99.96,"input_tokens":309,"retries":0}},{"id":"result-06-1","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Tool result: Search found evict_lru() in src/storage/cache.py with the eviction branch.\nCurrent task: Locate the cache-eviction implementation.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.256957,"confidence":0.743043,"probabilities":null,"latency_ms":36.32,"input_tokens":52,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.97,"confidence":0.97,"probabilities":null,"latency_ms":153.74,"input_tokens":320,"retries":0}},{"id":"result-07-0","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Current task: Locate the cache-eviction implementation.\nTool result: A marketing changelog mentions a faster cache, without code or paths.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.043887,"confidence":0.956113,"probabilities":null,"latency_ms":44.14,"input_tokens":38,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.11,"confidence":0.89,"probabilities":null,"latency_ms":116.03,"input_tokens":306,"retries":0}},{"id":"result-07-1","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Tool result: A marketing changelog mentions a faster cache, without code or paths.\nCurrent task: Locate the cache-eviction implementation.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.06548,"confidence":0.93452,"probabilities":null,"latency_ms":36.57,"input_tokens":49,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.14,"confidence":0.86,"probabilities":null,"latency_ms":104.54,"input_tokens":317,"retries":0}},{"id":"result-08-0","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Current task: Identify the failing billing unit test.\nTool result: Pytest reports test_refund_idempotency failed at tests/test_billing.py:54.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.41095,"confidence":0.58905,"probabilities":null,"latency_ms":36.2,"input_tokens":44,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.98,"confidence":0.98,"probabilities":null,"latency_ms":97.01,"input_tokens":312,"retries":0}},{"id":"result-08-1","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Tool result: Pytest reports test_refund_idempotency failed at tests/test_billing.py:54.\nCurrent task: Identify the failing billing unit test.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.245042,"confidence":0.754958,"probabilities":null,"latency_ms":42.74,"input_tokens":55,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.97,"confidence":0.97,"probabilities":null,"latency_ms":104.36,"input_tokens":323,"retries":0}},{"id":"result-09-0","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Current task: Identify the failing billing unit test.\nTool result: The frontend lint job passed; it did not run any billing tests.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.190464,"confidence":0.809536,"probabilities":null,"latency_ms":36.61,"input_tokens":38,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.34,"confidence":0.66,"probabilities":null,"latency_ms":113.13,"input_tokens":304,"retries":0}},{"id":"result-09-1","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Tool result: The frontend lint job passed; it did not run any billing tests.\nCurrent task: Identify the failing billing unit test.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.128788,"confidence":0.871212,"probabilities":null,"latency_ms":47.63,"input_tokens":49,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.48,"confidence":0.52,"probabilities":null,"latency_ms":109.78,"input_tokens":315,"retries":0}},{"id":"result-10-0","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Current task: Diagnose a database migration schema mismatch.\nTool result: The migration dry run says column legacy_id is missing in migration 042.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.447833,"confidence":0.552167,"probabilities":null,"latency_ms":35.29,"input_tokens":43,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.97,"confidence":0.97,"probabilities":null,"latency_ms":107.56,"input_tokens":307,"retries":0}},{"id":"result-10-1","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Tool result: The migration dry run says column legacy_id is missing in migration 042.\nCurrent task: Diagnose a database migration schema mismatch.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.18665,"confidence":0.81335,"probabilities":null,"latency_ms":37.09,"input_tokens":54,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.97,"confidence":0.97,"probabilities":null,"latency_ms":108.44,"input_tokens":318,"retries":0}},{"id":"result-11-0","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Current task: Diagnose a database migration schema mismatch.\nTool result: A dashboard displays the current customer count, with no schema details.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.110125,"confidence":0.889875,"probabilities":null,"latency_ms":35.91,"input_tokens":39,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.08,"confidence":0.92,"probabilities":null,"latency_ms":133.79,"input_tokens":303,"retries":0}},{"id":"result-11-1","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Tool result: A dashboard displays the current customer count, with no schema details.\nCurrent task: Diagnose a database migration schema mismatch.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.070837,"confidence":0.929163,"probabilities":null,"latency_ms":36.27,"input_tokens":50,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.08,"confidence":0.92,"probabilities":null,"latency_ms":124.58,"input_tokens":314,"retries":0}},{"id":"result-12-0","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Current task: Find the route's authorization check.\nTool result: Code search shows authorize(role) called by the target route handler.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.446608,"confidence":0.553392,"probabilities":null,"latency_ms":36.53,"input_tokens":37,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.97,"confidence":0.97,"probabilities":null,"latency_ms":107.47,"input_tokens":303,"retries":0}},{"id":"result-12-1","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Tool result: Code search shows authorize(role) called by the target route handler.\nCurrent task: Find the route's authorization check.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.185558,"confidence":0.814442,"probabilities":null,"latency_ms":34.84,"input_tokens":48,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.97,"confidence":0.97,"probabilities":null,"latency_ms":138.32,"input_tokens":314,"retries":0}},{"id":"result-13-0","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Current task: Find the route's authorization check.\nTool result: Search found the login page CSS selector .authorize-button.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.243295,"confidence":0.756705,"probabilities":null,"latency_ms":35.73,"input_tokens":35,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.42,"confidence":0.58,"probabilities":null,"latency_ms":96.8,"input_tokens":301,"retries":0}},{"id":"result-13-1","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Tool result: Search found the login page CSS selector .authorize-button.\nCurrent task: Find the route's authorization check.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.105617,"confidence":0.894383,"probabilities":null,"latency_ms":51.97,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.24,"confidence":0.76,"probabilities":null,"latency_ms":105.74,"input_tokens":312,"retries":0}},{"id":"result-14-0","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Current task: Assess whether the search index is current.\nTool result: Indexer status shows last indexed event at 10:42 and the newest event at 10:44.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.339678,"confidence":0.660322,"probabilities":null,"latency_ms":41.44,"input_tokens":51,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.98,"confidence":0.98,"probabilities":null,"latency_ms":117.22,"input_tokens":318,"retries":0}},{"id":"result-14-1","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Tool result: Indexer status shows last indexed event at 10:42 and the newest event at 10:44.\nCurrent task: Assess whether the search index is current.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.12096,"confidence":0.87904,"probabilities":null,"latency_ms":37.35,"input_tokens":62,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.97,"confidence":0.97,"probabilities":null,"latency_ms":144.82,"input_tokens":329,"retries":0}},{"id":"result-15-0","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Current task: Assess whether the search index is current.\nTool result: Search configuration lists ranking weights but no indexing timestamps.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.198601,"confidence":0.801399,"probabilities":null,"latency_ms":37.0,"input_tokens":35,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.6,"confidence":0.6,"probabilities":null,"latency_ms":143.2,"input_tokens":301,"retries":0}},{"id":"result-15-1","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Tool result: Search configuration lists ranking weights but no indexing timestamps.\nCurrent task: Assess whether the search index is current.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.112225,"confidence":0.887775,"probabilities":null,"latency_ms":48.3,"input_tokens":46,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.69,"confidence":0.69,"probabilities":null,"latency_ms":114.44,"input_tokens":312,"retries":0}},{"id":"result-16-0","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Current task: Verify an approved customer email was delivered.\nTool result: The mail provider reports delivery ID m-883 delivered to the verified recipient.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.485089,"confidence":0.514911,"probabilities":null,"latency_ms":37.13,"input_tokens":42,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.98,"confidence":0.98,"probabilities":null,"latency_ms":117.87,"input_tokens":308,"retries":0}},{"id":"result-16-1","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Tool result: The mail provider reports delivery ID m-883 delivered to the verified recipient.\nCurrent task: Verify an approved customer email was delivered.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.130887,"confidence":0.869113,"probabilities":null,"latency_ms":36.32,"input_tokens":53,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.97,"confidence":0.97,"probabilities":null,"latency_ms":192.46,"input_tokens":319,"retries":0}},{"id":"result-17-0","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Current task: Verify an approved customer email was delivered.\nTool result: A local draft file contains the approved message, with no send event.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.211393,"confidence":0.788607,"probabilities":null,"latency_ms":35.63,"input_tokens":39,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.78,"confidence":0.78,"probabilities":null,"latency_ms":111.62,"input_tokens":305,"retries":0}},{"id":"result-17-1","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Tool result: A local draft file contains the approved message, with no send event.\nCurrent task: Verify an approved customer email was delivered.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.061272,"confidence":0.938728,"probabilities":null,"latency_ms":35.76,"input_tokens":50,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.86,"confidence":0.86,"probabilities":null,"latency_ms":116.38,"input_tokens":316,"retries":0}},{"id":"result-18-0","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Current task: Check whether a supplied PDF parsed fully.\nTool result: Parser reports 24 of 24 pages processed and zero parse errors.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.296178,"confidence":0.703822,"probabilities":null,"latency_ms":37.22,"input_tokens":41,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.98,"confidence":0.98,"probabilities":null,"latency_ms":106.78,"input_tokens":307,"retries":0}},{"id":"result-18-1","kind":"result_relevance","group":null,"file":null,"expected":true,"state":"Tool result: Parser reports 24 of 24 pages processed and zero parse errors.\nCurrent task: Check whether a supplied PDF parsed fully.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.151798,"confidence":0.848202,"probabilities":null,"latency_ms":51.72,"input_tokens":52,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.97,"confidence":0.97,"probabilities":null,"latency_ms":174.37,"input_tokens":318,"retries":0}},{"id":"result-19-0","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Current task: Check whether a supplied PDF parsed fully.\nTool result: The generated thumbnail is blue and 800 pixels wide.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.058089,"confidence":0.941911,"probabilities":null,"latency_ms":36.52,"input_tokens":38,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.13,"confidence":0.87,"probabilities":null,"latency_ms":101.54,"input_tokens":304,"retries":0}},{"id":"result-19-1","kind":"result_relevance","group":null,"file":null,"expected":false,"state":"Tool result: The generated thumbnail is blue and 800 pixels wide.\nCurrent task: Check whether a supplied PDF parsed fully.\nAn unrelated status note says the office printer is offline.","questions":{"check":{"type":"noul","instructions":"Is this tool result directly relevant to the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.085717,"confidence":0.914283,"probabilities":null,"latency_ms":36.55,"input_tokens":49,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.03,"confidence":0.97,"probabilities":null,"latency_ms":115.64,"input_tokens":315,"retries":0}},{"id":"subgoal-00-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Subgoal: All tests pass on the current commit.\nObserved evidence: Focused tests and the complete test suite passed on the current commit.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":true,"correct":true,"score":0.56407,"confidence":0.56407,"probabilities":null,"latency_ms":107.18,"input_tokens":81,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.95,"confidence":0.95,"probabilities":null,"latency_ms":99.38,"input_tokens":311,"retries":0}},{"id":"subgoal-00-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Observed evidence: Focused tests and the complete test suite passed on the current commit.\nSubgoal to check: All tests pass on the current commit.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":false,"score":0.335572,"confidence":0.664428,"probabilities":null,"latency_ms":38.54,"input_tokens":52,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.94,"confidence":0.94,"probabilities":null,"latency_ms":112.3,"input_tokens":322,"retries":0}},{"id":"subgoal-01-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Subgoal: All tests pass on the current commit.\nObserved evidence: Only the focused unit test passed; the full suite was not run.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.077395,"confidence":0.922605,"probabilities":null,"latency_ms":50.06,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.05,"confidence":0.95,"probabilities":null,"latency_ms":107.86,"input_tokens":310,"retries":0}},{"id":"subgoal-01-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Observed evidence: Only the focused unit test passed; the full suite was not run.\nSubgoal to check: All tests pass on the current commit.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.055272,"confidence":0.944728,"probabilities":null,"latency_ms":37.44,"input_tokens":52,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.03,"confidence":0.97,"probabilities":null,"latency_ms":133.13,"input_tokens":321,"retries":0}},{"id":"subgoal-02-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Subgoal: The custom domain serves the site over HTTPS.\nObserved evidence: An HTTPS GET to the custom domain returned 200 and a valid certificate.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":true,"correct":true,"score":0.511422,"confidence":0.511422,"probabilities":null,"latency_ms":37.43,"input_tokens":45,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.96,"confidence":0.96,"probabilities":null,"latency_ms":111.13,"input_tokens":314,"retries":0}},{"id":"subgoal-02-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Observed evidence: An HTTPS GET to the custom domain returned 200 and a valid certificate.\nSubgoal to check: The custom domain serves the site over HTTPS.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":false,"score":0.376376,"confidence":0.623624,"probabilities":null,"latency_ms":36.8,"input_tokens":56,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.96,"confidence":0.96,"probabilities":null,"latency_ms":154.36,"input_tokens":325,"retries":0}},{"id":"subgoal-03-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Subgoal: The custom domain serves the site over HTTPS.\nObserved evidence: The pages.dev preview works; the custom domain still returns NXDOMAIN.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.11622,"confidence":0.88378,"probabilities":null,"latency_ms":35.52,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.02,"confidence":0.98,"probabilities":null,"latency_ms":222.86,"input_tokens":312,"retries":0}},{"id":"subgoal-03-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Observed evidence: The pages.dev preview works; the custom domain still returns NXDOMAIN.\nSubgoal to check: The custom domain serves the site over HTTPS.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.067714,"confidence":0.932286,"probabilities":null,"latency_ms":50.1,"input_tokens":53,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.05,"confidence":0.95,"probabilities":null,"latency_ms":108.96,"input_tokens":323,"retries":0}},{"id":"subgoal-04-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Subgoal: The verified duplicate invoice has been refunded.\nObserved evidence: The processor returned a settled refund ID for the verified duplicate charge.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":false,"score":0.339886,"confidence":0.660114,"probabilities":null,"latency_ms":37.36,"input_tokens":40,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.83,"confidence":0.83,"probabilities":null,"latency_ms":155.43,"input_tokens":309,"retries":0}},{"id":"subgoal-04-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Observed evidence: The processor returned a settled refund ID for the verified duplicate charge.\nSubgoal to check: The verified duplicate invoice has been refunded.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":false,"score":0.309713,"confidence":0.690287,"probabilities":null,"latency_ms":37.79,"input_tokens":51,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.82,"confidence":0.82,"probabilities":null,"latency_ms":109.04,"input_tokens":320,"retries":0}},{"id":"subgoal-05-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Subgoal: The verified duplicate invoice has been refunded.\nObserved evidence: The refund request was submitted, but its status is still pending.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.048444,"confidence":0.951556,"probabilities":null,"latency_ms":37.44,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.03,"confidence":0.97,"probabilities":null,"latency_ms":128.91,"input_tokens":309,"retries":0}},{"id":"subgoal-05-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Observed evidence: The refund request was submitted, but its status is still pending.\nSubgoal to check: The verified duplicate invoice has been refunded.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.077624,"confidence":0.922376,"probabilities":null,"latency_ms":37.79,"input_tokens":51,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.05,"confidence":0.95,"probabilities":null,"latency_ms":195.08,"input_tokens":320,"retries":0}},{"id":"subgoal-06-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Subgoal: The changed route works for the affected user role.\nObserved evidence: A test logged in as the affected role and exercised the changed route successfully.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":false,"score":0.405492,"confidence":0.594508,"probabilities":null,"latency_ms":47.84,"input_tokens":44,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.92,"confidence":0.92,"probabilities":null,"latency_ms":105.41,"input_tokens":313,"retries":0}},{"id":"subgoal-06-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Observed evidence: A test logged in as the affected role and exercised the changed route successfully.\nSubgoal to check: The changed route works for the affected user role.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":false,"score":0.299238,"confidence":0.700762,"probabilities":null,"latency_ms":36.82,"input_tokens":55,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.93,"confidence":0.93,"probabilities":null,"latency_ms":112.97,"input_tokens":324,"retries":0}},{"id":"subgoal-07-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Subgoal: The changed route works for the affected user role.\nObserved evidence: The permission code was edited, but no role-specific test or login was performed.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.065226,"confidence":0.934774,"probabilities":null,"latency_ms":36.07,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.05,"confidence":0.95,"probabilities":null,"latency_ms":116.23,"input_tokens":314,"retries":0}},{"id":"subgoal-07-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Observed evidence: The permission code was edited, but no role-specific test or login was performed.\nSubgoal to check: The changed route works for the affected user role.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.079861,"confidence":0.920139,"probabilities":null,"latency_ms":36.68,"input_tokens":56,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.03,"confidence":0.97,"probabilities":null,"latency_ms":107.92,"input_tokens":325,"retries":0}},{"id":"subgoal-08-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Subgoal: The search index includes all events through 10:44.\nObserved evidence: The indexer reports last event 10:44 and matching source and index counts.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":false,"score":0.490268,"confidence":0.509732,"probabilities":null,"latency_ms":35.75,"input_tokens":51,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.94,"confidence":0.94,"probabilities":null,"latency_ms":117.67,"input_tokens":321,"retries":0}},{"id":"subgoal-08-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Observed evidence: The indexer reports last event 10:44 and matching source and index counts.\nSubgoal to check: The search index includes all events through 10:44.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":false,"score":0.330811,"confidence":0.669189,"probabilities":null,"latency_ms":35.85,"input_tokens":62,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.72,"confidence":0.72,"probabilities":null,"latency_ms":137.01,"input_tokens":332,"retries":0}},{"id":"subgoal-09-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Subgoal: The search index includes all events through 10:44.\nObserved evidence: An indexing job started at 10:44 but has not finished.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.092085,"confidence":0.907915,"probabilities":null,"latency_ms":34.86,"input_tokens":49,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.14,"confidence":0.86,"probabilities":null,"latency_ms":139.75,"input_tokens":318,"retries":0}},{"id":"subgoal-09-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Observed evidence: An indexing job started at 10:44 but has not finished.\nSubgoal to check: The search index includes all events through 10:44.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.156856,"confidence":0.843144,"probabilities":null,"latency_ms":37.75,"input_tokens":60,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.07,"confidence":0.93,"probabilities":null,"latency_ms":107.02,"input_tokens":329,"retries":0}},{"id":"subgoal-10-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Subgoal: Every page of the supplied PDF was parsed.\nObserved evidence: The PDF has 24 pages and the parser reports 24 completed pages with no errors.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":true,"correct":true,"score":0.74202,"confidence":0.74202,"probabilities":null,"latency_ms":45.4,"input_tokens":48,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.97,"confidence":0.97,"probabilities":null,"latency_ms":193.71,"input_tokens":317,"retries":0}},{"id":"subgoal-10-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Observed evidence: The PDF has 24 pages and the parser reports 24 completed pages with no errors.\nSubgoal to check: Every page of the supplied PDF was parsed.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":true,"correct":true,"score":0.627158,"confidence":0.627158,"probabilities":null,"latency_ms":38.38,"input_tokens":59,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.94,"confidence":0.94,"probabilities":null,"latency_ms":128.95,"input_tokens":328,"retries":0}},{"id":"subgoal-11-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Subgoal: Every page of the supplied PDF was parsed.\nObserved evidence: The parser returned text from page 1, but the remaining 23 pages were skipped.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.07804,"confidence":0.92196,"probabilities":null,"latency_ms":38.51,"input_tokens":47,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.01,"confidence":0.99,"probabilities":null,"latency_ms":102.01,"input_tokens":316,"retries":0}},{"id":"subgoal-11-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Observed evidence: The parser returned text from page 1, but the remaining 23 pages were skipped.\nSubgoal to check: Every page of the supplied PDF was parsed.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.083059,"confidence":0.916941,"probabilities":null,"latency_ms":36.53,"input_tokens":58,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.02,"confidence":0.98,"probabilities":null,"latency_ms":166.39,"input_tokens":327,"retries":0}},{"id":"subgoal-12-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Subgoal: A tracked issue with reproduction steps exists.\nObserved evidence: Issue 482 was created and its body includes the reproduction steps.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":true,"correct":true,"score":0.553955,"confidence":0.553955,"probabilities":null,"latency_ms":38.22,"input_tokens":42,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.95,"confidence":0.95,"probabilities":null,"latency_ms":101.5,"input_tokens":311,"retries":0}},{"id":"subgoal-12-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Observed evidence: Issue 482 was created and its body includes the reproduction steps.\nSubgoal to check: A tracked issue with reproduction steps exists.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":true,"correct":true,"score":0.56877,"confidence":0.56877,"probabilities":null,"latency_ms":48.08,"input_tokens":53,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.96,"confidence":0.96,"probabilities":null,"latency_ms":104.67,"input_tokens":322,"retries":0}},{"id":"subgoal-13-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Subgoal: A tracked issue with reproduction steps exists.\nObserved evidence: A draft issue body was written locally but never submitted.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.179926,"confidence":0.820074,"probabilities":null,"latency_ms":36.68,"input_tokens":38,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.04,"confidence":0.96,"probabilities":null,"latency_ms":125.52,"input_tokens":307,"retries":0}},{"id":"subgoal-13-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Observed evidence: A draft issue body was written locally but never submitted.\nSubgoal to check: A tracked issue with reproduction steps exists.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.072674,"confidence":0.927326,"probabilities":null,"latency_ms":42.1,"input_tokens":49,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.04,"confidence":0.96,"probabilities":null,"latency_ms":101.73,"input_tokens":318,"retries":0}},{"id":"subgoal-14-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Subgoal: The backup can be restored.\nObserved evidence: The snapshot checksum matched and an isolated test restore passed validation.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":false,"score":0.431272,"confidence":0.568728,"probabilities":null,"latency_ms":37.2,"input_tokens":37,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.92,"confidence":0.92,"probabilities":null,"latency_ms":108.12,"input_tokens":306,"retries":0}},{"id":"subgoal-14-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Observed evidence: The snapshot checksum matched and an isolated test restore passed validation.\nSubgoal to check: The backup can be restored.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":false,"score":0.402838,"confidence":0.597162,"probabilities":null,"latency_ms":36.92,"input_tokens":48,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.92,"confidence":0.92,"probabilities":null,"latency_ms":113.39,"input_tokens":317,"retries":0}},{"id":"subgoal-15-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Subgoal: The backup can be restored.\nObserved evidence: A snapshot file exists, but no restore was attempted.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.076359,"confidence":0.923641,"probabilities":null,"latency_ms":46.73,"input_tokens":36,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.04,"confidence":0.96,"probabilities":null,"latency_ms":115.98,"input_tokens":305,"retries":0}},{"id":"subgoal-15-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Observed evidence: A snapshot file exists, but no restore was attempted.\nSubgoal to check: The backup can be restored.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.109672,"confidence":0.890328,"probabilities":null,"latency_ms":39.26,"input_tokens":47,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.04,"confidence":0.96,"probabilities":null,"latency_ms":126.42,"input_tokens":316,"retries":0}},{"id":"subgoal-16-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Subgoal: The queue has no pending or failed tasks.\nObserved evidence: A fresh queue query returned pending=0 and failed=0.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":true,"correct":true,"score":0.738153,"confidence":0.738153,"probabilities":null,"latency_ms":36.19,"input_tokens":41,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.96,"confidence":0.96,"probabilities":null,"latency_ms":96.26,"input_tokens":310,"retries":0}},{"id":"subgoal-16-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Observed evidence: A fresh queue query returned pending=0 and failed=0.\nSubgoal to check: The queue has no pending or failed tasks.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":false,"score":0.494262,"confidence":0.505738,"probabilities":null,"latency_ms":41.16,"input_tokens":52,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.97,"confidence":0.97,"probabilities":null,"latency_ms":121.59,"input_tokens":321,"retries":0}},{"id":"subgoal-17-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Subgoal: The queue has no pending or failed tasks.\nObserved evidence: Workers are processing items; the queue still shows two failed tasks.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.047413,"confidence":0.952587,"probabilities":null,"latency_ms":41.98,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.02,"confidence":0.98,"probabilities":null,"latency_ms":136.49,"input_tokens":310,"retries":0}},{"id":"subgoal-17-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Observed evidence: Workers are processing items; the queue still shows two failed tasks.\nSubgoal to check: The queue has no pending or failed tasks.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.054892,"confidence":0.945108,"probabilities":null,"latency_ms":41.69,"input_tokens":52,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.02,"confidence":0.98,"probabilities":null,"latency_ms":208.58,"input_tokens":321,"retries":0}},{"id":"subgoal-18-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Subgoal: The approved email reached the verified recipient.\nObserved evidence: Provider delivery receipt matches the verified recipient and approved message ID.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":true,"correct":true,"score":0.606561,"confidence":0.606561,"probabilities":null,"latency_ms":38.88,"input_tokens":39,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.92,"confidence":0.92,"probabilities":null,"latency_ms":113.41,"input_tokens":308,"retries":0}},{"id":"subgoal-18-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":true,"state":"Observed evidence: Provider delivery receipt matches the verified recipient and approved message ID.\nSubgoal to check: The approved email reached the verified recipient.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":false,"score":0.394218,"confidence":0.605782,"probabilities":null,"latency_ms":38.67,"input_tokens":50,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.86,"confidence":0.86,"probabilities":null,"latency_ms":148.83,"input_tokens":319,"retries":0}},{"id":"subgoal-19-0","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Subgoal: The approved email reached the verified recipient.\nObserved evidence: The send API accepted the request, but no delivery receipt exists yet.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.074232,"confidence":0.925768,"probabilities":null,"latency_ms":59.39,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.06,"confidence":0.94,"probabilities":null,"latency_ms":97.59,"input_tokens":310,"retries":0}},{"id":"subgoal-19-1","kind":"subgoal_satisfaction","group":null,"file":null,"expected":false,"state":"Observed evidence: The send API accepted the request, but no delivery receipt exists yet.\nSubgoal to check: The approved email reached the verified recipient.\nDo not assume unobserved steps happened.","questions":{"check":{"type":"noul","instructions":"Has this exact subgoal been satisfied by the observed evidence?"}},"clm":{"predicted":false,"correct":true,"score":0.083616,"confidence":0.916384,"probabilities":null,"latency_ms":35.9,"input_tokens":52,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.06,"confidence":0.94,"probabilities":null,"latency_ms":107.75,"input_tokens":321,"retries":0}},{"id":"file-00-search/query.py","kind":"file_context","group":"file-00","file":"search/query.py","expected":false,"state":"Agent task: Sessions expire immediately after a token refresh. Find the likely implementation.\nSearch hit: search/query.py — Ranks and filters search results at query time.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.330746,"confidence":0.669254,"probabilities":null,"latency_ms":94.32,"input_tokens":82,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.09,"confidence":0.91,"probabilities":null,"latency_ms":115.75,"input_tokens":310,"retries":0}},{"id":"file-00-ops/metrics.py","kind":"file_context","group":"file-00","file":"ops/metrics.py","expected":false,"state":"Agent task: Sessions expire immediately after a token refresh. Find the likely implementation.\nSearch hit: ops/metrics.py — Records service counters and dashboard metrics.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.217913,"confidence":0.782087,"probabilities":null,"latency_ms":36.27,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.09,"confidence":0.91,"probabilities":null,"latency_ms":109.19,"input_tokens":308,"retries":0}},{"id":"file-00-auth/session.py","kind":"file_context","group":"file-00","file":"auth/session.py","expected":true,"state":"Agent task: Sessions expire immediately after a token refresh. Find the likely implementation.\nSearch hit: auth/session.py — Creates, refreshes, and expires login sessions and tokens.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":true,"correct":true,"score":0.719965,"confidence":0.719965,"probabilities":null,"latency_ms":34.34,"input_tokens":46,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.94,"confidence":0.94,"probabilities":null,"latency_ms":112.33,"input_tokens":313,"retries":0}},{"id":"file-00-auth/permissions.py","kind":"file_context","group":"file-00","file":"auth/permissions.py","expected":false,"state":"Agent task: Sessions expire immediately after a token refresh. Find the likely implementation.\nSearch hit: auth/permissions.py — Checks role membership and resource permissions.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.174561,"confidence":0.825439,"probabilities":null,"latency_ms":34.31,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.15,"confidence":0.85,"probabilities":null,"latency_ms":103.27,"input_tokens":310,"retries":0}},{"id":"file-00-billing/invoices.py","kind":"file_context","group":"file-00","file":"billing/invoices.py","expected":false,"state":"Agent task: Sessions expire immediately after a token refresh. Find the likely implementation.\nSearch hit: billing/invoices.py — Creates renewal invoices and records payment attempts.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.263474,"confidence":0.736526,"probabilities":null,"latency_ms":36.37,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.07,"confidence":0.93,"probabilities":null,"latency_ms":123.73,"input_tokens":310,"retries":0}},{"id":"file-00-storage/migrations.py","kind":"file_context","group":"file-00","file":"storage/migrations.py","expected":false,"state":"Agent task: Sessions expire immediately after a token refresh. Find the likely implementation.\nSearch hit: storage/migrations.py — Runs and validates database schema migrations.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.140351,"confidence":0.859649,"probabilities":null,"latency_ms":34.14,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.1,"confidence":0.9,"probabilities":null,"latency_ms":106.08,"input_tokens":309,"retries":0}},{"id":"file-00-jobs/scheduler.py","kind":"file_context","group":"file-00","file":"jobs/scheduler.py","expected":false,"state":"Agent task: Sessions expire immediately after a token refresh. Find the likely implementation.\nSearch hit: jobs/scheduler.py — Starts scheduled background exports and maintenance jobs.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.269066,"confidence":0.730934,"probabilities":null,"latency_ms":34.06,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.11,"confidence":0.89,"probabilities":null,"latency_ms":115.41,"input_tokens":310,"retries":0}},{"id":"file-00-web/ui.py","kind":"file_context","group":"file-00","file":"web/ui.py","expected":false,"state":"Agent task: Sessions expire immediately after a token refresh. Find the likely implementation.\nSearch hit: web/ui.py — Stores browser UI state and renders panels.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.255489,"confidence":0.744511,"probabilities":null,"latency_ms":34.83,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.2,"confidence":0.8,"probabilities":null,"latency_ms":120.93,"input_tokens":308,"retries":0}},{"id":"file-00-api/routes.py","kind":"file_context","group":"file-00","file":"api/routes.py","expected":false,"state":"Agent task: Sessions expire immediately after a token refresh. Find the likely implementation.\nSearch hit: api/routes.py — Maps HTTP paths to handlers and emits endpoint events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.347682,"confidence":0.652318,"probabilities":null,"latency_ms":33.36,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.25,"confidence":0.75,"probabilities":null,"latency_ms":108.39,"input_tokens":310,"retries":0}},{"id":"file-00-billing/refunds.py","kind":"file_context","group":"file-00","file":"billing/refunds.py","expected":false,"state":"Agent task: Sessions expire immediately after a token refresh. Find the likely implementation.\nSearch hit: billing/refunds.py — Processes refund requests and payment-provider callbacks.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.186034,"confidence":0.813966,"probabilities":null,"latency_ms":34.56,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.09,"confidence":0.91,"probabilities":null,"latency_ms":96.86,"input_tokens":309,"retries":0}},{"id":"file-00-api/rate_limit.py","kind":"file_context","group":"file-00","file":"api/rate_limit.py","expected":false,"state":"Agent task: Sessions expire immediately after a token refresh. Find the likely implementation.\nSearch hit: api/rate_limit.py — Applies per-client throttling and 429 responses.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.175169,"confidence":0.824831,"probabilities":null,"latency_ms":34.57,"input_tokens":48,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.1,"confidence":0.9,"probabilities":null,"latency_ms":99.64,"input_tokens":315,"retries":0}},{"id":"file-00-search/index.py","kind":"file_context","group":"file-00","file":"search/index.py","expected":false,"state":"Agent task: Sessions expire immediately after a token refresh. Find the likely implementation.\nSearch hit: search/index.py — Updates the search index from change events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.259521,"confidence":0.740479,"probabilities":null,"latency_ms":34.4,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.07,"confidence":0.93,"probabilities":null,"latency_ms":114.67,"input_tokens":308,"retries":0}},{"id":"file-00-storage/cache.py","kind":"file_context","group":"file-00","file":"storage/cache.py","expected":false,"state":"Agent task: Sessions expire immediately after a token refresh. Find the likely implementation.\nSearch hit: storage/cache.py — Caches reads and evicts old entries.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.413156,"confidence":0.586844,"probabilities":null,"latency_ms":35.03,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.25,"confidence":0.75,"probabilities":null,"latency_ms":106.43,"input_tokens":309,"retries":0}},{"id":"file-00-jobs/retries.py","kind":"file_context","group":"file-00","file":"jobs/retries.py","expected":false,"state":"Agent task: Sessions expire immediately after a token refresh. Find the likely implementation.\nSearch hit: jobs/retries.py — Retries failed asynchronous jobs with backoff.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.189096,"confidence":0.810904,"probabilities":null,"latency_ms":46.15,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.09,"confidence":0.91,"probabilities":null,"latency_ms":100.64,"input_tokens":310,"retries":0}},{"id":"file-00-web/notifications.py","kind":"file_context","group":"file-00","file":"web/notifications.py","expected":false,"state":"Agent task: Sessions expire immediately after a token refresh. Find the likely implementation.\nSearch hit: web/notifications.py — Formats and dispatches user notifications.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.137942,"confidence":0.862058,"probabilities":null,"latency_ms":34.16,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.11,"confidence":0.89,"probabilities":null,"latency_ms":137.54,"input_tokens":310,"retries":0}},{"id":"file-01-web/notifications.py","kind":"file_context","group":"file-01","file":"web/notifications.py","expected":false,"state":"Agent task: A viewer can access an admin-only resource. Find the relevant check.\nSearch hit: web/notifications.py — Formats and dispatches user notifications.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.181043,"confidence":0.818957,"probabilities":null,"latency_ms":42.8,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.17,"confidence":0.83,"probabilities":null,"latency_ms":128.73,"input_tokens":311,"retries":0}},{"id":"file-01-web/ui.py","kind":"file_context","group":"file-01","file":"web/ui.py","expected":false,"state":"Agent task: A viewer can access an admin-only resource. Find the relevant check.\nSearch hit: web/ui.py — Stores browser UI state and renders panels.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.186698,"confidence":0.813302,"probabilities":null,"latency_ms":37.34,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.24,"confidence":0.76,"probabilities":null,"latency_ms":120.59,"input_tokens":309,"retries":0}},{"id":"file-01-ops/metrics.py","kind":"file_context","group":"file-01","file":"ops/metrics.py","expected":false,"state":"Agent task: A viewer can access an admin-only resource. Find the relevant check.\nSearch hit: ops/metrics.py — Records service counters and dashboard metrics.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.312777,"confidence":0.687223,"probabilities":null,"latency_ms":37.53,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.16,"confidence":0.84,"probabilities":null,"latency_ms":143.3,"input_tokens":309,"retries":0}},{"id":"file-01-search/index.py","kind":"file_context","group":"file-01","file":"search/index.py","expected":false,"state":"Agent task: A viewer can access an admin-only resource. Find the relevant check.\nSearch hit: search/index.py — Updates the search index from change events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.19186,"confidence":0.80814,"probabilities":null,"latency_ms":51.38,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.13,"confidence":0.87,"probabilities":null,"latency_ms":147.0,"input_tokens":309,"retries":0}},{"id":"file-01-storage/cache.py","kind":"file_context","group":"file-01","file":"storage/cache.py","expected":false,"state":"Agent task: A viewer can access an admin-only resource. Find the relevant check.\nSearch hit: storage/cache.py — Caches reads and evicts old entries.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.204488,"confidence":0.795512,"probabilities":null,"latency_ms":39.32,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.11,"confidence":0.89,"probabilities":null,"latency_ms":130.47,"input_tokens":310,"retries":0}},{"id":"file-01-auth/session.py","kind":"file_context","group":"file-01","file":"auth/session.py","expected":false,"state":"Agent task: A viewer can access an admin-only resource. Find the relevant check.\nSearch hit: auth/session.py — Creates, refreshes, and expires login sessions and tokens.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.27956,"confidence":0.72044,"probabilities":null,"latency_ms":35.28,"input_tokens":47,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.47,"confidence":0.53,"probabilities":null,"latency_ms":168.29,"input_tokens":314,"retries":0}},{"id":"file-01-api/routes.py","kind":"file_context","group":"file-01","file":"api/routes.py","expected":false,"state":"Agent task: A viewer can access an admin-only resource. Find the relevant check.\nSearch hit: api/routes.py — Maps HTTP paths to handlers and emits endpoint events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.312694,"confidence":0.687306,"probabilities":null,"latency_ms":36.66,"input_tokens":45,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.63,"confidence":0.63,"probabilities":null,"latency_ms":129.17,"input_tokens":311,"retries":0}},{"id":"file-01-billing/refunds.py","kind":"file_context","group":"file-01","file":"billing/refunds.py","expected":false,"state":"Agent task: A viewer can access an admin-only resource. Find the relevant check.\nSearch hit: billing/refunds.py — Processes refund requests and payment-provider callbacks.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.361542,"confidence":0.638458,"probabilities":null,"latency_ms":37.38,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.36,"confidence":0.64,"probabilities":null,"latency_ms":107.26,"input_tokens":310,"retries":0}},{"id":"file-01-jobs/retries.py","kind":"file_context","group":"file-01","file":"jobs/retries.py","expected":false,"state":"Agent task: A viewer can access an admin-only resource. Find the relevant check.\nSearch hit: jobs/retries.py — Retries failed asynchronous jobs with backoff.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.213703,"confidence":0.786297,"probabilities":null,"latency_ms":48.37,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.09,"confidence":0.91,"probabilities":null,"latency_ms":159.54,"input_tokens":311,"retries":0}},{"id":"file-01-api/rate_limit.py","kind":"file_context","group":"file-01","file":"api/rate_limit.py","expected":false,"state":"Agent task: A viewer can access an admin-only resource. Find the relevant check.\nSearch hit: api/rate_limit.py — Applies per-client throttling and 429 responses.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.153591,"confidence":0.846409,"probabilities":null,"latency_ms":34.47,"input_tokens":49,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.12,"confidence":0.88,"probabilities":null,"latency_ms":96.94,"input_tokens":316,"retries":0}},{"id":"file-01-billing/invoices.py","kind":"file_context","group":"file-01","file":"billing/invoices.py","expected":false,"state":"Agent task: A viewer can access an admin-only resource. Find the relevant check.\nSearch hit: billing/invoices.py — Creates renewal invoices and records payment attempts.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.348275,"confidence":0.651725,"probabilities":null,"latency_ms":34.5,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.15,"confidence":0.85,"probabilities":null,"latency_ms":102.12,"input_tokens":311,"retries":0}},{"id":"file-01-storage/migrations.py","kind":"file_context","group":"file-01","file":"storage/migrations.py","expected":false,"state":"Agent task: A viewer can access an admin-only resource. Find the relevant check.\nSearch hit: storage/migrations.py — Runs and validates database schema migrations.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.170958,"confidence":0.829042,"probabilities":null,"latency_ms":32.85,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.17,"confidence":0.83,"probabilities":null,"latency_ms":147.66,"input_tokens":310,"retries":0}},{"id":"file-01-jobs/scheduler.py","kind":"file_context","group":"file-01","file":"jobs/scheduler.py","expected":false,"state":"Agent task: A viewer can access an admin-only resource. Find the relevant check.\nSearch hit: jobs/scheduler.py — Starts scheduled background exports and maintenance jobs.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.399354,"confidence":0.600646,"probabilities":null,"latency_ms":33.87,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.18,"confidence":0.82,"probabilities":null,"latency_ms":114.18,"input_tokens":311,"retries":0}},{"id":"file-01-search/query.py","kind":"file_context","group":"file-01","file":"search/query.py","expected":false,"state":"Agent task: A viewer can access an admin-only resource. Find the relevant check.\nSearch hit: search/query.py — Ranks and filters search results at query time.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.147215,"confidence":0.852785,"probabilities":null,"latency_ms":37.86,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.24,"confidence":0.76,"probabilities":null,"latency_ms":103.0,"input_tokens":311,"retries":0}},{"id":"file-01-auth/permissions.py","kind":"file_context","group":"file-01","file":"auth/permissions.py","expected":true,"state":"Agent task: A viewer can access an admin-only resource. Find the relevant check.\nSearch hit: auth/permissions.py — Checks role membership and resource permissions.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.455769,"confidence":0.544231,"probabilities":null,"latency_ms":45.26,"input_tokens":43,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.93,"confidence":0.93,"probabilities":null,"latency_ms":110.91,"input_tokens":311,"retries":0}},{"id":"file-02-jobs/retries.py","kind":"file_context","group":"file-02","file":"jobs/retries.py","expected":false,"state":"Agent task: Renewal created two invoices for one billing period. Find the invoice code.\nSearch hit: jobs/retries.py — Retries failed asynchronous jobs with backoff.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.112969,"confidence":0.887031,"probabilities":null,"latency_ms":35.04,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.07,"confidence":0.93,"probabilities":null,"latency_ms":109.38,"input_tokens":313,"retries":0}},{"id":"file-02-ops/metrics.py","kind":"file_context","group":"file-02","file":"ops/metrics.py","expected":false,"state":"Agent task: Renewal created two invoices for one billing period. Find the invoice code.\nSearch hit: ops/metrics.py — Records service counters and dashboard metrics.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.138755,"confidence":0.861245,"probabilities":null,"latency_ms":50.33,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.06,"confidence":0.94,"probabilities":null,"latency_ms":139.24,"input_tokens":311,"retries":0}},{"id":"file-02-jobs/scheduler.py","kind":"file_context","group":"file-02","file":"jobs/scheduler.py","expected":false,"state":"Agent task: Renewal created two invoices for one billing period. Find the invoice code.\nSearch hit: jobs/scheduler.py — Starts scheduled background exports and maintenance jobs.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.158221,"confidence":0.841779,"probabilities":null,"latency_ms":33.59,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.08,"confidence":0.92,"probabilities":null,"latency_ms":112.94,"input_tokens":313,"retries":0}},{"id":"file-02-web/notifications.py","kind":"file_context","group":"file-02","file":"web/notifications.py","expected":false,"state":"Agent task: Renewal created two invoices for one billing period. Find the invoice code.\nSearch hit: web/notifications.py — Formats and dispatches user notifications.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.079496,"confidence":0.920504,"probabilities":null,"latency_ms":32.99,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.09,"confidence":0.91,"probabilities":null,"latency_ms":126.09,"input_tokens":313,"retries":0}},{"id":"file-02-api/rate_limit.py","kind":"file_context","group":"file-02","file":"api/rate_limit.py","expected":false,"state":"Agent task: Renewal created two invoices for one billing period. Find the invoice code.\nSearch hit: api/rate_limit.py — Applies per-client throttling and 429 responses.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.092237,"confidence":0.907763,"probabilities":null,"latency_ms":34.16,"input_tokens":50,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.04,"confidence":0.96,"probabilities":null,"latency_ms":111.82,"input_tokens":318,"retries":0}},{"id":"file-02-auth/session.py","kind":"file_context","group":"file-02","file":"auth/session.py","expected":false,"state":"Agent task: Renewal created two invoices for one billing period. Find the invoice code.\nSearch hit: auth/session.py — Creates, refreshes, and expires login sessions and tokens.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.096077,"confidence":0.903923,"probabilities":null,"latency_ms":34.17,"input_tokens":48,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.03,"confidence":0.97,"probabilities":null,"latency_ms":129.24,"input_tokens":316,"retries":0}},{"id":"file-02-billing/refunds.py","kind":"file_context","group":"file-02","file":"billing/refunds.py","expected":false,"state":"Agent task: Renewal created two invoices for one billing period. Find the invoice code.\nSearch hit: billing/refunds.py — Processes refund requests and payment-provider callbacks.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.13,"confidence":0.87,"probabilities":null,"latency_ms":43.67,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.17,"confidence":0.83,"probabilities":null,"latency_ms":114.71,"input_tokens":312,"retries":0}},{"id":"file-02-search/index.py","kind":"file_context","group":"file-02","file":"search/index.py","expected":false,"state":"Agent task: Renewal created two invoices for one billing period. Find the invoice code.\nSearch hit: search/index.py — Updates the search index from change events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.1699,"confidence":0.8301,"probabilities":null,"latency_ms":54.05,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.07,"confidence":0.93,"probabilities":null,"latency_ms":145.75,"input_tokens":311,"retries":0}},{"id":"file-02-storage/migrations.py","kind":"file_context","group":"file-02","file":"storage/migrations.py","expected":false,"state":"Agent task: Renewal created two invoices for one billing period. Find the invoice code.\nSearch hit: storage/migrations.py — Runs and validates database schema migrations.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.12898,"confidence":0.87102,"probabilities":null,"latency_ms":35.48,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.07,"confidence":0.93,"probabilities":null,"latency_ms":112.8,"input_tokens":312,"retries":0}},{"id":"file-02-storage/cache.py","kind":"file_context","group":"file-02","file":"storage/cache.py","expected":false,"state":"Agent task: Renewal created two invoices for one billing period. Find the invoice code.\nSearch hit: storage/cache.py — Caches reads and evicts old entries.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.161597,"confidence":0.838403,"probabilities":null,"latency_ms":37.43,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.06,"confidence":0.94,"probabilities":null,"latency_ms":102.95,"input_tokens":312,"retries":0}},{"id":"file-02-auth/permissions.py","kind":"file_context","group":"file-02","file":"auth/permissions.py","expected":false,"state":"Agent task: Renewal created two invoices for one billing period. Find the invoice code.\nSearch hit: auth/permissions.py — Checks role membership and resource permissions.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.076059,"confidence":0.923941,"probabilities":null,"latency_ms":34.89,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.05,"confidence":0.95,"probabilities":null,"latency_ms":103.01,"input_tokens":313,"retries":0}},{"id":"file-02-billing/invoices.py","kind":"file_context","group":"file-02","file":"billing/invoices.py","expected":true,"state":"Agent task: Renewal created two invoices for one billing period. Find the invoice code.\nSearch hit: billing/invoices.py — Creates renewal invoices and records payment attempts.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.416053,"confidence":0.583947,"probabilities":null,"latency_ms":56.71,"input_tokens":45,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.87,"confidence":0.87,"probabilities":null,"latency_ms":125.09,"input_tokens":313,"retries":0}},{"id":"file-02-web/ui.py","kind":"file_context","group":"file-02","file":"web/ui.py","expected":false,"state":"Agent task: Renewal created two invoices for one billing period. Find the invoice code.\nSearch hit: web/ui.py — Stores browser UI state and renders panels.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.118022,"confidence":0.881978,"probabilities":null,"latency_ms":33.43,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.07,"confidence":0.93,"probabilities":null,"latency_ms":103.47,"input_tokens":311,"retries":0}},{"id":"file-02-search/query.py","kind":"file_context","group":"file-02","file":"search/query.py","expected":false,"state":"Agent task: Renewal created two invoices for one billing period. Find the invoice code.\nSearch hit: search/query.py — Ranks and filters search results at query time.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.20343,"confidence":0.79657,"probabilities":null,"latency_ms":42.23,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.17,"confidence":0.83,"probabilities":null,"latency_ms":108.32,"input_tokens":313,"retries":0}},{"id":"file-02-api/routes.py","kind":"file_context","group":"file-02","file":"api/routes.py","expected":false,"state":"Agent task: Renewal created two invoices for one billing period. Find the invoice code.\nSearch hit: api/routes.py — Maps HTTP paths to handlers and emits endpoint events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.124655,"confidence":0.875345,"probabilities":null,"latency_ms":33.04,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.09,"confidence":0.91,"probabilities":null,"latency_ms":123.06,"input_tokens":313,"retries":0}},{"id":"file-03-auth/session.py","kind":"file_context","group":"file-03","file":"auth/session.py","expected":false,"state":"Agent task: A settled refund never updated after the payment provider callback.\nSearch hit: auth/session.py — Creates, refreshes, and expires login sessions and tokens.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.206844,"confidence":0.793156,"probabilities":null,"latency_ms":39.94,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.07,"confidence":0.93,"probabilities":null,"latency_ms":141.18,"input_tokens":311,"retries":0}},{"id":"file-03-web/ui.py","kind":"file_context","group":"file-03","file":"web/ui.py","expected":false,"state":"Agent task: A settled refund never updated after the payment provider callback.\nSearch hit: web/ui.py — Stores browser UI state and renders panels.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.191779,"confidence":0.808221,"probabilities":null,"latency_ms":44.23,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.15,"confidence":0.85,"probabilities":null,"latency_ms":122.23,"input_tokens":306,"retries":0}},{"id":"file-03-billing/refunds.py","kind":"file_context","group":"file-03","file":"billing/refunds.py","expected":true,"state":"Agent task: A settled refund never updated after the payment provider callback.\nSearch hit: billing/refunds.py — Processes refund requests and payment-provider callbacks.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.471797,"confidence":0.528203,"probabilities":null,"latency_ms":39.43,"input_tokens":41,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.93,"confidence":0.93,"probabilities":null,"latency_ms":115.85,"input_tokens":307,"retries":0}},{"id":"file-03-web/notifications.py","kind":"file_context","group":"file-03","file":"web/notifications.py","expected":false,"state":"Agent task: A settled refund never updated after the payment provider callback.\nSearch hit: web/notifications.py — Formats and dispatches user notifications.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.162295,"confidence":0.837705,"probabilities":null,"latency_ms":35.13,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.22,"confidence":0.78,"probabilities":null,"latency_ms":105.01,"input_tokens":308,"retries":0}},{"id":"file-03-jobs/scheduler.py","kind":"file_context","group":"file-03","file":"jobs/scheduler.py","expected":false,"state":"Agent task: A settled refund never updated after the payment provider callback.\nSearch hit: jobs/scheduler.py — Starts scheduled background exports and maintenance jobs.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.293181,"confidence":0.706819,"probabilities":null,"latency_ms":34.76,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.18,"confidence":0.82,"probabilities":null,"latency_ms":142.2,"input_tokens":308,"retries":0}},{"id":"file-03-api/rate_limit.py","kind":"file_context","group":"file-03","file":"api/rate_limit.py","expected":false,"state":"Agent task: A settled refund never updated after the payment provider callback.\nSearch hit: api/rate_limit.py — Applies per-client throttling and 429 responses.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.127994,"confidence":0.872006,"probabilities":null,"latency_ms":36.11,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.12,"confidence":0.88,"probabilities":null,"latency_ms":147.64,"input_tokens":313,"retries":0}},{"id":"file-03-storage/cache.py","kind":"file_context","group":"file-03","file":"storage/cache.py","expected":false,"state":"Agent task: A settled refund never updated after the payment provider callback.\nSearch hit: storage/cache.py — Caches reads and evicts old entries.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.323681,"confidence":0.676319,"probabilities":null,"latency_ms":38.66,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.24,"confidence":0.76,"probabilities":null,"latency_ms":129.72,"input_tokens":307,"retries":0}},{"id":"file-03-jobs/retries.py","kind":"file_context","group":"file-03","file":"jobs/retries.py","expected":false,"state":"Agent task: A settled refund never updated after the payment provider callback.\nSearch hit: jobs/retries.py — Retries failed asynchronous jobs with backoff.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.413319,"confidence":0.586681,"probabilities":null,"latency_ms":52.14,"input_tokens":42,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.61,"confidence":0.61,"probabilities":null,"latency_ms":101.84,"input_tokens":308,"retries":0}},{"id":"file-03-auth/permissions.py","kind":"file_context","group":"file-03","file":"auth/permissions.py","expected":false,"state":"Agent task: A settled refund never updated after the payment provider callback.\nSearch hit: auth/permissions.py — Checks role membership and resource permissions.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.119772,"confidence":0.880228,"probabilities":null,"latency_ms":36.05,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.1,"confidence":0.9,"probabilities":null,"latency_ms":100.45,"input_tokens":308,"retries":0}},{"id":"file-03-billing/invoices.py","kind":"file_context","group":"file-03","file":"billing/invoices.py","expected":false,"state":"Agent task: A settled refund never updated after the payment provider callback.\nSearch hit: billing/invoices.py — Creates renewal invoices and records payment attempts.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.316206,"confidence":0.683794,"probabilities":null,"latency_ms":40.97,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.4,"confidence":0.6,"probabilities":null,"latency_ms":111.37,"input_tokens":308,"retries":0}},{"id":"file-03-ops/metrics.py","kind":"file_context","group":"file-03","file":"ops/metrics.py","expected":false,"state":"Agent task: A settled refund never updated after the payment provider callback.\nSearch hit: ops/metrics.py — Records service counters and dashboard metrics.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.261581,"confidence":0.738419,"probabilities":null,"latency_ms":35.72,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.14,"confidence":0.86,"probabilities":null,"latency_ms":112.17,"input_tokens":306,"retries":0}},{"id":"file-03-api/routes.py","kind":"file_context","group":"file-03","file":"api/routes.py","expected":false,"state":"Agent task: A settled refund never updated after the payment provider callback.\nSearch hit: api/routes.py — Maps HTTP paths to handlers and emits endpoint events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.301548,"confidence":0.698452,"probabilities":null,"latency_ms":60.22,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.33,"confidence":0.67,"probabilities":null,"latency_ms":120.89,"input_tokens":308,"retries":0}},{"id":"file-03-storage/migrations.py","kind":"file_context","group":"file-03","file":"storage/migrations.py","expected":false,"state":"Agent task: A settled refund never updated after the payment provider callback.\nSearch hit: storage/migrations.py — Runs and validates database schema migrations.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.258727,"confidence":0.741273,"probabilities":null,"latency_ms":35.27,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.17,"confidence":0.83,"probabilities":null,"latency_ms":123.65,"input_tokens":307,"retries":0}},{"id":"file-03-search/index.py","kind":"file_context","group":"file-03","file":"search/index.py","expected":false,"state":"Agent task: A settled refund never updated after the payment provider callback.\nSearch hit: search/index.py — Updates the search index from change events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.212237,"confidence":0.787763,"probabilities":null,"latency_ms":65.31,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.13,"confidence":0.87,"probabilities":null,"latency_ms":125.12,"input_tokens":306,"retries":0}},{"id":"file-03-search/query.py","kind":"file_context","group":"file-03","file":"search/query.py","expected":false,"state":"Agent task: A settled refund never updated after the payment provider callback.\nSearch hit: search/query.py — Ranks and filters search results at query time.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.241966,"confidence":0.758034,"probabilities":null,"latency_ms":33.81,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.09,"confidence":0.91,"probabilities":null,"latency_ms":131.2,"input_tokens":308,"retries":0}},{"id":"file-04-jobs/scheduler.py","kind":"file_context","group":"file-04","file":"jobs/scheduler.py","expected":false,"state":"Agent task: Clients receive 429 responses despite being below the documented limit.\nSearch hit: jobs/scheduler.py — Starts scheduled background exports and maintenance jobs.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.426501,"confidence":0.573499,"probabilities":null,"latency_ms":40.7,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.21,"confidence":0.79,"probabilities":null,"latency_ms":127.42,"input_tokens":311,"retries":0}},{"id":"file-04-storage/migrations.py","kind":"file_context","group":"file-04","file":"storage/migrations.py","expected":false,"state":"Agent task: Clients receive 429 responses despite being below the documented limit.\nSearch hit: storage/migrations.py — Runs and validates database schema migrations.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.17647,"confidence":0.82353,"probabilities":null,"latency_ms":52.18,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.11,"confidence":0.89,"probabilities":null,"latency_ms":127.14,"input_tokens":310,"retries":0}},{"id":"file-04-search/index.py","kind":"file_context","group":"file-04","file":"search/index.py","expected":false,"state":"Agent task: Clients receive 429 responses despite being below the documented limit.\nSearch hit: search/index.py — Updates the search index from change events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.237025,"confidence":0.762975,"probabilities":null,"latency_ms":36.25,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.16,"confidence":0.84,"probabilities":null,"latency_ms":100.18,"input_tokens":309,"retries":0}},{"id":"file-04-ops/metrics.py","kind":"file_context","group":"file-04","file":"ops/metrics.py","expected":false,"state":"Agent task: Clients receive 429 responses despite being below the documented limit.\nSearch hit: ops/metrics.py — Records service counters and dashboard metrics.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.393572,"confidence":0.606428,"probabilities":null,"latency_ms":34.23,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.38,"confidence":0.62,"probabilities":null,"latency_ms":121.23,"input_tokens":309,"retries":0}},{"id":"file-04-billing/refunds.py","kind":"file_context","group":"file-04","file":"billing/refunds.py","expected":false,"state":"Agent task: Clients receive 429 responses despite being below the documented limit.\nSearch hit: billing/refunds.py — Processes refund requests and payment-provider callbacks.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.337532,"confidence":0.662468,"probabilities":null,"latency_ms":34.29,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.17,"confidence":0.83,"probabilities":null,"latency_ms":131.02,"input_tokens":310,"retries":0}},{"id":"file-04-storage/cache.py","kind":"file_context","group":"file-04","file":"storage/cache.py","expected":false,"state":"Agent task: Clients receive 429 responses despite being below the documented limit.\nSearch hit: storage/cache.py — Caches reads and evicts old entries.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.394561,"confidence":0.605439,"probabilities":null,"latency_ms":38.59,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.31,"confidence":0.69,"probabilities":null,"latency_ms":106.93,"input_tokens":310,"retries":0}},{"id":"file-04-api/rate_limit.py","kind":"file_context","group":"file-04","file":"api/rate_limit.py","expected":true,"state":"Agent task: Clients receive 429 responses despite being below the documented limit.\nSearch hit: api/rate_limit.py — Applies per-client throttling and 429 responses.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.470077,"confidence":0.529923,"probabilities":null,"latency_ms":52.63,"input_tokens":49,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.95,"confidence":0.95,"probabilities":null,"latency_ms":109.28,"input_tokens":316,"retries":0}},{"id":"file-04-web/notifications.py","kind":"file_context","group":"file-04","file":"web/notifications.py","expected":false,"state":"Agent task: Clients receive 429 responses despite being below the documented limit.\nSearch hit: web/notifications.py — Formats and dispatches user notifications.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.205782,"confidence":0.794218,"probabilities":null,"latency_ms":39.89,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.14,"confidence":0.86,"probabilities":null,"latency_ms":114.33,"input_tokens":311,"retries":0}},{"id":"file-04-billing/invoices.py","kind":"file_context","group":"file-04","file":"billing/invoices.py","expected":false,"state":"Agent task: Clients receive 429 responses despite being below the documented limit.\nSearch hit: billing/invoices.py — Creates renewal invoices and records payment attempts.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.255846,"confidence":0.744154,"probabilities":null,"latency_ms":32.12,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.14,"confidence":0.86,"probabilities":null,"latency_ms":141.13,"input_tokens":311,"retries":0}},{"id":"file-04-api/routes.py","kind":"file_context","group":"file-04","file":"api/routes.py","expected":false,"state":"Agent task: Clients receive 429 responses despite being below the documented limit.\nSearch hit: api/routes.py — Maps HTTP paths to handlers and emits endpoint events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.315371,"confidence":0.684629,"probabilities":null,"latency_ms":31.79,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.3,"confidence":0.7,"probabilities":null,"latency_ms":101.02,"input_tokens":311,"retries":0}},{"id":"file-04-auth/permissions.py","kind":"file_context","group":"file-04","file":"auth/permissions.py","expected":false,"state":"Agent task: Clients receive 429 responses despite being below the documented limit.\nSearch hit: auth/permissions.py — Checks role membership and resource permissions.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.217752,"confidence":0.782248,"probabilities":null,"latency_ms":33.08,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.21,"confidence":0.79,"probabilities":null,"latency_ms":140.23,"input_tokens":311,"retries":0}},{"id":"file-04-jobs/retries.py","kind":"file_context","group":"file-04","file":"jobs/retries.py","expected":false,"state":"Agent task: Clients receive 429 responses despite being below the documented limit.\nSearch hit: jobs/retries.py — Retries failed asynchronous jobs with backoff.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.308722,"confidence":0.691278,"probabilities":null,"latency_ms":32.05,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.17,"confidence":0.83,"probabilities":null,"latency_ms":157.66,"input_tokens":311,"retries":0}},{"id":"file-04-auth/session.py","kind":"file_context","group":"file-04","file":"auth/session.py","expected":false,"state":"Agent task: Clients receive 429 responses despite being below the documented limit.\nSearch hit: auth/session.py — Creates, refreshes, and expires login sessions and tokens.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.330643,"confidence":0.669357,"probabilities":null,"latency_ms":32.81,"input_tokens":47,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.29,"confidence":0.71,"probabilities":null,"latency_ms":134.15,"input_tokens":314,"retries":0}},{"id":"file-04-search/query.py","kind":"file_context","group":"file-04","file":"search/query.py","expected":false,"state":"Agent task: Clients receive 429 responses despite being below the documented limit.\nSearch hit: search/query.py — Ranks and filters search results at query time.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.359727,"confidence":0.640273,"probabilities":null,"latency_ms":43.37,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.3,"confidence":0.7,"probabilities":null,"latency_ms":116.62,"input_tokens":311,"retries":0}},{"id":"file-04-web/ui.py","kind":"file_context","group":"file-04","file":"web/ui.py","expected":false,"state":"Agent task: Clients receive 429 responses despite being below the documented limit.\nSearch hit: web/ui.py — Stores browser UI state and renders panels.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.254819,"confidence":0.745181,"probabilities":null,"latency_ms":40.93,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.12,"confidence":0.88,"probabilities":null,"latency_ms":157.51,"input_tokens":309,"retries":0}},{"id":"file-05-billing/refunds.py","kind":"file_context","group":"file-05","file":"billing/refunds.py","expected":false,"state":"Agent task: A newly added API path returns 404. Find the path registration.\nSearch hit: billing/refunds.py — Processes refund requests and payment-provider callbacks.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.346075,"confidence":0.653925,"probabilities":null,"latency_ms":36.59,"input_tokens":46,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.54,"confidence":0.54,"probabilities":null,"latency_ms":101.91,"input_tokens":312,"retries":0}},{"id":"file-05-ops/metrics.py","kind":"file_context","group":"file-05","file":"ops/metrics.py","expected":false,"state":"Agent task: A newly added API path returns 404. Find the path registration.\nSearch hit: ops/metrics.py — Records service counters and dashboard metrics.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.345223,"confidence":0.654777,"probabilities":null,"latency_ms":47.15,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.13,"confidence":0.87,"probabilities":null,"latency_ms":135.67,"input_tokens":311,"retries":0}},{"id":"file-05-auth/session.py","kind":"file_context","group":"file-05","file":"auth/session.py","expected":false,"state":"Agent task: A newly added API path returns 404. Find the path registration.\nSearch hit: auth/session.py — Creates, refreshes, and expires login sessions and tokens.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.304026,"confidence":0.695974,"probabilities":null,"latency_ms":35.57,"input_tokens":49,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.29,"confidence":0.71,"probabilities":null,"latency_ms":118.85,"input_tokens":316,"retries":0}},{"id":"file-05-api/rate_limit.py","kind":"file_context","group":"file-05","file":"api/rate_limit.py","expected":false,"state":"Agent task: A newly added API path returns 404. Find the path registration.\nSearch hit: api/rate_limit.py — Applies per-client throttling and 429 responses.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.218964,"confidence":0.781036,"probabilities":null,"latency_ms":36.7,"input_tokens":51,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.15,"confidence":0.85,"probabilities":null,"latency_ms":137.38,"input_tokens":318,"retries":0}},{"id":"file-05-jobs/retries.py","kind":"file_context","group":"file-05","file":"jobs/retries.py","expected":false,"state":"Agent task: A newly added API path returns 404. Find the path registration.\nSearch hit: jobs/retries.py — Retries failed asynchronous jobs with backoff.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.229651,"confidence":0.770349,"probabilities":null,"latency_ms":35.09,"input_tokens":47,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.11,"confidence":0.89,"probabilities":null,"latency_ms":114.29,"input_tokens":313,"retries":0}},{"id":"file-05-auth/permissions.py","kind":"file_context","group":"file-05","file":"auth/permissions.py","expected":false,"state":"Agent task: A newly added API path returns 404. Find the path registration.\nSearch hit: auth/permissions.py — Checks role membership and resource permissions.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.235713,"confidence":0.764287,"probabilities":null,"latency_ms":33.51,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.16,"confidence":0.84,"probabilities":null,"latency_ms":107.3,"input_tokens":313,"retries":0}},{"id":"file-05-search/index.py","kind":"file_context","group":"file-05","file":"search/index.py","expected":false,"state":"Agent task: A newly added API path returns 404. Find the path registration.\nSearch hit: search/index.py — Updates the search index from change events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.245158,"confidence":0.754842,"probabilities":null,"latency_ms":34.77,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.14,"confidence":0.86,"probabilities":null,"latency_ms":100.82,"input_tokens":311,"retries":0}},{"id":"file-05-web/notifications.py","kind":"file_context","group":"file-05","file":"web/notifications.py","expected":false,"state":"Agent task: A newly added API path returns 404. Find the path registration.\nSearch hit: web/notifications.py — Formats and dispatches user notifications.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.292635,"confidence":0.707365,"probabilities":null,"latency_ms":35.71,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.21,"confidence":0.79,"probabilities":null,"latency_ms":156.53,"input_tokens":313,"retries":0}},{"id":"file-05-storage/migrations.py","kind":"file_context","group":"file-05","file":"storage/migrations.py","expected":false,"state":"Agent task: A newly added API path returns 404. Find the path registration.\nSearch hit: storage/migrations.py — Runs and validates database schema migrations.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.113551,"confidence":0.886449,"probabilities":null,"latency_ms":34.32,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.08,"confidence":0.92,"probabilities":null,"latency_ms":138.85,"input_tokens":312,"retries":0}},{"id":"file-05-billing/invoices.py","kind":"file_context","group":"file-05","file":"billing/invoices.py","expected":false,"state":"Agent task: A newly added API path returns 404. Find the path registration.\nSearch hit: billing/invoices.py — Creates renewal invoices and records payment attempts.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.304122,"confidence":0.695878,"probabilities":null,"latency_ms":35.46,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.3,"confidence":0.7,"probabilities":null,"latency_ms":119.12,"input_tokens":313,"retries":0}},{"id":"file-05-storage/cache.py","kind":"file_context","group":"file-05","file":"storage/cache.py","expected":false,"state":"Agent task: A newly added API path returns 404. Find the path registration.\nSearch hit: storage/cache.py — Caches reads and evicts old entries.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.200301,"confidence":0.799699,"probabilities":null,"latency_ms":34.75,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.1,"confidence":0.9,"probabilities":null,"latency_ms":104.72,"input_tokens":312,"retries":0}},{"id":"file-05-web/ui.py","kind":"file_context","group":"file-05","file":"web/ui.py","expected":false,"state":"Agent task: A newly added API path returns 404. Find the path registration.\nSearch hit: web/ui.py — Stores browser UI state and renders panels.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.118648,"confidence":0.881352,"probabilities":null,"latency_ms":36.63,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.16,"confidence":0.84,"probabilities":null,"latency_ms":115.02,"input_tokens":311,"retries":0}},{"id":"file-05-jobs/scheduler.py","kind":"file_context","group":"file-05","file":"jobs/scheduler.py","expected":false,"state":"Agent task: A newly added API path returns 404. Find the path registration.\nSearch hit: jobs/scheduler.py — Starts scheduled background exports and maintenance jobs.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.414482,"confidence":0.585518,"probabilities":null,"latency_ms":44.48,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.13,"confidence":0.87,"probabilities":null,"latency_ms":100.62,"input_tokens":313,"retries":0}},{"id":"file-05-search/query.py","kind":"file_context","group":"file-05","file":"search/query.py","expected":false,"state":"Agent task: A newly added API path returns 404. Find the path registration.\nSearch hit: search/query.py — Ranks and filters search results at query time.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.221862,"confidence":0.778138,"probabilities":null,"latency_ms":34.36,"input_tokens":47,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.15,"confidence":0.85,"probabilities":null,"latency_ms":123.96,"input_tokens":313,"retries":0}},{"id":"file-05-api/routes.py","kind":"file_context","group":"file-05","file":"api/routes.py","expected":true,"state":"Agent task: A newly added API path returns 404. Find the path registration.\nSearch hit: api/routes.py — Maps HTTP paths to handlers and emits endpoint events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":true,"correct":true,"score":0.536679,"confidence":0.536679,"probabilities":null,"latency_ms":43.61,"input_tokens":47,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.94,"confidence":0.94,"probabilities":null,"latency_ms":139.45,"input_tokens":313,"retries":0}},{"id":"file-06-auth/permissions.py","kind":"file_context","group":"file-06","file":"auth/permissions.py","expected":false,"state":"Agent task: Search results stay stale after changes because index updates may be cached.\nSearch hit: auth/permissions.py — Checks role membership and resource permissions.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.257308,"confidence":0.742692,"probabilities":null,"latency_ms":41.08,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.42,"confidence":0.58,"probabilities":null,"latency_ms":118.82,"input_tokens":310,"retries":0}},{"id":"file-06-web/ui.py","kind":"file_context","group":"file-06","file":"web/ui.py","expected":false,"state":"Agent task: Search results stay stale after changes because index updates may be cached.\nSearch hit: web/ui.py — Stores browser UI state and renders panels.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.321966,"confidence":0.678034,"probabilities":null,"latency_ms":37.51,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.27,"confidence":0.73,"probabilities":null,"latency_ms":100.31,"input_tokens":308,"retries":0}},{"id":"file-06-search/index.py","kind":"file_context","group":"file-06","file":"search/index.py","expected":true,"state":"Agent task: Search results stay stale after changes because index updates may be cached.\nSearch hit: search/index.py — Updates the search index from change events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.408281,"confidence":0.591719,"probabilities":null,"latency_ms":42.35,"input_tokens":42,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.89,"confidence":0.89,"probabilities":null,"latency_ms":101.94,"input_tokens":308,"retries":0}},{"id":"file-06-ops/metrics.py","kind":"file_context","group":"file-06","file":"ops/metrics.py","expected":false,"state":"Agent task: Search results stay stale after changes because index updates may be cached.\nSearch hit: ops/metrics.py — Records service counters and dashboard metrics.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.363413,"confidence":0.636587,"probabilities":null,"latency_ms":37.74,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.45,"confidence":0.55,"probabilities":null,"latency_ms":114.29,"input_tokens":308,"retries":0}},{"id":"file-06-api/routes.py","kind":"file_context","group":"file-06","file":"api/routes.py","expected":false,"state":"Agent task: Search results stay stale after changes because index updates may be cached.\nSearch hit: api/routes.py — Maps HTTP paths to handlers and emits endpoint events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.293305,"confidence":0.706695,"probabilities":null,"latency_ms":35.3,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.47,"confidence":0.53,"probabilities":null,"latency_ms":106.72,"input_tokens":310,"retries":0}},{"id":"file-06-web/notifications.py","kind":"file_context","group":"file-06","file":"web/notifications.py","expected":false,"state":"Agent task: Search results stay stale after changes because index updates may be cached.\nSearch hit: web/notifications.py — Formats and dispatches user notifications.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.243844,"confidence":0.756156,"probabilities":null,"latency_ms":37.63,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.2,"confidence":0.8,"probabilities":null,"latency_ms":105.36,"input_tokens":310,"retries":0}},{"id":"file-06-billing/invoices.py","kind":"file_context","group":"file-06","file":"billing/invoices.py","expected":false,"state":"Agent task: Search results stay stale after changes because index updates may be cached.\nSearch hit: billing/invoices.py — Creates renewal invoices and records payment attempts.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.289412,"confidence":0.710588,"probabilities":null,"latency_ms":34.87,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.27,"confidence":0.73,"probabilities":null,"latency_ms":112.25,"input_tokens":310,"retries":0}},{"id":"file-06-storage/migrations.py","kind":"file_context","group":"file-06","file":"storage/migrations.py","expected":false,"state":"Agent task: Search results stay stale after changes because index updates may be cached.\nSearch hit: storage/migrations.py — Runs and validates database schema migrations.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.351223,"confidence":0.648777,"probabilities":null,"latency_ms":34.52,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.24,"confidence":0.76,"probabilities":null,"latency_ms":109.55,"input_tokens":309,"retries":0}},{"id":"file-06-jobs/scheduler.py","kind":"file_context","group":"file-06","file":"jobs/scheduler.py","expected":false,"state":"Agent task: Search results stay stale after changes because index updates may be cached.\nSearch hit: jobs/scheduler.py — Starts scheduled background exports and maintenance jobs.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.394807,"confidence":0.605193,"probabilities":null,"latency_ms":35.92,"input_tokens":43,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.67,"confidence":0.67,"probabilities":null,"latency_ms":97.2,"input_tokens":310,"retries":0}},{"id":"file-06-jobs/retries.py","kind":"file_context","group":"file-06","file":"jobs/retries.py","expected":false,"state":"Agent task: Search results stay stale after changes because index updates may be cached.\nSearch hit: jobs/retries.py — Retries failed asynchronous jobs with backoff.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.304499,"confidence":0.695501,"probabilities":null,"latency_ms":35.74,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.29,"confidence":0.71,"probabilities":null,"latency_ms":116.89,"input_tokens":310,"retries":0}},{"id":"file-06-api/rate_limit.py","kind":"file_context","group":"file-06","file":"api/rate_limit.py","expected":false,"state":"Agent task: Search results stay stale after changes because index updates may be cached.\nSearch hit: api/rate_limit.py — Applies per-client throttling and 429 responses.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.260495,"confidence":0.739505,"probabilities":null,"latency_ms":34.88,"input_tokens":48,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.22,"confidence":0.78,"probabilities":null,"latency_ms":107.34,"input_tokens":315,"retries":0}},{"id":"file-06-billing/refunds.py","kind":"file_context","group":"file-06","file":"billing/refunds.py","expected":false,"state":"Agent task: Search results stay stale after changes because index updates may be cached.\nSearch hit: billing/refunds.py — Processes refund requests and payment-provider callbacks.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.312625,"confidence":0.687375,"probabilities":null,"latency_ms":36.41,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.36,"confidence":0.64,"probabilities":null,"latency_ms":137.2,"input_tokens":309,"retries":0}},{"id":"file-06-auth/session.py","kind":"file_context","group":"file-06","file":"auth/session.py","expected":false,"state":"Agent task: Search results stay stale after changes because index updates may be cached.\nSearch hit: auth/session.py — Creates, refreshes, and expires login sessions and tokens.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.282618,"confidence":0.717382,"probabilities":null,"latency_ms":45.97,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.37,"confidence":0.63,"probabilities":null,"latency_ms":134.5,"input_tokens":313,"retries":0}},{"id":"file-06-search/query.py","kind":"file_context","group":"file-06","file":"search/query.py","expected":false,"state":"Agent task: Search results stay stale after changes because index updates may be cached.\nSearch hit: search/query.py — Ranks and filters search results at query time.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.48377,"confidence":0.51623,"probabilities":null,"latency_ms":36.18,"input_tokens":44,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.58,"confidence":0.58,"probabilities":null,"latency_ms":113.36,"input_tokens":310,"retries":0}},{"id":"file-06-storage/cache.py","kind":"file_context","group":"file-06","file":"storage/cache.py","expected":true,"state":"Agent task: Search results stay stale after changes because index updates may be cached.\nSearch hit: storage/cache.py — Caches reads and evicts old entries.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.484799,"confidence":0.515201,"probabilities":null,"latency_ms":35.32,"input_tokens":43,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.85,"confidence":0.85,"probabilities":null,"latency_ms":118.36,"input_tokens":309,"retries":0}},{"id":"file-07-auth/permissions.py","kind":"file_context","group":"file-07","file":"auth/permissions.py","expected":false,"state":"Agent task: Relevant documents appear below unrelated results. Find ranking logic.\nSearch hit: auth/permissions.py — Checks role membership and resource permissions.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.285333,"confidence":0.714667,"probabilities":null,"latency_ms":35.16,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.43,"confidence":0.57,"probabilities":null,"latency_ms":119.17,"input_tokens":308,"retries":0}},{"id":"file-07-api/rate_limit.py","kind":"file_context","group":"file-07","file":"api/rate_limit.py","expected":false,"state":"Agent task: Relevant documents appear below unrelated results. Find ranking logic.\nSearch hit: api/rate_limit.py — Applies per-client throttling and 429 responses.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.200378,"confidence":0.799622,"probabilities":null,"latency_ms":34.26,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.37,"confidence":0.63,"probabilities":null,"latency_ms":115.47,"input_tokens":313,"retries":0}},{"id":"file-07-search/query.py","kind":"file_context","group":"file-07","file":"search/query.py","expected":true,"state":"Agent task: Relevant documents appear below unrelated results. Find ranking logic.\nSearch hit: search/query.py — Ranks and filters search results at query time.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.475465,"confidence":0.524535,"probabilities":null,"latency_ms":33.76,"input_tokens":42,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.83,"confidence":0.83,"probabilities":null,"latency_ms":120.62,"input_tokens":308,"retries":0}},{"id":"file-07-web/ui.py","kind":"file_context","group":"file-07","file":"web/ui.py","expected":false,"state":"Agent task: Relevant documents appear below unrelated results. Find ranking logic.\nSearch hit: web/ui.py — Stores browser UI state and renders panels.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.331897,"confidence":0.668103,"probabilities":null,"latency_ms":34.05,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.26,"confidence":0.74,"probabilities":null,"latency_ms":100.94,"input_tokens":306,"retries":0}},{"id":"file-07-web/notifications.py","kind":"file_context","group":"file-07","file":"web/notifications.py","expected":false,"state":"Agent task: Relevant documents appear below unrelated results. Find ranking logic.\nSearch hit: web/notifications.py — Formats and dispatches user notifications.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.261152,"confidence":0.738848,"probabilities":null,"latency_ms":35.24,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.25,"confidence":0.75,"probabilities":null,"latency_ms":97.01,"input_tokens":308,"retries":0}},{"id":"file-07-search/index.py","kind":"file_context","group":"file-07","file":"search/index.py","expected":false,"state":"Agent task: Relevant documents appear below unrelated results. Find ranking logic.\nSearch hit: search/index.py — Updates the search index from change events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.308521,"confidence":0.691479,"probabilities":null,"latency_ms":34.77,"input_tokens":40,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.5,"confidence":0.5,"probabilities":null,"latency_ms":98.74,"input_tokens":306,"retries":0}},{"id":"file-07-ops/metrics.py","kind":"file_context","group":"file-07","file":"ops/metrics.py","expected":false,"state":"Agent task: Relevant documents appear below unrelated results. Find ranking logic.\nSearch hit: ops/metrics.py — Records service counters and dashboard metrics.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.382271,"confidence":0.617729,"probabilities":null,"latency_ms":45.46,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.33,"confidence":0.67,"probabilities":null,"latency_ms":115.68,"input_tokens":306,"retries":0}},{"id":"file-07-auth/session.py","kind":"file_context","group":"file-07","file":"auth/session.py","expected":false,"state":"Agent task: Relevant documents appear below unrelated results. Find ranking logic.\nSearch hit: auth/session.py — Creates, refreshes, and expires login sessions and tokens.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.251501,"confidence":0.748499,"probabilities":null,"latency_ms":39.19,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.31,"confidence":0.69,"probabilities":null,"latency_ms":120.05,"input_tokens":311,"retries":0}},{"id":"file-07-storage/migrations.py","kind":"file_context","group":"file-07","file":"storage/migrations.py","expected":false,"state":"Agent task: Relevant documents appear below unrelated results. Find ranking logic.\nSearch hit: storage/migrations.py — Runs and validates database schema migrations.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.340364,"confidence":0.659636,"probabilities":null,"latency_ms":35.8,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.29,"confidence":0.71,"probabilities":null,"latency_ms":113.73,"input_tokens":307,"retries":0}},{"id":"file-07-billing/invoices.py","kind":"file_context","group":"file-07","file":"billing/invoices.py","expected":false,"state":"Agent task: Relevant documents appear below unrelated results. Find ranking logic.\nSearch hit: billing/invoices.py — Creates renewal invoices and records payment attempts.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.346017,"confidence":0.653983,"probabilities":null,"latency_ms":35.0,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.36,"confidence":0.64,"probabilities":null,"latency_ms":140.85,"input_tokens":308,"retries":0}},{"id":"file-07-billing/refunds.py","kind":"file_context","group":"file-07","file":"billing/refunds.py","expected":false,"state":"Agent task: Relevant documents appear below unrelated results. Find ranking logic.\nSearch hit: billing/refunds.py — Processes refund requests and payment-provider callbacks.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.347196,"confidence":0.652804,"probabilities":null,"latency_ms":34.94,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.27,"confidence":0.73,"probabilities":null,"latency_ms":124.54,"input_tokens":307,"retries":0}},{"id":"file-07-jobs/retries.py","kind":"file_context","group":"file-07","file":"jobs/retries.py","expected":false,"state":"Agent task: Relevant documents appear below unrelated results. Find ranking logic.\nSearch hit: jobs/retries.py — Retries failed asynchronous jobs with backoff.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.365263,"confidence":0.634737,"probabilities":null,"latency_ms":34.43,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.32,"confidence":0.68,"probabilities":null,"latency_ms":99.25,"input_tokens":308,"retries":0}},{"id":"file-07-storage/cache.py","kind":"file_context","group":"file-07","file":"storage/cache.py","expected":false,"state":"Agent task: Relevant documents appear below unrelated results. Find ranking logic.\nSearch hit: storage/cache.py — Caches reads and evicts old entries.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.294299,"confidence":0.705701,"probabilities":null,"latency_ms":34.1,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.31,"confidence":0.69,"probabilities":null,"latency_ms":129.43,"input_tokens":307,"retries":0}},{"id":"file-07-jobs/scheduler.py","kind":"file_context","group":"file-07","file":"jobs/scheduler.py","expected":false,"state":"Agent task: Relevant documents appear below unrelated results. Find ranking logic.\nSearch hit: jobs/scheduler.py — Starts scheduled background exports and maintenance jobs.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.40887,"confidence":0.59113,"probabilities":null,"latency_ms":35.81,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.33,"confidence":0.67,"probabilities":null,"latency_ms":130.68,"input_tokens":308,"retries":0}},{"id":"file-07-api/routes.py","kind":"file_context","group":"file-07","file":"api/routes.py","expected":false,"state":"Agent task: Relevant documents appear below unrelated results. Find ranking logic.\nSearch hit: api/routes.py — Maps HTTP paths to handlers and emits endpoint events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.276844,"confidence":0.723156,"probabilities":null,"latency_ms":33.98,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.47,"confidence":0.53,"probabilities":null,"latency_ms":171.85,"input_tokens":308,"retries":0}},{"id":"file-08-storage/migrations.py","kind":"file_context","group":"file-08","file":"storage/migrations.py","expected":false,"state":"Agent task: The in-memory cache keeps growing after old records should expire.\nSearch hit: storage/migrations.py — Runs and validates database schema migrations.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.246263,"confidence":0.753737,"probabilities":null,"latency_ms":36.13,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.11,"confidence":0.89,"probabilities":null,"latency_ms":107.02,"input_tokens":308,"retries":0}},{"id":"file-08-billing/invoices.py","kind":"file_context","group":"file-08","file":"billing/invoices.py","expected":false,"state":"Agent task: The in-memory cache keeps growing after old records should expire.\nSearch hit: billing/invoices.py — Creates renewal invoices and records payment attempts.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.405218,"confidence":0.594782,"probabilities":null,"latency_ms":33.92,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.28,"confidence":0.72,"probabilities":null,"latency_ms":125.67,"input_tokens":309,"retries":0}},{"id":"file-08-web/notifications.py","kind":"file_context","group":"file-08","file":"web/notifications.py","expected":false,"state":"Agent task: The in-memory cache keeps growing after old records should expire.\nSearch hit: web/notifications.py — Formats and dispatches user notifications.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.229376,"confidence":0.770624,"probabilities":null,"latency_ms":34.59,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.15,"confidence":0.85,"probabilities":null,"latency_ms":111.69,"input_tokens":309,"retries":0}},{"id":"file-08-billing/refunds.py","kind":"file_context","group":"file-08","file":"billing/refunds.py","expected":false,"state":"Agent task: The in-memory cache keeps growing after old records should expire.\nSearch hit: billing/refunds.py — Processes refund requests and payment-provider callbacks.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.391361,"confidence":0.608639,"probabilities":null,"latency_ms":33.16,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.29,"confidence":0.71,"probabilities":null,"latency_ms":114.85,"input_tokens":308,"retries":0}},{"id":"file-08-storage/cache.py","kind":"file_context","group":"file-08","file":"storage/cache.py","expected":true,"state":"Agent task: The in-memory cache keeps growing after old records should expire.\nSearch hit: storage/cache.py — Caches reads and evicts old entries.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":true,"correct":true,"score":0.554942,"confidence":0.554942,"probabilities":null,"latency_ms":32.81,"input_tokens":42,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.95,"confidence":0.95,"probabilities":null,"latency_ms":96.05,"input_tokens":308,"retries":0}},{"id":"file-08-jobs/retries.py","kind":"file_context","group":"file-08","file":"jobs/retries.py","expected":false,"state":"Agent task: The in-memory cache keeps growing after old records should expire.\nSearch hit: jobs/retries.py — Retries failed asynchronous jobs with backoff.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.32016,"confidence":0.67984,"probabilities":null,"latency_ms":33.9,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.16,"confidence":0.84,"probabilities":null,"latency_ms":109.18,"input_tokens":309,"retries":0}},{"id":"file-08-api/rate_limit.py","kind":"file_context","group":"file-08","file":"api/rate_limit.py","expected":false,"state":"Agent task: The in-memory cache keeps growing after old records should expire.\nSearch hit: api/rate_limit.py — Applies per-client throttling and 429 responses.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.31569,"confidence":0.68431,"probabilities":null,"latency_ms":33.91,"input_tokens":47,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.21,"confidence":0.79,"probabilities":null,"latency_ms":130.32,"input_tokens":314,"retries":0}},{"id":"file-08-search/query.py","kind":"file_context","group":"file-08","file":"search/query.py","expected":false,"state":"Agent task: The in-memory cache keeps growing after old records should expire.\nSearch hit: search/query.py — Ranks and filters search results at query time.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.314768,"confidence":0.685232,"probabilities":null,"latency_ms":33.85,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.26,"confidence":0.74,"probabilities":null,"latency_ms":100.04,"input_tokens":309,"retries":0}},{"id":"file-08-jobs/scheduler.py","kind":"file_context","group":"file-08","file":"jobs/scheduler.py","expected":false,"state":"Agent task: The in-memory cache keeps growing after old records should expire.\nSearch hit: jobs/scheduler.py — Starts scheduled background exports and maintenance jobs.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.385564,"confidence":0.614436,"probabilities":null,"latency_ms":34.08,"input_tokens":42,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.64,"confidence":0.64,"probabilities":null,"latency_ms":116.64,"input_tokens":309,"retries":0}},{"id":"file-08-ops/metrics.py","kind":"file_context","group":"file-08","file":"ops/metrics.py","expected":false,"state":"Agent task: The in-memory cache keeps growing after old records should expire.\nSearch hit: ops/metrics.py — Records service counters and dashboard metrics.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.398121,"confidence":0.601879,"probabilities":null,"latency_ms":33.9,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.24,"confidence":0.76,"probabilities":null,"latency_ms":115.56,"input_tokens":307,"retries":0}},{"id":"file-08-auth/permissions.py","kind":"file_context","group":"file-08","file":"auth/permissions.py","expected":false,"state":"Agent task: The in-memory cache keeps growing after old records should expire.\nSearch hit: auth/permissions.py — Checks role membership and resource permissions.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.2081,"confidence":0.7919,"probabilities":null,"latency_ms":34.09,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.18,"confidence":0.82,"probabilities":null,"latency_ms":95.78,"input_tokens":309,"retries":0}},{"id":"file-08-web/ui.py","kind":"file_context","group":"file-08","file":"web/ui.py","expected":false,"state":"Agent task: The in-memory cache keeps growing after old records should expire.\nSearch hit: web/ui.py — Stores browser UI state and renders panels.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.262399,"confidence":0.737601,"probabilities":null,"latency_ms":47.12,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.16,"confidence":0.84,"probabilities":null,"latency_ms":103.75,"input_tokens":307,"retries":0}},{"id":"file-08-search/index.py","kind":"file_context","group":"file-08","file":"search/index.py","expected":false,"state":"Agent task: The in-memory cache keeps growing after old records should expire.\nSearch hit: search/index.py — Updates the search index from change events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.326453,"confidence":0.673547,"probabilities":null,"latency_ms":35.02,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.25,"confidence":0.75,"probabilities":null,"latency_ms":128.9,"input_tokens":307,"retries":0}},{"id":"file-08-auth/session.py","kind":"file_context","group":"file-08","file":"auth/session.py","expected":false,"state":"Agent task: The in-memory cache keeps growing after old records should expire.\nSearch hit: auth/session.py — Creates, refreshes, and expires login sessions and tokens.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.458686,"confidence":0.541314,"probabilities":null,"latency_ms":36.39,"input_tokens":45,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.83,"confidence":0.83,"probabilities":null,"latency_ms":114.67,"input_tokens":312,"retries":0}},{"id":"file-08-api/routes.py","kind":"file_context","group":"file-08","file":"api/routes.py","expected":false,"state":"Agent task: The in-memory cache keeps growing after old records should expire.\nSearch hit: api/routes.py — Maps HTTP paths to handlers and emits endpoint events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.286698,"confidence":0.713302,"probabilities":null,"latency_ms":35.26,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.19,"confidence":0.81,"probabilities":null,"latency_ms":101.24,"input_tokens":309,"retries":0}},{"id":"file-09-auth/permissions.py","kind":"file_context","group":"file-09","file":"auth/permissions.py","expected":false,"state":"Agent task: Migration 042 fails because a column is missing. Find migration execution.\nSearch hit: auth/permissions.py — Checks role membership and resource permissions.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.237701,"confidence":0.762299,"probabilities":null,"latency_ms":35.59,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.13,"confidence":0.87,"probabilities":null,"latency_ms":110.28,"input_tokens":313,"retries":0}},{"id":"file-09-web/notifications.py","kind":"file_context","group":"file-09","file":"web/notifications.py","expected":false,"state":"Agent task: Migration 042 fails because a column is missing. Find migration execution.\nSearch hit: web/notifications.py — Formats and dispatches user notifications.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.244349,"confidence":0.755651,"probabilities":null,"latency_ms":33.39,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.07,"confidence":0.93,"probabilities":null,"latency_ms":126.43,"input_tokens":313,"retries":0}},{"id":"file-09-storage/migrations.py","kind":"file_context","group":"file-09","file":"storage/migrations.py","expected":true,"state":"Agent task: Migration 042 fails because a column is missing. Find migration execution.\nSearch hit: storage/migrations.py — Runs and validates database schema migrations.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":true,"correct":true,"score":0.510915,"confidence":0.510915,"probabilities":null,"latency_ms":34.33,"input_tokens":45,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.89,"confidence":0.89,"probabilities":null,"latency_ms":165.51,"input_tokens":312,"retries":0}},{"id":"file-09-ops/metrics.py","kind":"file_context","group":"file-09","file":"ops/metrics.py","expected":false,"state":"Agent task: Migration 042 fails because a column is missing. Find migration execution.\nSearch hit: ops/metrics.py — Records service counters and dashboard metrics.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.434036,"confidence":0.565964,"probabilities":null,"latency_ms":33.12,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.09,"confidence":0.91,"probabilities":null,"latency_ms":142.37,"input_tokens":311,"retries":0}},{"id":"file-09-storage/cache.py","kind":"file_context","group":"file-09","file":"storage/cache.py","expected":false,"state":"Agent task: Migration 042 fails because a column is missing. Find migration execution.\nSearch hit: storage/cache.py — Caches reads and evicts old entries.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.257018,"confidence":0.742982,"probabilities":null,"latency_ms":34.17,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.09,"confidence":0.91,"probabilities":null,"latency_ms":143.79,"input_tokens":312,"retries":0}},{"id":"file-09-billing/refunds.py","kind":"file_context","group":"file-09","file":"billing/refunds.py","expected":false,"state":"Agent task: Migration 042 fails because a column is missing. Find migration execution.\nSearch hit: billing/refunds.py — Processes refund requests and payment-provider callbacks.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.332775,"confidence":0.667225,"probabilities":null,"latency_ms":33.85,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.19,"confidence":0.81,"probabilities":null,"latency_ms":117.93,"input_tokens":312,"retries":0}},{"id":"file-09-api/rate_limit.py","kind":"file_context","group":"file-09","file":"api/rate_limit.py","expected":false,"state":"Agent task: Migration 042 fails because a column is missing. Find migration execution.\nSearch hit: api/rate_limit.py — Applies per-client throttling and 429 responses.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.175134,"confidence":0.824866,"probabilities":null,"latency_ms":34.23,"input_tokens":51,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.07,"confidence":0.93,"probabilities":null,"latency_ms":151.31,"input_tokens":318,"retries":0}},{"id":"file-09-jobs/scheduler.py","kind":"file_context","group":"file-09","file":"jobs/scheduler.py","expected":false,"state":"Agent task: Migration 042 fails because a column is missing. Find migration execution.\nSearch hit: jobs/scheduler.py — Starts scheduled background exports and maintenance jobs.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.33921,"confidence":0.66079,"probabilities":null,"latency_ms":34.06,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.2,"confidence":0.8,"probabilities":null,"latency_ms":112.76,"input_tokens":313,"retries":0}},{"id":"file-09-search/index.py","kind":"file_context","group":"file-09","file":"search/index.py","expected":false,"state":"Agent task: Migration 042 fails because a column is missing. Find migration execution.\nSearch hit: search/index.py — Updates the search index from change events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.29815,"confidence":0.70185,"probabilities":null,"latency_ms":35.97,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.14,"confidence":0.86,"probabilities":null,"latency_ms":141.5,"input_tokens":311,"retries":0}},{"id":"file-09-web/ui.py","kind":"file_context","group":"file-09","file":"web/ui.py","expected":false,"state":"Agent task: Migration 042 fails because a column is missing. Find migration execution.\nSearch hit: web/ui.py — Stores browser UI state and renders panels.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.295828,"confidence":0.704172,"probabilities":null,"latency_ms":33.88,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.06,"confidence":0.94,"probabilities":null,"latency_ms":99.69,"input_tokens":311,"retries":0}},{"id":"file-09-auth/session.py","kind":"file_context","group":"file-09","file":"auth/session.py","expected":false,"state":"Agent task: Migration 042 fails because a column is missing. Find migration execution.\nSearch hit: auth/session.py — Creates, refreshes, and expires login sessions and tokens.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.341655,"confidence":0.658345,"probabilities":null,"latency_ms":35.56,"input_tokens":49,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.13,"confidence":0.87,"probabilities":null,"latency_ms":133.17,"input_tokens":316,"retries":0}},{"id":"file-09-search/query.py","kind":"file_context","group":"file-09","file":"search/query.py","expected":false,"state":"Agent task: Migration 042 fails because a column is missing. Find migration execution.\nSearch hit: search/query.py — Ranks and filters search results at query time.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.225873,"confidence":0.774127,"probabilities":null,"latency_ms":35.92,"input_tokens":47,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.11,"confidence":0.89,"probabilities":null,"latency_ms":151.08,"input_tokens":313,"retries":0}},{"id":"file-09-api/routes.py","kind":"file_context","group":"file-09","file":"api/routes.py","expected":false,"state":"Agent task: Migration 042 fails because a column is missing. Find migration execution.\nSearch hit: api/routes.py — Maps HTTP paths to handlers and emits endpoint events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.33889,"confidence":0.66111,"probabilities":null,"latency_ms":48.43,"input_tokens":47,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.09,"confidence":0.91,"probabilities":null,"latency_ms":100.61,"input_tokens":313,"retries":0}},{"id":"file-09-billing/invoices.py","kind":"file_context","group":"file-09","file":"billing/invoices.py","expected":false,"state":"Agent task: Migration 042 fails because a column is missing. Find migration execution.\nSearch hit: billing/invoices.py — Creates renewal invoices and records payment attempts.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.381937,"confidence":0.618063,"probabilities":null,"latency_ms":33.75,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.25,"confidence":0.75,"probabilities":null,"latency_ms":117.68,"input_tokens":313,"retries":0}},{"id":"file-09-jobs/retries.py","kind":"file_context","group":"file-09","file":"jobs/retries.py","expected":false,"state":"Agent task: Migration 042 fails because a column is missing. Find migration execution.\nSearch hit: jobs/retries.py — Retries failed asynchronous jobs with backoff.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.217247,"confidence":0.782753,"probabilities":null,"latency_ms":34.02,"input_tokens":47,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.21,"confidence":0.79,"probabilities":null,"latency_ms":95.74,"input_tokens":313,"retries":0}},{"id":"file-10-web/ui.py","kind":"file_context","group":"file-10","file":"web/ui.py","expected":true,"state":"Agent task: A settings panel loses its local selection after rerender.\nSearch hit: web/ui.py — Stores browser UI state and renders panels.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.424792,"confidence":0.575208,"probabilities":null,"latency_ms":35.5,"input_tokens":40,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.85,"confidence":0.85,"probabilities":null,"latency_ms":97.4,"input_tokens":306,"retries":0}},{"id":"file-10-storage/migrations.py","kind":"file_context","group":"file-10","file":"storage/migrations.py","expected":false,"state":"Agent task: A settings panel loses its local selection after rerender.\nSearch hit: storage/migrations.py — Runs and validates database schema migrations.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.148512,"confidence":0.851488,"probabilities":null,"latency_ms":37.09,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.08,"confidence":0.92,"probabilities":null,"latency_ms":105.32,"input_tokens":307,"retries":0}},{"id":"file-10-billing/invoices.py","kind":"file_context","group":"file-10","file":"billing/invoices.py","expected":false,"state":"Agent task: A settings panel loses its local selection after rerender.\nSearch hit: billing/invoices.py — Creates renewal invoices and records payment attempts.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.174941,"confidence":0.825059,"probabilities":null,"latency_ms":34.17,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.07,"confidence":0.93,"probabilities":null,"latency_ms":115.54,"input_tokens":308,"retries":0}},{"id":"file-10-search/query.py","kind":"file_context","group":"file-10","file":"search/query.py","expected":false,"state":"Agent task: A settings panel loses its local selection after rerender.\nSearch hit: search/query.py — Ranks and filters search results at query time.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.15859,"confidence":0.84141,"probabilities":null,"latency_ms":35.18,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.11,"confidence":0.89,"probabilities":null,"latency_ms":147.41,"input_tokens":308,"retries":0}},{"id":"file-10-auth/session.py","kind":"file_context","group":"file-10","file":"auth/session.py","expected":false,"state":"Agent task: A settings panel loses its local selection after rerender.\nSearch hit: auth/session.py — Creates, refreshes, and expires login sessions and tokens.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.116537,"confidence":0.883463,"probabilities":null,"latency_ms":42.66,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.09,"confidence":0.91,"probabilities":null,"latency_ms":117.6,"input_tokens":311,"retries":0}},{"id":"file-10-api/rate_limit.py","kind":"file_context","group":"file-10","file":"api/rate_limit.py","expected":false,"state":"Agent task: A settings panel loses its local selection after rerender.\nSearch hit: api/rate_limit.py — Applies per-client throttling and 429 responses.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.091949,"confidence":0.908051,"probabilities":null,"latency_ms":45.03,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.06,"confidence":0.94,"probabilities":null,"latency_ms":96.81,"input_tokens":313,"retries":0}},{"id":"file-10-billing/refunds.py","kind":"file_context","group":"file-10","file":"billing/refunds.py","expected":false,"state":"Agent task: A settings panel loses its local selection after rerender.\nSearch hit: billing/refunds.py — Processes refund requests and payment-provider callbacks.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.169684,"confidence":0.830316,"probabilities":null,"latency_ms":34.49,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.07,"confidence":0.93,"probabilities":null,"latency_ms":100.78,"input_tokens":307,"retries":0}},{"id":"file-10-api/routes.py","kind":"file_context","group":"file-10","file":"api/routes.py","expected":false,"state":"Agent task: A settings panel loses its local selection after rerender.\nSearch hit: api/routes.py — Maps HTTP paths to handlers and emits endpoint events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.167826,"confidence":0.832174,"probabilities":null,"latency_ms":38.22,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.13,"confidence":0.87,"probabilities":null,"latency_ms":121.86,"input_tokens":308,"retries":0}},{"id":"file-10-auth/permissions.py","kind":"file_context","group":"file-10","file":"auth/permissions.py","expected":false,"state":"Agent task: A settings panel loses its local selection after rerender.\nSearch hit: auth/permissions.py — Checks role membership and resource permissions.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.135549,"confidence":0.864451,"probabilities":null,"latency_ms":36.85,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.09,"confidence":0.91,"probabilities":null,"latency_ms":143.92,"input_tokens":308,"retries":0}},{"id":"file-10-ops/metrics.py","kind":"file_context","group":"file-10","file":"ops/metrics.py","expected":false,"state":"Agent task: A settings panel loses its local selection after rerender.\nSearch hit: ops/metrics.py — Records service counters and dashboard metrics.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.18835,"confidence":0.81165,"probabilities":null,"latency_ms":34.8,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.07,"confidence":0.93,"probabilities":null,"latency_ms":148.17,"input_tokens":306,"retries":0}},{"id":"file-10-storage/cache.py","kind":"file_context","group":"file-10","file":"storage/cache.py","expected":false,"state":"Agent task: A settings panel loses its local selection after rerender.\nSearch hit: storage/cache.py — Caches reads and evicts old entries.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.238988,"confidence":0.761012,"probabilities":null,"latency_ms":35.09,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.31,"confidence":0.69,"probabilities":null,"latency_ms":99.98,"input_tokens":307,"retries":0}},{"id":"file-10-jobs/scheduler.py","kind":"file_context","group":"file-10","file":"jobs/scheduler.py","expected":false,"state":"Agent task: A settings panel loses its local selection after rerender.\nSearch hit: jobs/scheduler.py — Starts scheduled background exports and maintenance jobs.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.138407,"confidence":0.861593,"probabilities":null,"latency_ms":34.84,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.09,"confidence":0.91,"probabilities":null,"latency_ms":91.11,"input_tokens":308,"retries":0}},{"id":"file-10-search/index.py","kind":"file_context","group":"file-10","file":"search/index.py","expected":false,"state":"Agent task: A settings panel loses its local selection after rerender.\nSearch hit: search/index.py — Updates the search index from change events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.196301,"confidence":0.803699,"probabilities":null,"latency_ms":35.13,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.12,"confidence":0.88,"probabilities":null,"latency_ms":114.01,"input_tokens":306,"retries":0}},{"id":"file-10-web/notifications.py","kind":"file_context","group":"file-10","file":"web/notifications.py","expected":false,"state":"Agent task: A settings panel loses its local selection after rerender.\nSearch hit: web/notifications.py — Formats and dispatches user notifications.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.119216,"confidence":0.880784,"probabilities":null,"latency_ms":35.02,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.08,"confidence":0.92,"probabilities":null,"latency_ms":100.56,"input_tokens":308,"retries":0}},{"id":"file-10-jobs/retries.py","kind":"file_context","group":"file-10","file":"jobs/retries.py","expected":false,"state":"Agent task: A settings panel loses its local selection after rerender.\nSearch hit: jobs/retries.py — Retries failed asynchronous jobs with backoff.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.131791,"confidence":0.868209,"probabilities":null,"latency_ms":35.47,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.07,"confidence":0.93,"probabilities":null,"latency_ms":106.18,"input_tokens":308,"retries":0}},{"id":"file-11-auth/permissions.py","kind":"file_context","group":"file-11","file":"auth/permissions.py","expected":false,"state":"Agent task: Notification emails are missing after an asynchronous retry fails.\nSearch hit: auth/permissions.py — Checks role membership and resource permissions.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.125362,"confidence":0.874638,"probabilities":null,"latency_ms":35.6,"input_tokens":39,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.12,"confidence":0.88,"probabilities":null,"latency_ms":99.07,"input_tokens":307,"retries":0}},{"id":"file-11-api/routes.py","kind":"file_context","group":"file-11","file":"api/routes.py","expected":false,"state":"Agent task: Notification emails are missing after an asynchronous retry fails.\nSearch hit: api/routes.py — Maps HTTP paths to handlers and emits endpoint events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.247519,"confidence":0.752481,"probabilities":null,"latency_ms":44.16,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.3,"confidence":0.7,"probabilities":null,"latency_ms":98.98,"input_tokens":307,"retries":0}},{"id":"file-11-jobs/scheduler.py","kind":"file_context","group":"file-11","file":"jobs/scheduler.py","expected":false,"state":"Agent task: Notification emails are missing after an asynchronous retry fails.\nSearch hit: jobs/scheduler.py — Starts scheduled background exports and maintenance jobs.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.311897,"confidence":0.688103,"probabilities":null,"latency_ms":34.95,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.33,"confidence":0.67,"probabilities":null,"latency_ms":111.0,"input_tokens":307,"retries":0}},{"id":"file-11-search/query.py","kind":"file_context","group":"file-11","file":"search/query.py","expected":false,"state":"Agent task: Notification emails are missing after an asynchronous retry fails.\nSearch hit: search/query.py — Ranks and filters search results at query time.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.208589,"confidence":0.791411,"probabilities":null,"latency_ms":34.51,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.1,"confidence":0.9,"probabilities":null,"latency_ms":116.21,"input_tokens":307,"retries":0}},{"id":"file-11-ops/metrics.py","kind":"file_context","group":"file-11","file":"ops/metrics.py","expected":false,"state":"Agent task: Notification emails are missing after an asynchronous retry fails.\nSearch hit: ops/metrics.py — Records service counters and dashboard metrics.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.24077,"confidence":0.75923,"probabilities":null,"latency_ms":36.17,"input_tokens":39,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.18,"confidence":0.82,"probabilities":null,"latency_ms":137.78,"input_tokens":305,"retries":0}},{"id":"file-11-web/notifications.py","kind":"file_context","group":"file-11","file":"web/notifications.py","expected":true,"state":"Agent task: Notification emails are missing after an asynchronous retry fails.\nSearch hit: web/notifications.py — Formats and dispatches user notifications.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.287986,"confidence":0.712014,"probabilities":null,"latency_ms":37.38,"input_tokens":39,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.9,"confidence":0.9,"probabilities":null,"latency_ms":101.25,"input_tokens":307,"retries":0}},{"id":"file-11-billing/refunds.py","kind":"file_context","group":"file-11","file":"billing/refunds.py","expected":false,"state":"Agent task: Notification emails are missing after an asynchronous retry fails.\nSearch hit: billing/refunds.py — Processes refund requests and payment-provider callbacks.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.23436,"confidence":0.76564,"probabilities":null,"latency_ms":37.96,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.48,"confidence":0.52,"probabilities":null,"latency_ms":108.96,"input_tokens":306,"retries":0}},{"id":"file-11-billing/invoices.py","kind":"file_context","group":"file-11","file":"billing/invoices.py","expected":false,"state":"Agent task: Notification emails are missing after an asynchronous retry fails.\nSearch hit: billing/invoices.py — Creates renewal invoices and records payment attempts.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.255095,"confidence":0.744905,"probabilities":null,"latency_ms":36.67,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.49,"confidence":0.51,"probabilities":null,"latency_ms":108.82,"input_tokens":307,"retries":0}},{"id":"file-11-web/ui.py","kind":"file_context","group":"file-11","file":"web/ui.py","expected":false,"state":"Agent task: Notification emails are missing after an asynchronous retry fails.\nSearch hit: web/ui.py — Stores browser UI state and renders panels.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.18416,"confidence":0.81584,"probabilities":null,"latency_ms":37.24,"input_tokens":39,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.15,"confidence":0.85,"probabilities":null,"latency_ms":147.7,"input_tokens":305,"retries":0}},{"id":"file-11-api/rate_limit.py","kind":"file_context","group":"file-11","file":"api/rate_limit.py","expected":false,"state":"Agent task: Notification emails are missing after an asynchronous retry fails.\nSearch hit: api/rate_limit.py — Applies per-client throttling and 429 responses.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.176736,"confidence":0.823264,"probabilities":null,"latency_ms":51.35,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.23,"confidence":0.77,"probabilities":null,"latency_ms":97.29,"input_tokens":312,"retries":0}},{"id":"file-11-jobs/retries.py","kind":"file_context","group":"file-11","file":"jobs/retries.py","expected":true,"state":"Agent task: Notification emails are missing after an asynchronous retry fails.\nSearch hit: jobs/retries.py — Retries failed asynchronous jobs with backoff.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.386089,"confidence":0.613911,"probabilities":null,"latency_ms":37.45,"input_tokens":41,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.73,"confidence":0.73,"probabilities":null,"latency_ms":104.49,"input_tokens":307,"retries":0}},{"id":"file-11-search/index.py","kind":"file_context","group":"file-11","file":"search/index.py","expected":false,"state":"Agent task: Notification emails are missing after an asynchronous retry fails.\nSearch hit: search/index.py — Updates the search index from change events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.199631,"confidence":0.800369,"probabilities":null,"latency_ms":36.51,"input_tokens":39,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.17,"confidence":0.83,"probabilities":null,"latency_ms":113.98,"input_tokens":305,"retries":0}},{"id":"file-11-storage/cache.py","kind":"file_context","group":"file-11","file":"storage/cache.py","expected":false,"state":"Agent task: Notification emails are missing after an asynchronous retry fails.\nSearch hit: storage/cache.py — Caches reads and evicts old entries.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.300521,"confidence":0.699479,"probabilities":null,"latency_ms":36.95,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.23,"confidence":0.77,"probabilities":null,"latency_ms":97.08,"input_tokens":306,"retries":0}},{"id":"file-11-storage/migrations.py","kind":"file_context","group":"file-11","file":"storage/migrations.py","expected":false,"state":"Agent task: Notification emails are missing after an asynchronous retry fails.\nSearch hit: storage/migrations.py — Runs and validates database schema migrations.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.204493,"confidence":0.795507,"probabilities":null,"latency_ms":36.85,"input_tokens":39,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.14,"confidence":0.86,"probabilities":null,"latency_ms":106.47,"input_tokens":306,"retries":0}},{"id":"file-11-auth/session.py","kind":"file_context","group":"file-11","file":"auth/session.py","expected":false,"state":"Agent task: Notification emails are missing after an asynchronous retry fails.\nSearch hit: auth/session.py — Creates, refreshes, and expires login sessions and tokens.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.175435,"confidence":0.824565,"probabilities":null,"latency_ms":52.89,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.13,"confidence":0.87,"probabilities":null,"latency_ms":102.26,"input_tokens":310,"retries":0}},{"id":"file-12-search/index.py","kind":"file_context","group":"file-12","file":"search/index.py","expected":false,"state":"Agent task: The nightly export did not start at its scheduled time.\nSearch hit: search/index.py — Updates the search index from change events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.231508,"confidence":0.768492,"probabilities":null,"latency_ms":38.25,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.27,"confidence":0.73,"probabilities":null,"latency_ms":157.55,"input_tokens":306,"retries":0}},{"id":"file-12-storage/cache.py","kind":"file_context","group":"file-12","file":"storage/cache.py","expected":false,"state":"Agent task: The nightly export did not start at its scheduled time.\nSearch hit: storage/cache.py — Caches reads and evicts old entries.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.250358,"confidence":0.749642,"probabilities":null,"latency_ms":38.72,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.19,"confidence":0.81,"probabilities":null,"latency_ms":108.91,"input_tokens":307,"retries":0}},{"id":"file-12-ops/metrics.py","kind":"file_context","group":"file-12","file":"ops/metrics.py","expected":false,"state":"Agent task: The nightly export did not start at its scheduled time.\nSearch hit: ops/metrics.py — Records service counters and dashboard metrics.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.301711,"confidence":0.698289,"probabilities":null,"latency_ms":46.92,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.22,"confidence":0.78,"probabilities":null,"latency_ms":108.12,"input_tokens":306,"retries":0}},{"id":"file-12-web/ui.py","kind":"file_context","group":"file-12","file":"web/ui.py","expected":false,"state":"Agent task: The nightly export did not start at its scheduled time.\nSearch hit: web/ui.py — Stores browser UI state and renders panels.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.181551,"confidence":0.818449,"probabilities":null,"latency_ms":35.82,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.1,"confidence":0.9,"probabilities":null,"latency_ms":105.49,"input_tokens":306,"retries":0}},{"id":"file-12-auth/session.py","kind":"file_context","group":"file-12","file":"auth/session.py","expected":false,"state":"Agent task: The nightly export did not start at its scheduled time.\nSearch hit: auth/session.py — Creates, refreshes, and expires login sessions and tokens.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.112141,"confidence":0.887859,"probabilities":null,"latency_ms":40.84,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.11,"confidence":0.89,"probabilities":null,"latency_ms":134.27,"input_tokens":311,"retries":0}},{"id":"file-12-billing/refunds.py","kind":"file_context","group":"file-12","file":"billing/refunds.py","expected":false,"state":"Agent task: The nightly export did not start at its scheduled time.\nSearch hit: billing/refunds.py — Processes refund requests and payment-provider callbacks.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.210113,"confidence":0.789887,"probabilities":null,"latency_ms":35.52,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.15,"confidence":0.85,"probabilities":null,"latency_ms":147.26,"input_tokens":307,"retries":0}},{"id":"file-12-storage/migrations.py","kind":"file_context","group":"file-12","file":"storage/migrations.py","expected":false,"state":"Agent task: The nightly export did not start at its scheduled time.\nSearch hit: storage/migrations.py — Runs and validates database schema migrations.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.243619,"confidence":0.756381,"probabilities":null,"latency_ms":34.62,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.24,"confidence":0.76,"probabilities":null,"latency_ms":103.31,"input_tokens":307,"retries":0}},{"id":"file-12-api/routes.py","kind":"file_context","group":"file-12","file":"api/routes.py","expected":false,"state":"Agent task: The nightly export did not start at its scheduled time.\nSearch hit: api/routes.py — Maps HTTP paths to handlers and emits endpoint events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.242145,"confidence":0.757855,"probabilities":null,"latency_ms":34.96,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.23,"confidence":0.77,"probabilities":null,"latency_ms":105.9,"input_tokens":308,"retries":0}},{"id":"file-12-jobs/retries.py","kind":"file_context","group":"file-12","file":"jobs/retries.py","expected":false,"state":"Agent task: The nightly export did not start at its scheduled time.\nSearch hit: jobs/retries.py — Retries failed asynchronous jobs with backoff.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.356802,"confidence":0.643198,"probabilities":null,"latency_ms":34.78,"input_tokens":42,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.68,"confidence":0.68,"probabilities":null,"latency_ms":121.71,"input_tokens":308,"retries":0}},{"id":"file-12-search/query.py","kind":"file_context","group":"file-12","file":"search/query.py","expected":false,"state":"Agent task: The nightly export did not start at its scheduled time.\nSearch hit: search/query.py — Ranks and filters search results at query time.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.175265,"confidence":0.824735,"probabilities":null,"latency_ms":35.38,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.11,"confidence":0.89,"probabilities":null,"latency_ms":115.33,"input_tokens":308,"retries":0}},{"id":"file-12-auth/permissions.py","kind":"file_context","group":"file-12","file":"auth/permissions.py","expected":false,"state":"Agent task: The nightly export did not start at its scheduled time.\nSearch hit: auth/permissions.py — Checks role membership and resource permissions.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.115849,"confidence":0.884151,"probabilities":null,"latency_ms":46.9,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.19,"confidence":0.81,"probabilities":null,"latency_ms":114.67,"input_tokens":308,"retries":0}},{"id":"file-12-jobs/scheduler.py","kind":"file_context","group":"file-12","file":"jobs/scheduler.py","expected":true,"state":"Agent task: The nightly export did not start at its scheduled time.\nSearch hit: jobs/scheduler.py — Starts scheduled background exports and maintenance jobs.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.494876,"confidence":0.505124,"probabilities":null,"latency_ms":36.02,"input_tokens":41,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.93,"confidence":0.93,"probabilities":null,"latency_ms":139.99,"input_tokens":308,"retries":0}},{"id":"file-12-api/rate_limit.py","kind":"file_context","group":"file-12","file":"api/rate_limit.py","expected":false,"state":"Agent task: The nightly export did not start at its scheduled time.\nSearch hit: api/rate_limit.py — Applies per-client throttling and 429 responses.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.141498,"confidence":0.858502,"probabilities":null,"latency_ms":36.06,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.28,"confidence":0.72,"probabilities":null,"latency_ms":103.78,"input_tokens":313,"retries":0}},{"id":"file-12-billing/invoices.py","kind":"file_context","group":"file-12","file":"billing/invoices.py","expected":false,"state":"Agent task: The nightly export did not start at its scheduled time.\nSearch hit: billing/invoices.py — Creates renewal invoices and records payment attempts.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.237081,"confidence":0.762919,"probabilities":null,"latency_ms":36.0,"input_tokens":41,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.35,"confidence":0.65,"probabilities":null,"latency_ms":132.13,"input_tokens":308,"retries":0}},{"id":"file-12-web/notifications.py","kind":"file_context","group":"file-12","file":"web/notifications.py","expected":false,"state":"Agent task: The nightly export did not start at its scheduled time.\nSearch hit: web/notifications.py — Formats and dispatches user notifications.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.197448,"confidence":0.802552,"probabilities":null,"latency_ms":34.35,"input_tokens":40,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.14,"confidence":0.86,"probabilities":null,"latency_ms":112.32,"input_tokens":308,"retries":0}},{"id":"file-13-storage/cache.py","kind":"file_context","group":"file-13","file":"storage/cache.py","expected":false,"state":"Agent task: A failed job loops forever instead of stopping after the retry limit.\nSearch hit: storage/cache.py — Caches reads and evicts old entries.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.262825,"confidence":0.737175,"probabilities":null,"latency_ms":64.17,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.11,"confidence":0.89,"probabilities":null,"latency_ms":96.99,"input_tokens":309,"retries":0}},{"id":"file-13-billing/invoices.py","kind":"file_context","group":"file-13","file":"billing/invoices.py","expected":false,"state":"Agent task: A failed job loops forever instead of stopping after the retry limit.\nSearch hit: billing/invoices.py — Creates renewal invoices and records payment attempts.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.261967,"confidence":0.738033,"probabilities":null,"latency_ms":37.17,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.24,"confidence":0.76,"probabilities":null,"latency_ms":113.27,"input_tokens":310,"retries":0}},{"id":"file-13-auth/session.py","kind":"file_context","group":"file-13","file":"auth/session.py","expected":false,"state":"Agent task: A failed job loops forever instead of stopping after the retry limit.\nSearch hit: auth/session.py — Creates, refreshes, and expires login sessions and tokens.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.165334,"confidence":0.834666,"probabilities":null,"latency_ms":37.22,"input_tokens":46,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.1,"confidence":0.9,"probabilities":null,"latency_ms":113.57,"input_tokens":313,"retries":0}},{"id":"file-13-jobs/retries.py","kind":"file_context","group":"file-13","file":"jobs/retries.py","expected":true,"state":"Agent task: A failed job loops forever instead of stopping after the retry limit.\nSearch hit: jobs/retries.py — Retries failed asynchronous jobs with backoff.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.372761,"confidence":0.627239,"probabilities":null,"latency_ms":37.99,"input_tokens":44,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.88,"confidence":0.88,"probabilities":null,"latency_ms":110.52,"input_tokens":310,"retries":0}},{"id":"file-13-api/rate_limit.py","kind":"file_context","group":"file-13","file":"api/rate_limit.py","expected":false,"state":"Agent task: A failed job loops forever instead of stopping after the retry limit.\nSearch hit: api/rate_limit.py — Applies per-client throttling and 429 responses.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.196734,"confidence":0.803266,"probabilities":null,"latency_ms":35.54,"input_tokens":48,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.11,"confidence":0.89,"probabilities":null,"latency_ms":92.05,"input_tokens":315,"retries":0}},{"id":"file-13-search/index.py","kind":"file_context","group":"file-13","file":"search/index.py","expected":false,"state":"Agent task: A failed job loops forever instead of stopping after the retry limit.\nSearch hit: search/index.py — Updates the search index from change events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.271687,"confidence":0.728313,"probabilities":null,"latency_ms":44.68,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.24,"confidence":0.76,"probabilities":null,"latency_ms":122.51,"input_tokens":308,"retries":0}},{"id":"file-13-web/notifications.py","kind":"file_context","group":"file-13","file":"web/notifications.py","expected":false,"state":"Agent task: A failed job loops forever instead of stopping after the retry limit.\nSearch hit: web/notifications.py — Formats and dispatches user notifications.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.163111,"confidence":0.836889,"probabilities":null,"latency_ms":33.84,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.1,"confidence":0.9,"probabilities":null,"latency_ms":116.37,"input_tokens":310,"retries":0}},{"id":"file-13-storage/migrations.py","kind":"file_context","group":"file-13","file":"storage/migrations.py","expected":false,"state":"Agent task: A failed job loops forever instead of stopping after the retry limit.\nSearch hit: storage/migrations.py — Runs and validates database schema migrations.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.296494,"confidence":0.703506,"probabilities":null,"latency_ms":34.2,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.25,"confidence":0.75,"probabilities":null,"latency_ms":99.69,"input_tokens":309,"retries":0}},{"id":"file-13-api/routes.py","kind":"file_context","group":"file-13","file":"api/routes.py","expected":false,"state":"Agent task: A failed job loops forever instead of stopping after the retry limit.\nSearch hit: api/routes.py — Maps HTTP paths to handlers and emits endpoint events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.250069,"confidence":0.749931,"probabilities":null,"latency_ms":34.08,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.16,"confidence":0.84,"probabilities":null,"latency_ms":92.35,"input_tokens":310,"retries":0}},{"id":"file-13-ops/metrics.py","kind":"file_context","group":"file-13","file":"ops/metrics.py","expected":false,"state":"Agent task: A failed job loops forever instead of stopping after the retry limit.\nSearch hit: ops/metrics.py — Records service counters and dashboard metrics.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.276468,"confidence":0.723532,"probabilities":null,"latency_ms":34.68,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.13,"confidence":0.87,"probabilities":null,"latency_ms":146.95,"input_tokens":308,"retries":0}},{"id":"file-13-auth/permissions.py","kind":"file_context","group":"file-13","file":"auth/permissions.py","expected":false,"state":"Agent task: A failed job loops forever instead of stopping after the retry limit.\nSearch hit: auth/permissions.py — Checks role membership and resource permissions.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.154382,"confidence":0.845618,"probabilities":null,"latency_ms":34.98,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.08,"confidence":0.92,"probabilities":null,"latency_ms":114.5,"input_tokens":310,"retries":0}},{"id":"file-13-search/query.py","kind":"file_context","group":"file-13","file":"search/query.py","expected":false,"state":"Agent task: A failed job loops forever instead of stopping after the retry limit.\nSearch hit: search/query.py — Ranks and filters search results at query time.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.255808,"confidence":0.744192,"probabilities":null,"latency_ms":34.82,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.08,"confidence":0.92,"probabilities":null,"latency_ms":119.97,"input_tokens":310,"retries":0}},{"id":"file-13-billing/refunds.py","kind":"file_context","group":"file-13","file":"billing/refunds.py","expected":false,"state":"Agent task: A failed job loops forever instead of stopping after the retry limit.\nSearch hit: billing/refunds.py — Processes refund requests and payment-provider callbacks.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.267731,"confidence":0.732269,"probabilities":null,"latency_ms":35.0,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.17,"confidence":0.83,"probabilities":null,"latency_ms":93.12,"input_tokens":309,"retries":0}},{"id":"file-13-web/ui.py","kind":"file_context","group":"file-13","file":"web/ui.py","expected":false,"state":"Agent task: A failed job loops forever instead of stopping after the retry limit.\nSearch hit: web/ui.py — Stores browser UI state and renders panels.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.199795,"confidence":0.800205,"probabilities":null,"latency_ms":34.73,"input_tokens":42,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.13,"confidence":0.87,"probabilities":null,"latency_ms":103.67,"input_tokens":308,"retries":0}},{"id":"file-13-jobs/scheduler.py","kind":"file_context","group":"file-13","file":"jobs/scheduler.py","expected":false,"state":"Agent task: A failed job loops forever instead of stopping after the retry limit.\nSearch hit: jobs/scheduler.py — Starts scheduled background exports and maintenance jobs.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.343082,"confidence":0.656918,"probabilities":null,"latency_ms":39.21,"input_tokens":43,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.78,"confidence":0.78,"probabilities":null,"latency_ms":103.33,"input_tokens":310,"retries":0}},{"id":"file-14-auth/session.py","kind":"file_context","group":"file-14","file":"auth/session.py","expected":false,"state":"Agent task: Checkout requests work, but the checkout metric is missing from dashboards.\nSearch hit: auth/session.py — Creates, refreshes, and expires login sessions and tokens.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.089582,"confidence":0.910418,"probabilities":null,"latency_ms":37.97,"input_tokens":47,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.1,"confidence":0.9,"probabilities":null,"latency_ms":149.48,"input_tokens":315,"retries":0}},{"id":"file-14-billing/invoices.py","kind":"file_context","group":"file-14","file":"billing/invoices.py","expected":false,"state":"Agent task: Checkout requests work, but the checkout metric is missing from dashboards.\nSearch hit: billing/invoices.py — Creates renewal invoices and records payment attempts.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.154363,"confidence":0.845637,"probabilities":null,"latency_ms":44.8,"input_tokens":44,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.61,"confidence":0.61,"probabilities":null,"latency_ms":123.64,"input_tokens":312,"retries":0}},{"id":"file-14-storage/migrations.py","kind":"file_context","group":"file-14","file":"storage/migrations.py","expected":false,"state":"Agent task: Checkout requests work, but the checkout metric is missing from dashboards.\nSearch hit: storage/migrations.py — Runs and validates database schema migrations.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.164392,"confidence":0.835608,"probabilities":null,"latency_ms":36.79,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.22,"confidence":0.78,"probabilities":null,"latency_ms":101.47,"input_tokens":311,"retries":0}},{"id":"file-14-api/routes.py","kind":"file_context","group":"file-14","file":"api/routes.py","expected":true,"state":"Agent task: Checkout requests work, but the checkout metric is missing from dashboards.\nSearch hit: api/routes.py — Maps HTTP paths to handlers and emits endpoint events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.241496,"confidence":0.758504,"probabilities":null,"latency_ms":41.71,"input_tokens":45,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.5,"confidence":0.5,"probabilities":null,"latency_ms":115.54,"input_tokens":312,"retries":0}},{"id":"file-14-billing/refunds.py","kind":"file_context","group":"file-14","file":"billing/refunds.py","expected":false,"state":"Agent task: Checkout requests work, but the checkout metric is missing from dashboards.\nSearch hit: billing/refunds.py — Processes refund requests and payment-provider callbacks.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.160804,"confidence":0.839196,"probabilities":null,"latency_ms":35.48,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.27,"confidence":0.73,"probabilities":null,"latency_ms":112.02,"input_tokens":311,"retries":0}},{"id":"file-14-jobs/retries.py","kind":"file_context","group":"file-14","file":"jobs/retries.py","expected":false,"state":"Agent task: Checkout requests work, but the checkout metric is missing from dashboards.\nSearch hit: jobs/retries.py — Retries failed asynchronous jobs with backoff.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.127487,"confidence":0.872513,"probabilities":null,"latency_ms":35.34,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.27,"confidence":0.73,"probabilities":null,"latency_ms":148.2,"input_tokens":312,"retries":0}},{"id":"file-14-api/rate_limit.py","kind":"file_context","group":"file-14","file":"api/rate_limit.py","expected":false,"state":"Agent task: Checkout requests work, but the checkout metric is missing from dashboards.\nSearch hit: api/rate_limit.py — Applies per-client throttling and 429 responses.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.099039,"confidence":0.900961,"probabilities":null,"latency_ms":35.01,"input_tokens":49,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.14,"confidence":0.86,"probabilities":null,"latency_ms":97.27,"input_tokens":317,"retries":0}},{"id":"file-14-auth/permissions.py","kind":"file_context","group":"file-14","file":"auth/permissions.py","expected":false,"state":"Agent task: Checkout requests work, but the checkout metric is missing from dashboards.\nSearch hit: auth/permissions.py — Checks role membership and resource permissions.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.113553,"confidence":0.886447,"probabilities":null,"latency_ms":38.31,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.2,"confidence":0.8,"probabilities":null,"latency_ms":103.17,"input_tokens":312,"retries":0}},{"id":"file-14-jobs/scheduler.py","kind":"file_context","group":"file-14","file":"jobs/scheduler.py","expected":false,"state":"Agent task: Checkout requests work, but the checkout metric is missing from dashboards.\nSearch hit: jobs/scheduler.py — Starts scheduled background exports and maintenance jobs.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.1867,"confidence":0.8133,"probabilities":null,"latency_ms":48.76,"input_tokens":44,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.54,"confidence":0.54,"probabilities":null,"latency_ms":114.09,"input_tokens":312,"retries":0}},{"id":"file-14-web/ui.py","kind":"file_context","group":"file-14","file":"web/ui.py","expected":false,"state":"Agent task: Checkout requests work, but the checkout metric is missing from dashboards.\nSearch hit: web/ui.py — Stores browser UI state and renders panels.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.247837,"confidence":0.752163,"probabilities":null,"latency_ms":36.38,"input_tokens":43,"retries":0},"jev":{"predicted":true,"correct":false,"score":0.6,"confidence":0.6,"probabilities":null,"latency_ms":109.66,"input_tokens":310,"retries":0}},{"id":"file-14-search/query.py","kind":"file_context","group":"file-14","file":"search/query.py","expected":false,"state":"Agent task: Checkout requests work, but the checkout metric is missing from dashboards.\nSearch hit: search/query.py — Ranks and filters search results at query time.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.1212,"confidence":0.8788,"probabilities":null,"latency_ms":36.07,"input_tokens":45,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.15,"confidence":0.85,"probabilities":null,"latency_ms":102.77,"input_tokens":312,"retries":0}},{"id":"file-14-ops/metrics.py","kind":"file_context","group":"file-14","file":"ops/metrics.py","expected":true,"state":"Agent task: Checkout requests work, but the checkout metric is missing from dashboards.\nSearch hit: ops/metrics.py — Records service counters and dashboard metrics.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":false,"score":0.355557,"confidence":0.644443,"probabilities":null,"latency_ms":44.94,"input_tokens":43,"retries":0},"jev":{"predicted":true,"correct":true,"score":0.9,"confidence":0.9,"probabilities":null,"latency_ms":148.62,"input_tokens":310,"retries":0}},{"id":"file-14-web/notifications.py","kind":"file_context","group":"file-14","file":"web/notifications.py","expected":false,"state":"Agent task: Checkout requests work, but the checkout metric is missing from dashboards.\nSearch hit: web/notifications.py — Formats and dispatches user notifications.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.108591,"confidence":0.891409,"probabilities":null,"latency_ms":55.74,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.14,"confidence":0.86,"probabilities":null,"latency_ms":110.14,"input_tokens":312,"retries":0}},{"id":"file-14-search/index.py","kind":"file_context","group":"file-14","file":"search/index.py","expected":false,"state":"Agent task: Checkout requests work, but the checkout metric is missing from dashboards.\nSearch hit: search/index.py — Updates the search index from change events.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.160988,"confidence":0.839012,"probabilities":null,"latency_ms":35.52,"input_tokens":43,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.12,"confidence":0.88,"probabilities":null,"latency_ms":105.12,"input_tokens":310,"retries":0}},{"id":"file-14-storage/cache.py","kind":"file_context","group":"file-14","file":"storage/cache.py","expected":false,"state":"Agent task: Checkout requests work, but the checkout metric is missing from dashboards.\nSearch hit: storage/cache.py — Caches reads and evicts old entries.","questions":{"check":{"type":"noul","instructions":"Is this file directly useful context for the current task?"}},"clm":{"predicted":false,"correct":true,"score":0.210419,"confidence":0.789581,"probabilities":null,"latency_ms":37.23,"input_tokens":44,"retries":0},"jev":{"predicted":false,"correct":true,"score":0.23,"confidence":0.77,"probabilities":null,"latency_ms":115.55,"input_tokens":311,"retries":0}}]}