{"date":"2026-09-24","source_report":"evaluation/results/2026-09-24/README.md","runs":{"clm_realtime.json":{"summary":{"model":"clm-latest","endpoint":"http://127.0.0.1:8700 · clm-latest · 6 requests in flight","mode":"realtime","shield":true,"course_style":"original","prompt":"labeled","duration_s":60.0,"seeds":5,"survived":5,"deaths":0,"mean_best_score":697.0,"mean_decisions":2848.8,"mean_agreement_with_planner":0.665,"latency_ms_p50_median":16.5,"model_ms_p50_median":2.3,"errors":0,"answers_discarded":1815,"shield_interventions":4060,"inflight":6},"results":[{"seed":0,"survived":true,"deaths":0,"best_score":697,"scores":[697],"decisions":2879,"agreement_with_planner":0.68,"latency_ms_p50":16.5,"latency_ms_p95":82.5,"model_ms_p50":2.5,"answers_discarded":649,"errors":0,"last_error":null,"best_effort_decisions":0,"shield_interventions":48,"arrival_saves":677,"emergency_saves":1,"input_tokens":48390,"game_seconds":60.0,"wall_seconds":60.0,"host_stall_seconds_dropped":0.0,"model":"clm-latest","endpoint":"http://127.0.0.1:8700 · clm-latest · 6 requests in flight"},{"seed":1,"survived":true,"deaths":0,"best_score":697,"scores":[697],"decisions":2452,"agreement_with_planner":0.665,"latency_ms_p50":16.5,"latency_ms_p95":67.2,"model_ms_p50":2.3,"answers_discarded":435,"errors":0,"last_error":null,"best_effort_decisions":22,"shield_interventions":77,"arrival_saves":651,"emergency_saves":14,"input_tokens":19627,"game_seconds":60.0,"wall_seconds":60.0,"host_stall_seconds_dropped":0.0,"model":"clm-latest","endpoint":"http://127.0.0.1:8700 · clm-latest · 6 requests in flight"},{"seed":2,"survived":true,"deaths":0,"best_score":697,"scores":[697],"decisions":2205,"agreement_with_planner":0.683,"latency_ms_p50":16.5,"latency_ms_p95":81.5,"model_ms_p50":2.3,"answers_discarded":360,"errors":0,"last_error":null,"best_effort_decisions":30,"shield_interventions":53,"arrival_saves":506,"emergency_saves":16,"input_tokens":10394,"game_seconds":60.0,"wall_seconds":60.0,"host_stall_seconds_dropped":0.0,"model":"clm-latest","endpoint":"http://127.0.0.1:8700 · clm-latest · 6 requests in flight"},{"seed":3,"survived":true,"deaths":0,"best_score":697,"scores":[697],"decisions":3248,"agreement_with_planner":0.656,"latency_ms_p50":16.5,"latency_ms_p95":48.4,"model_ms_p50":2.4,"answers_discarded":241,"errors":0,"last_error":null,"best_effort_decisions":3,"shield_interventions":75,"arrival_saves":876,"emergency_saves":0,"input_tokens":7983,"game_seconds":60.0,"wall_seconds":60.0,"host_stall_seconds_dropped":0.0,"model":"clm-latest","endpoint":"http://127.0.0.1:8700 · clm-latest · 6 requests in flight"},{"seed":4,"survived":true,"deaths":0,"best_score":697,"scores":[697],"decisions":3460,"agreement_with_planner":0.642,"latency_ms_p50":16.5,"latency_ms_p95":16.9,"model_ms_p50":2.3,"answers_discarded":130,"errors":0,"last_error":null,"best_effort_decisions":0,"shield_interventions":74,"arrival_saves":992,"emergency_saves":0,"input_tokens":5543,"game_seconds":60.0,"wall_seconds":60.0,"host_stall_seconds_dropped":0.0,"model":"clm-latest","endpoint":"http://127.0.0.1:8700 · clm-latest · 6 requests in flight"}]},"jev_realtime.json":{"summary":{"model":"jev-1.13.0","endpoint":"https://api.typesafe.ai · jev-1.13.0 · 6 requests in flight","mode":"realtime","shield":true,"course_style":"original","prompt":"labeled","duration_s":60.0,"seeds":5,"survived":5,"deaths":0,"mean_best_score":697.0,"mean_decisions":499.8,"mean_agreement_with_planner":0.955,"latency_ms_p50_median":282.9,"model_ms_p50_median":261.1,"errors":0,"answers_discarded":1318,"shield_interventions":63,"inflight":6},"results":[{"seed":0,"survived":true,"deaths":0,"best_score":697,"scores":[697],"decisions":612,"agreement_with_planner":0.964,"latency_ms_p50":282.9,"latency_ms_p95":317.9,"model_ms_p50":261.1,"answers_discarded":274,"errors":0,"last_error":null,"best_effort_decisions":0,"shield_interventions":0,"arrival_saves":1,"emergency_saves":1,"input_tokens":322424,"game_seconds":60.0,"wall_seconds":60.0,"host_stall_seconds_dropped":0.0,"model":"jev-1.13.0","endpoint":"https://api.typesafe.ai · jev-1.13.0 · 6 requests in flight"},{"seed":1,"survived":true,"deaths":0,"best_score":697,"scores":[697],"decisions":298,"agreement_with_planner":0.933,"latency_ms_p50":283.8,"latency_ms_p95":416.3,"model_ms_p50":261.3,"answers_discarded":257,"errors":0,"last_error":null,"best_effort_decisions":39,"shield_interventions":0,"arrival_saves":2,"emergency_saves":29,"input_tokens":201895,"game_seconds":60.0,"wall_seconds":60.0,"host_stall_seconds_dropped":0.0,"model":"jev-1.13.0","endpoint":"https://api.typesafe.ai · jev-1.13.0 · 6 requests in flight"},{"seed":2,"survived":true,"deaths":0,"best_score":697,"scores":[697],"decisions":414,"agreement_with_planner":0.949,"latency_ms_p50":284.4,"latency_ms_p95":383.4,"model_ms_p50":263.5,"answers_discarded":264,"errors":0,"last_error":null,"best_effort_decisions":30,"shield_interventions":0,"arrival_saves":7,"emergency_saves":11,"input_tokens":246843,"game_seconds":60.0,"wall_seconds":60.0,"host_stall_seconds_dropped":0.0,"model":"jev-1.13.0","endpoint":"https://api.typesafe.ai · jev-1.13.0 · 6 requests in flight"},{"seed":3,"survived":true,"deaths":0,"best_score":697,"scores":[697],"decisions":554,"agreement_with_planner":0.964,"latency_ms_p50":282.8,"latency_ms_p95":332.4,"model_ms_p50":258.5,"answers_discarded":257,"errors":0,"last_error":null,"best_effort_decisions":22,"shield_interventions":0,"arrival_saves":2,"emergency_saves":9,"input_tokens":295314,"game_seconds":60.0,"wall_seconds":60.0,"host_stall_seconds_dropped":0.0,"model":"jev-1.13.0","endpoint":"https://api.typesafe.ai · jev-1.13.0 · 6 requests in flight"},{"seed":4,"survived":true,"deaths":0,"best_score":697,"scores":[697],"decisions":621,"agreement_with_planner":0.966,"latency_ms_p50":282.7,"latency_ms_p95":317.1,"model_ms_p50":257.9,"answers_discarded":266,"errors":0,"last_error":null,"best_effort_decisions":0,"shield_interventions":0,"arrival_saves":0,"emergency_saves":1,"input_tokens":323034,"game_seconds":60.0,"wall_seconds":60.0,"host_stall_seconds_dropped":0.0,"model":"jev-1.13.0","endpoint":"https://api.typesafe.ai · jev-1.13.0 · 6 requests in flight"}]},"clm_noshield.json":{"summary":{"model":"clm-latest","endpoint":"http://127.0.0.1:8700 · clm-latest · 6 requests in flight","mode":"realtime","shield":false,"course_style":"original","prompt":"labeled","duration_s":30.0,"seeds":3,"survived":0,"deaths":8,"mean_best_score":106.0,"mean_decisions":1543.7,"mean_agreement_with_planner":0.893,"latency_ms_p50_median":16.5,"model_ms_p50_median":2.3,"errors":0,"answers_discarded":49,"shield_interventions":0,"inflight":6},"results":[{"seed":0,"survived":false,"deaths":3,"best_score":108,"scores":[83,108,42,2],"decisions":1494,"agreement_with_planner":0.897,"latency_ms_p50":16.5,"latency_ms_p95":16.7,"model_ms_p50":2.4,"answers_discarded":36,"errors":0,"last_error":null,"best_effort_decisions":12,"shield_interventions":0,"arrival_saves":0,"emergency_saves":0,"input_tokens":1015,"game_seconds":25.5,"wall_seconds":30.0,"host_stall_seconds_dropped":0.0,"model":"clm-latest","endpoint":"http://127.0.0.1:8700 · clm-latest · 6 requests in flight"},{"seed":1,"survived":false,"deaths":4,"best_score":52,"scores":[46,49,52,51,18],"decisions":1433,"agreement_with_planner":0.925,"latency_ms_p50":16.5,"latency_ms_p95":16.8,"model_ms_p50":2.3,"answers_discarded":7,"errors":0,"last_error":null,"best_effort_decisions":9,"shield_interventions":0,"arrival_saves":0,"emergency_saves":0,"input_tokens":219,"game_seconds":24.0,"wall_seconds":30.0,"host_stall_seconds_dropped":0.0,"model":"clm-latest","endpoint":"http://127.0.0.1:8700 · clm-latest · 6 requests in flight"},{"seed":2,"survived":false,"deaths":1,"best_score":158,"scores":[158,112],"decisions":1704,"agreement_with_planner":0.858,"latency_ms_p50":16.5,"latency_ms_p95":16.8,"model_ms_p50":2.3,"answers_discarded":6,"errors":0,"last_error":null,"best_effort_decisions":2,"shield_interventions":0,"arrival_saves":0,"emergency_saves":0,"input_tokens":216,"game_seconds":28.5,"wall_seconds":30.0,"host_stall_seconds_dropped":0.0,"model":"clm-latest","endpoint":"http://127.0.0.1:8700 · clm-latest · 6 requests in flight"}]},"jev_noshield.json":{"summary":{"model":"jev-1.13.0","endpoint":"https://api.typesafe.ai · jev-1.13.0 · 6 requests in flight","mode":"realtime","shield":false,"course_style":"original","prompt":"labeled","duration_s":30.0,"seeds":3,"survived":2,"deaths":1,"mean_best_score":277.0,"mean_decisions":318.7,"mean_agreement_with_planner":0.986,"latency_ms_p50_median":283.2,"model_ms_p50_median":265.1,"errors":0,"answers_discarded":326,"shield_interventions":0,"inflight":6},"results":[{"seed":0,"survived":true,"deaths":0,"best_score":305,"scores":[305],"decisions":322,"agreement_with_planner":0.984,"latency_ms_p50":284.5,"latency_ms_p95":333.0,"model_ms_p50":265.1,"answers_discarded":112,"errors":0,"last_error":null,"best_effort_decisions":2,"shield_interventions":0,"arrival_saves":0,"emergency_saves":0,"input_tokens":157958,"game_seconds":30.0,"wall_seconds":30.0,"host_stall_seconds_dropped":0.0,"model":"jev-1.13.0","endpoint":"https://api.typesafe.ai · jev-1.13.0 · 6 requests in flight"},{"seed":1,"survived":true,"deaths":0,"best_score":305,"scores":[305],"decisions":316,"agreement_with_planner":0.984,"latency_ms_p50":283.2,"latency_ms_p95":333.1,"model_ms_p50":259.8,"answers_discarded":120,"errors":0,"last_error":null,"best_effort_decisions":0,"shield_interventions":0,"arrival_saves":0,"emergency_saves":0,"input_tokens":158631,"game_seconds":30.0,"wall_seconds":30.0,"host_stall_seconds_dropped":0.0,"model":"jev-1.13.0","endpoint":"https://api.typesafe.ai · jev-1.13.0 · 6 requests in flight"},{"seed":2,"survived":false,"deaths":1,"best_score":221,"scores":[221,55],"decisions":318,"agreement_with_planner":0.991,"latency_ms_p50":283.1,"latency_ms_p95":318.3,"model_ms_p50":265.9,"answers_discarded":94,"errors":0,"last_error":null,"best_effort_decisions":4,"shield_interventions":0,"arrival_saves":0,"emergency_saves":0,"input_tokens":149086,"game_seconds":28.5,"wall_seconds":30.0,"host_stall_seconds_dropped":0.0,"model":"jev-1.13.0","endpoint":"https://api.typesafe.ai · jev-1.13.0 · 6 requests in flight"}]},"clm_tickets.json":{"model":"clm-latest","endpoint":"http://127.0.0.1:8700","benchmark":"support_ticket_routing","passes":{"new_states":{"count":40,"correct":27,"latency_ms_p50":35.39,"latency_ms_p95":37.18,"latency_ms_mean":35.53,"input_tokens":844,"rows":[{"expected":"sales","predicted":"sales","latency_ms":35.46,"server_ms":33.3,"input_tokens":22},{"expected":"account","predicted":"account","latency_ms":35.41,"server_ms":33.6,"input_tokens":20},{"expected":"technical","predicted":"account","latency_ms":35.2,"server_ms":33.5,"input_tokens":21},{"expected":"sales","predicted":"sales","latency_ms":35.43,"server_ms":33.7,"input_tokens":23},{"expected":"technical","predicted":"sales","latency_ms":34.98,"server_ms":33.2,"input_tokens":20},{"expected":"technical","predicted":"sales","latency_ms":35.31,"server_ms":33.5,"input_tokens":20},{"expected":"account","predicted":"account","latency_ms":35.68,"server_ms":33.8,"input_tokens":21},{"expected":"sales","predicted":"sales","latency_ms":35.37,"server_ms":33.5,"input_tokens":20},{"expected":"sales","predicted":"sales","latency_ms":34.9,"server_ms":33.0,"input_tokens":18},{"expected":"sales","predicted":"sales","latency_ms":35.08,"server_ms":33.2,"input_tokens":22},{"expected":"billing","predicted":"account","latency_ms":35.53,"server_ms":33.7,"input_tokens":20},{"expected":"technical","predicted":"account","latency_ms":37.18,"server_ms":35.1,"input_tokens":22},{"expected":"billing","predicted":"billing","latency_ms":36.35,"server_ms":34.4,"input_tokens":24},{"expected":"technical","predicted":"account","latency_ms":37.36,"server_ms":33.7,"input_tokens":19},{"expected":"account","predicted":"account","latency_ms":35.26,"server_ms":33.4,"input_tokens":20},{"expected":"technical","predicted":"account","latency_ms":34.83,"server_ms":33.0,"input_tokens":21},{"expected":"sales","predicted":"sales","latency_ms":35.3,"server_ms":33.5,"input_tokens":23},{"expected":"billing","predicted":"billing","latency_ms":35.82,"server_ms":34.0,"input_tokens":20},{"expected":"account","predicted":"account","latency_ms":35.13,"server_ms":33.3,"input_tokens":20},{"expected":"account","predicted":"account","latency_ms":36.44,"server_ms":34.5,"input_tokens":22},{"expected":"billing","predicted":"billing","latency_ms":35.85,"server_ms":33.9,"input_tokens":23},{"expected":"billing","predicted":"account","latency_ms":35.79,"server_ms":33.9,"input_tokens":22},{"expected":"billing","predicted":"billing","latency_ms":35.39,"server_ms":33.5,"input_tokens":20},{"expected":"sales","predicted":"sales","latency_ms":35.08,"server_ms":33.2,"input_tokens":21},{"expected":"billing","predicted":"billing","latency_ms":35.32,"server_ms":33.4,"input_tokens":22},{"expected":"account","predicted":"account","latency_ms":35.54,"server_ms":33.7,"input_tokens":23},{"expected":"technical","predicted":"account","latency_ms":35.16,"server_ms":33.3,"input_tokens":21},{"expected":"technical","predicted":"account","latency_ms":36.36,"server_ms":34.4,"input_tokens":25},{"expected":"technical","predicted":"account","latency_ms":35.93,"server_ms":34.0,"input_tokens":22},{"expected":"sales","predicted":"account","latency_ms":35.25,"server_ms":33.4,"input_tokens":20},{"expected":"account","predicted":"account","latency_ms":35.1,"server_ms":33.3,"input_tokens":19},{"expected":"account","predicted":"account","latency_ms":35.21,"server_ms":33.3,"input_tokens":19},{"expected":"account","predicted":"account","latency_ms":35.57,"server_ms":33.4,"input_tokens":22},{"expected":"technical","predicted":"account","latency_ms":34.98,"server_ms":33.1,"input_tokens":19},{"expected":"account","predicted":"account","latency_ms":35.35,"server_ms":33.5,"input_tokens":23},{"expected":"billing","predicted":"billing","latency_ms":36.15,"server_ms":34.2,"input_tokens":22},{"expected":"sales","predicted":"sales","latency_ms":35.81,"server_ms":33.9,"input_tokens":21},{"expected":"billing","predicted":"billing","latency_ms":35.6,"server_ms":33.8,"input_tokens":21},{"expected":"sales","predicted":"sales","latency_ms":34.89,"server_ms":33.0,"input_tokens":20},{"expected":"billing","predicted":"billing","latency_ms":35.03,"server_ms":33.2,"input_tokens":21}]},"repeated_states":{"count":40,"correct":27,"latency_ms_p50":1.95,"latency_ms_p95":2.43,"latency_ms_mean":2.01,"input_tokens":0,"rows":[{"expected":"sales","predicted":"sales","latency_ms":2.45,"server_ms":0.6,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":2.4,"server_ms":0.6,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":2.43,"server_ms":0.6,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":2.28,"server_ms":0.5,"input_tokens":0},{"expected":"technical","predicted":"sales","latency_ms":2.27,"server_ms":0.5,"input_tokens":0},{"expected":"technical","predicted":"sales","latency_ms":2.2,"server_ms":0.5,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":2.08,"server_ms":0.5,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":2.0,"server_ms":0.5,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":1.9,"server_ms":0.4,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":1.97,"server_ms":0.5,"input_tokens":0},{"expected":"billing","predicted":"account","latency_ms":1.96,"server_ms":0.5,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":1.95,"server_ms":0.5,"input_tokens":0},{"expected":"billing","predicted":"billing","latency_ms":2.22,"server_ms":0.5,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":1.95,"server_ms":0.5,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":1.99,"server_ms":0.5,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":1.93,"server_ms":0.5,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":1.97,"server_ms":0.5,"input_tokens":0},{"expected":"billing","predicted":"billing","latency_ms":1.95,"server_ms":0.5,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":2.02,"server_ms":0.6,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":1.96,"server_ms":0.5,"input_tokens":0},{"expected":"billing","predicted":"billing","latency_ms":1.93,"server_ms":0.5,"input_tokens":0},{"expected":"billing","predicted":"account","latency_ms":1.91,"server_ms":0.5,"input_tokens":0},{"expected":"billing","predicted":"billing","latency_ms":1.95,"server_ms":0.5,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":1.89,"server_ms":0.5,"input_tokens":0},{"expected":"billing","predicted":"billing","latency_ms":1.93,"server_ms":0.5,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":1.83,"server_ms":0.4,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":1.97,"server_ms":0.6,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":1.92,"server_ms":0.5,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":1.87,"server_ms":0.5,"input_tokens":0},{"expected":"sales","predicted":"account","latency_ms":1.84,"server_ms":0.4,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":1.92,"server_ms":0.5,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":1.83,"server_ms":0.4,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":1.92,"server_ms":0.5,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":2.21,"server_ms":0.5,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":1.94,"server_ms":0.5,"input_tokens":0},{"expected":"billing","predicted":"billing","latency_ms":1.96,"server_ms":0.5,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":1.9,"server_ms":0.5,"input_tokens":0},{"expected":"billing","predicted":"billing","latency_ms":1.86,"server_ms":0.4,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":1.99,"server_ms":0.5,"input_tokens":0},{"expected":"billing","predicted":"billing","latency_ms":1.8,"server_ms":0.4,"input_tokens":0}]}}},"jev_tickets.json":{"model":"jev-1.13.0","endpoint":"https://api.typesafe.ai","benchmark":"support_ticket_routing","passes":{"new_states":{"count":40,"correct":40,"latency_ms_p50":252.49,"latency_ms_p95":287.68,"latency_ms_mean":255.89,"input_tokens":15924,"rows":[{"expected":"sales","predicted":"sales","latency_ms":245.65,"server_ms":null,"input_tokens":399},{"expected":"account","predicted":"account","latency_ms":240.42,"server_ms":null,"input_tokens":397},{"expected":"technical","predicted":"technical","latency_ms":248.63,"server_ms":null,"input_tokens":398},{"expected":"sales","predicted":"sales","latency_ms":267.07,"server_ms":null,"input_tokens":400},{"expected":"technical","predicted":"technical","latency_ms":238.75,"server_ms":null,"input_tokens":397},{"expected":"technical","predicted":"technical","latency_ms":250.49,"server_ms":null,"input_tokens":397},{"expected":"account","predicted":"account","latency_ms":242.79,"server_ms":null,"input_tokens":398},{"expected":"sales","predicted":"sales","latency_ms":252.22,"server_ms":null,"input_tokens":397},{"expected":"sales","predicted":"sales","latency_ms":258.78,"server_ms":null,"input_tokens":395},{"expected":"sales","predicted":"sales","latency_ms":240.14,"server_ms":null,"input_tokens":399},{"expected":"billing","predicted":"billing","latency_ms":243.48,"server_ms":null,"input_tokens":397},{"expected":"technical","predicted":"technical","latency_ms":287.68,"server_ms":null,"input_tokens":399},{"expected":"billing","predicted":"billing","latency_ms":241.78,"server_ms":null,"input_tokens":401},{"expected":"technical","predicted":"technical","latency_ms":244.78,"server_ms":null,"input_tokens":396},{"expected":"account","predicted":"account","latency_ms":264.48,"server_ms":null,"input_tokens":397},{"expected":"technical","predicted":"technical","latency_ms":266.38,"server_ms":null,"input_tokens":398},{"expected":"sales","predicted":"sales","latency_ms":249.51,"server_ms":null,"input_tokens":400},{"expected":"billing","predicted":"billing","latency_ms":252.49,"server_ms":null,"input_tokens":397},{"expected":"account","predicted":"account","latency_ms":284.17,"server_ms":null,"input_tokens":397},{"expected":"account","predicted":"account","latency_ms":245.44,"server_ms":null,"input_tokens":399},{"expected":"billing","predicted":"billing","latency_ms":246.1,"server_ms":null,"input_tokens":400},{"expected":"billing","predicted":"billing","latency_ms":249.73,"server_ms":null,"input_tokens":399},{"expected":"billing","predicted":"billing","latency_ms":259.07,"server_ms":null,"input_tokens":397},{"expected":"sales","predicted":"sales","latency_ms":239.07,"server_ms":null,"input_tokens":398},{"expected":"billing","predicted":"billing","latency_ms":236.33,"server_ms":null,"input_tokens":399},{"expected":"account","predicted":"account","latency_ms":265.77,"server_ms":null,"input_tokens":400},{"expected":"technical","predicted":"technical","latency_ms":265.99,"server_ms":null,"input_tokens":398},{"expected":"technical","predicted":"technical","latency_ms":257.09,"server_ms":null,"input_tokens":402},{"expected":"technical","predicted":"technical","latency_ms":263.54,"server_ms":null,"input_tokens":399},{"expected":"sales","predicted":"sales","latency_ms":272.16,"server_ms":null,"input_tokens":397},{"expected":"account","predicted":"account","latency_ms":242.56,"server_ms":null,"input_tokens":396},{"expected":"account","predicted":"account","latency_ms":255.93,"server_ms":null,"input_tokens":396},{"expected":"account","predicted":"account","latency_ms":240.47,"server_ms":null,"input_tokens":399},{"expected":"technical","predicted":"technical","latency_ms":291.86,"server_ms":null,"input_tokens":396},{"expected":"account","predicted":"account","latency_ms":279.96,"server_ms":null,"input_tokens":400},{"expected":"billing","predicted":"billing","latency_ms":258.11,"server_ms":null,"input_tokens":399},{"expected":"sales","predicted":"sales","latency_ms":244.37,"server_ms":null,"input_tokens":398},{"expected":"billing","predicted":"billing","latency_ms":267.88,"server_ms":null,"input_tokens":398},{"expected":"sales","predicted":"sales","latency_ms":270.2,"server_ms":null,"input_tokens":397},{"expected":"billing","predicted":"billing","latency_ms":264.39,"server_ms":null,"input_tokens":398}]},"repeated_states":{"count":40,"correct":40,"latency_ms_p50":263.1,"latency_ms_p95":305.3,"latency_ms_mean":266.56,"input_tokens":15924,"rows":[{"expected":"sales","predicted":"sales","latency_ms":305.3,"server_ms":null,"input_tokens":399},{"expected":"account","predicted":"account","latency_ms":289.28,"server_ms":null,"input_tokens":397},{"expected":"technical","predicted":"technical","latency_ms":245.95,"server_ms":null,"input_tokens":398},{"expected":"sales","predicted":"sales","latency_ms":296.52,"server_ms":null,"input_tokens":400},{"expected":"technical","predicted":"technical","latency_ms":253.13,"server_ms":null,"input_tokens":397},{"expected":"technical","predicted":"technical","latency_ms":278.99,"server_ms":null,"input_tokens":397},{"expected":"account","predicted":"account","latency_ms":241.3,"server_ms":null,"input_tokens":398},{"expected":"sales","predicted":"sales","latency_ms":269.31,"server_ms":null,"input_tokens":397},{"expected":"sales","predicted":"sales","latency_ms":270.91,"server_ms":null,"input_tokens":395},{"expected":"sales","predicted":"sales","latency_ms":238.9,"server_ms":null,"input_tokens":399},{"expected":"billing","predicted":"billing","latency_ms":240.17,"server_ms":null,"input_tokens":397},{"expected":"technical","predicted":"technical","latency_ms":261.11,"server_ms":null,"input_tokens":399},{"expected":"billing","predicted":"billing","latency_ms":243.67,"server_ms":null,"input_tokens":401},{"expected":"technical","predicted":"technical","latency_ms":262.59,"server_ms":null,"input_tokens":396},{"expected":"account","predicted":"account","latency_ms":253.11,"server_ms":null,"input_tokens":397},{"expected":"technical","predicted":"technical","latency_ms":252.49,"server_ms":null,"input_tokens":398},{"expected":"sales","predicted":"sales","latency_ms":291.11,"server_ms":null,"input_tokens":400},{"expected":"billing","predicted":"billing","latency_ms":252.16,"server_ms":null,"input_tokens":397},{"expected":"account","predicted":"account","latency_ms":294.76,"server_ms":null,"input_tokens":397},{"expected":"account","predicted":"account","latency_ms":244.19,"server_ms":null,"input_tokens":399},{"expected":"billing","predicted":"billing","latency_ms":301.55,"server_ms":null,"input_tokens":400},{"expected":"billing","predicted":"billing","latency_ms":239.41,"server_ms":null,"input_tokens":399},{"expected":"billing","predicted":"billing","latency_ms":246.67,"server_ms":null,"input_tokens":397},{"expected":"sales","predicted":"sales","latency_ms":241.68,"server_ms":null,"input_tokens":398},{"expected":"billing","predicted":"billing","latency_ms":246.64,"server_ms":null,"input_tokens":399},{"expected":"account","predicted":"account","latency_ms":269.93,"server_ms":null,"input_tokens":400},{"expected":"technical","predicted":"technical","latency_ms":294.01,"server_ms":null,"input_tokens":398},{"expected":"technical","predicted":"technical","latency_ms":279.87,"server_ms":null,"input_tokens":402},{"expected":"technical","predicted":"technical","latency_ms":243.37,"server_ms":null,"input_tokens":399},{"expected":"sales","predicted":"sales","latency_ms":288.43,"server_ms":null,"input_tokens":397},{"expected":"account","predicted":"account","latency_ms":268.67,"server_ms":null,"input_tokens":396},{"expected":"account","predicted":"account","latency_ms":257.83,"server_ms":null,"input_tokens":396},{"expected":"account","predicted":"account","latency_ms":264.99,"server_ms":null,"input_tokens":399},{"expected":"technical","predicted":"technical","latency_ms":259.51,"server_ms":null,"input_tokens":396},{"expected":"account","predicted":"account","latency_ms":293.67,"server_ms":null,"input_tokens":400},{"expected":"billing","predicted":"billing","latency_ms":257.01,"server_ms":null,"input_tokens":399},{"expected":"sales","predicted":"sales","latency_ms":266.46,"server_ms":null,"input_tokens":398},{"expected":"billing","predicted":"billing","latency_ms":266.81,"server_ms":null,"input_tokens":398},{"expected":"sales","predicted":"sales","latency_ms":327.79,"server_ms":null,"input_tokens":397},{"expected":"billing","predicted":"billing","latency_ms":263.1,"server_ms":null,"input_tokens":398}]}}},"clm_cuda_tickets.json":{"model":"clm-latest","endpoint":"http://127.0.0.1:8701","benchmark":"support_ticket_routing","passes":{"new_states":{"count":40,"correct":26,"latency_ms_p50":29.86,"latency_ms_p95":35.34,"latency_ms_mean":30.09,"input_tokens":844,"rows":[{"expected":"sales","predicted":"sales","latency_ms":30.15,"server_ms":28.5,"input_tokens":22},{"expected":"account","predicted":"account","latency_ms":35.34,"server_ms":34.2,"input_tokens":20},{"expected":"technical","predicted":"account","latency_ms":28.56,"server_ms":27.4,"input_tokens":21},{"expected":"sales","predicted":"sales","latency_ms":33.33,"server_ms":32.3,"input_tokens":23},{"expected":"technical","predicted":"sales","latency_ms":38.75,"server_ms":37.4,"input_tokens":20},{"expected":"technical","predicted":"sales","latency_ms":29.92,"server_ms":28.4,"input_tokens":20},{"expected":"account","predicted":"account","latency_ms":30.45,"server_ms":28.7,"input_tokens":21},{"expected":"sales","predicted":"sales","latency_ms":29.17,"server_ms":27.3,"input_tokens":20},{"expected":"sales","predicted":"sales","latency_ms":29.44,"server_ms":28.0,"input_tokens":18},{"expected":"sales","predicted":"sales","latency_ms":28.21,"server_ms":27.1,"input_tokens":22},{"expected":"billing","predicted":"account","latency_ms":28.29,"server_ms":27.0,"input_tokens":20},{"expected":"technical","predicted":"account","latency_ms":29.29,"server_ms":28.0,"input_tokens":22},{"expected":"billing","predicted":"billing","latency_ms":29.56,"server_ms":27.9,"input_tokens":24},{"expected":"technical","predicted":"account","latency_ms":30.66,"server_ms":27.6,"input_tokens":19},{"expected":"account","predicted":"account","latency_ms":29.52,"server_ms":27.5,"input_tokens":20},{"expected":"technical","predicted":"account","latency_ms":29.62,"server_ms":27.9,"input_tokens":21},{"expected":"sales","predicted":"sales","latency_ms":29.87,"server_ms":28.1,"input_tokens":23},{"expected":"billing","predicted":"billing","latency_ms":29.95,"server_ms":28.2,"input_tokens":20},{"expected":"account","predicted":"account","latency_ms":29.13,"server_ms":27.4,"input_tokens":20},{"expected":"account","predicted":"account","latency_ms":30.5,"server_ms":28.8,"input_tokens":22},{"expected":"billing","predicted":"billing","latency_ms":29.82,"server_ms":28.0,"input_tokens":23},{"expected":"billing","predicted":"account","latency_ms":29.99,"server_ms":28.1,"input_tokens":22},{"expected":"billing","predicted":"billing","latency_ms":29.93,"server_ms":28.1,"input_tokens":20},{"expected":"sales","predicted":"sales","latency_ms":29.88,"server_ms":28.1,"input_tokens":21},{"expected":"billing","predicted":"sales","latency_ms":29.86,"server_ms":28.0,"input_tokens":22},{"expected":"account","predicted":"account","latency_ms":29.23,"server_ms":27.4,"input_tokens":23},{"expected":"technical","predicted":"account","latency_ms":29.68,"server_ms":27.9,"input_tokens":21},{"expected":"technical","predicted":"account","latency_ms":30.01,"server_ms":28.2,"input_tokens":25},{"expected":"technical","predicted":"account","latency_ms":28.5,"server_ms":26.6,"input_tokens":22},{"expected":"sales","predicted":"account","latency_ms":28.94,"server_ms":27.5,"input_tokens":20},{"expected":"account","predicted":"account","latency_ms":27.87,"server_ms":26.5,"input_tokens":19},{"expected":"account","predicted":"account","latency_ms":29.34,"server_ms":27.5,"input_tokens":19},{"expected":"account","predicted":"account","latency_ms":29.19,"server_ms":27.1,"input_tokens":22},{"expected":"technical","predicted":"account","latency_ms":29.79,"server_ms":28.0,"input_tokens":19},{"expected":"account","predicted":"account","latency_ms":29.48,"server_ms":27.7,"input_tokens":23},{"expected":"billing","predicted":"billing","latency_ms":30.27,"server_ms":28.4,"input_tokens":22},{"expected":"sales","predicted":"sales","latency_ms":30.26,"server_ms":28.5,"input_tokens":21},{"expected":"billing","predicted":"billing","latency_ms":30.4,"server_ms":28.6,"input_tokens":21},{"expected":"sales","predicted":"sales","latency_ms":30.4,"server_ms":28.6,"input_tokens":20},{"expected":"billing","predicted":"billing","latency_ms":31.22,"server_ms":29.4,"input_tokens":21}]},"repeated_states":{"count":40,"correct":26,"latency_ms_p50":2.1,"latency_ms_p95":2.55,"latency_ms_mean":2.17,"input_tokens":0,"rows":[{"expected":"sales","predicted":"sales","latency_ms":2.62,"server_ms":0.8,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":2.55,"server_ms":0.7,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":2.5,"server_ms":0.7,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":2.51,"server_ms":0.7,"input_tokens":0},{"expected":"technical","predicted":"sales","latency_ms":2.54,"server_ms":0.7,"input_tokens":0},{"expected":"technical","predicted":"sales","latency_ms":2.32,"server_ms":0.7,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":2.13,"server_ms":0.6,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":2.31,"server_ms":0.8,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":2.21,"server_ms":0.7,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":2.19,"server_ms":0.7,"input_tokens":0},{"expected":"billing","predicted":"account","latency_ms":2.15,"server_ms":0.8,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":2.19,"server_ms":0.8,"input_tokens":0},{"expected":"billing","predicted":"billing","latency_ms":2.49,"server_ms":0.8,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":2.21,"server_ms":0.8,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":2.2,"server_ms":0.8,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":2.2,"server_ms":0.8,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":2.12,"server_ms":0.7,"input_tokens":0},{"expected":"billing","predicted":"billing","latency_ms":2.15,"server_ms":0.8,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":2.04,"server_ms":0.7,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":2.06,"server_ms":0.7,"input_tokens":0},{"expected":"billing","predicted":"billing","latency_ms":2.06,"server_ms":0.7,"input_tokens":0},{"expected":"billing","predicted":"account","latency_ms":2.1,"server_ms":0.7,"input_tokens":0},{"expected":"billing","predicted":"billing","latency_ms":2.03,"server_ms":0.6,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":2.03,"server_ms":0.6,"input_tokens":0},{"expected":"billing","predicted":"sales","latency_ms":2.05,"server_ms":0.6,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":2.03,"server_ms":0.6,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":2.02,"server_ms":0.6,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":2.03,"server_ms":0.6,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":2.06,"server_ms":0.6,"input_tokens":0},{"expected":"sales","predicted":"account","latency_ms":2.05,"server_ms":0.7,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":2.03,"server_ms":0.6,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":2.05,"server_ms":0.6,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":2.05,"server_ms":0.6,"input_tokens":0},{"expected":"technical","predicted":"account","latency_ms":2.35,"server_ms":0.6,"input_tokens":0},{"expected":"account","predicted":"account","latency_ms":2.05,"server_ms":0.6,"input_tokens":0},{"expected":"billing","predicted":"billing","latency_ms":2.02,"server_ms":0.6,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":2.02,"server_ms":0.6,"input_tokens":0},{"expected":"billing","predicted":"billing","latency_ms":2.02,"server_ms":0.6,"input_tokens":0},{"expected":"sales","predicted":"sales","latency_ms":2.02,"server_ms":0.6,"input_tokens":0},{"expected":"billing","predicted":"billing","latency_ms":2.02,"server_ms":0.6,"input_tokens":0}]}}}}}
