{ "title": "GLM-5.3-Flash SGLang rescue on four H200 NVL GPUs", "completed_utc": "2026-10-12", "published": "2026-10-13", "audit_errors": [], "caveats": [ "Finite short-chat tests, not a long-duration soak or long-context qualification.", "Cache-enabled repeats reuse prompts; report each repeat as well as medians.", "Two isolated GPU quads ran concurrently on the same host; not an isolated performance A/B.", "Output-token counts include reasoning tokens; 256-token generation limit." ], "sources": [ { "url": "https://github.com/sgl-project/sglang/blob/main/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx", "use": "Official H200 TP8/EP8 and EAGLE5/topk1/6-token guidance; adapted to a four-GPU NVLink group, not an exact official recipe reproduction." }, { "url": "https://github.com/sgl-project/sglang/blob/main/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx", "use": "Keep prefix caching enabled; distinguish KV and KDA state pools." }, { "url": "https://github.com/sgl-project/sglang/issues/37712", "use": "H200 long-prefill OOM report motivated 4096 prefill chunks and 0.78 static memory; original failure here is illegal memory access, not established OOM." }, { "url": "https://docs.nvidia.com/dynamo/v1.5.0/recipes/glm-5-3-flash", "use": "Additional TP8/MTP7/FLASHMLA_SPARSE direction; not tested in this round." } ], "archive": { "name": "h200-glm-sglang-rescue-20261012.tar.zst", "sha256": "7fa12f22c16e2ac71f8bc4dc9d749d4cefafa49b6b2d8214a43518e2e716911e" }, "runs": [ { "name": "reproduce", "state": "failed", "raw_completed": 2, "raw_failed": 1, "cuda_illegal_memory": true, "quality": { "structured_before.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true } }, "medians": {}, "reason": null, "config": { "hf_repo": "zai-org/GLM-5.3-Flash", "model_revision": "eb9eb208eb0d988989d07a6a12d0fdeb5f52574a", "image_digest": "lmsysorg/sglang@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", "engine_version": "v0.5.21", "tp": 4, "ep": 1, "gpus": 4, "cuda_devices": "0,1,2,3", "weight_dtype": "fp8", "kv_dtype": "bfloat16", "max_model_len": 32768, "max_num_batched_tokens": 8192, "memory_fraction": 0.85, "prefix_caching": false, "disable_cuda_graph": false, "disable_overlap_schedule": false, "requested_max_running_requests": 64, "effective_max_running_requests": 64, "effective_state_pool_slots": 64, "engine_kv_tokens": 3343552, "spec_method": null, "spec_num_steps": null, "spec_num_tokens": null }, "points": [] }, { "name": "no-cuda-graphs", "state": "diagnostic_pass", "raw_completed": 808, "raw_failed": 0, "cuda_illegal_memory": false, "quality": { "structured_after-first-sweep.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true }, "structured_before.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true } }, "medians": { "1": { "repeats": 1, "output_tok_s": 7.127764236328813, "ttft_p99_s": 22.738563086616338, "latency_p99_s": 54.14256675332115 }, "16": { "repeats": 1, "output_tok_s": 83.34489949242561, "ttft_p99_s": 24.560182506071868, "latency_p99_s": 55.63449692468246 }, "64": { "repeats": 1, "output_tok_s": 351.8727573605711, "ttft_p99_s": 15.630398686900977, "latency_p99_s": 55.075061435559476 }, "128": { "repeats": 1, "output_tok_s": 381.58544791469023, "ttft_p99_s": 66.34375637894759, "latency_p99_s": 100.22422587619978 } }, "reason": "One complete C1/C16/C64/C128 sweep plus before/after32 correctness checks passed. Further repeats intentionally omitted because graph-disabled single-request throughput was about7tok/s; diagnostic fallback, not a performance candidate.", "config": { "hf_repo": "zai-org/GLM-5.3-Flash", "model_revision": "eb9eb208eb0d988989d07a6a12d0fdeb5f52574a", "image_digest": "lmsysorg/sglang@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", "engine_version": "v0.5.21", "tp": 4, "ep": 1, "gpus": 4, "cuda_devices": "0,1,2,3", "weight_dtype": "fp8", "kv_dtype": "bfloat16", "max_model_len": 32768, "max_num_batched_tokens": 8192, "memory_fraction": 0.85, "prefix_caching": false, "disable_cuda_graph": true, "disable_overlap_schedule": false, "requested_max_running_requests": 64, "effective_max_running_requests": 64, "effective_state_pool_slots": 64, "engine_kv_tokens": 3343552, "spec_method": null, "spec_num_steps": null, "spec_num_tokens": null }, "points": [ { "file": "r1-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129456, "output_tok_s": 381.58544791469023, "ttft_s": { "mean": 39.11297443829483, "p50": 36.25351455299824, "p90": 65.54042534709443, "p99": 66.34375637894759 }, "latency_s": { "mean": 77.7932884003892, "p50": 79.85938955300662, "p90": 100.19881022168265, "p99": 100.22422587619978 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 83.34489949242561, "ttft_s": { "mean": 16.724196845967526, "p50": 16.377652564493474, "p90": 21.860063897282817, "p99": 24.560182506071868 }, "latency_s": { "mean": 49.13745952684167, "p50": 49.1382106319943, "p90": 55.63348414349021, "p99": 55.63449692468246 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 7.127764236328813, "ttft_s": { "mean": 4.705665293127822, "p50": 0.2679350265098037, "p90": 14.551987214101244, "p99": 22.738563086616338 }, "latency_s": { "mean": 35.91542971751187, "p50": 31.847790159503347, "p90": 45.7358327610098, "p99": 54.14256675332115 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64677, "output_tok_s": 351.8727573605711, "ttft_s": { "mean": 3.521674936512113, "p50": 1.159026305496809, "p90": 12.42558311499306, "p99": 15.630398686900977 }, "latency_s": { "mean": 44.09513838454086, "p50": 44.52212094599963, "p90": 55.06858866651601, "p99": 55.075061435559476 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } } ] }, { "name": "no-overlap", "state": "failed", "raw_completed": 0, "raw_failed": 0, "cuda_illegal_memory": true, "quality": { "structured_before.json": { "summary": { "json": "12/24", "tool": "0/8" }, "raw_records_pass": false } }, "medians": {}, "reason": null, "config": { "hf_repo": "zai-org/GLM-5.3-Flash", "model_revision": "eb9eb208eb0d988989d07a6a12d0fdeb5f52574a", "image_digest": "lmsysorg/sglang@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", "engine_version": "v0.5.21", "tp": 4, "ep": 1, "gpus": 4, "cuda_devices": "0,1,2,3", "weight_dtype": "fp8", "kv_dtype": "bfloat16", "max_model_len": 32768, "max_num_batched_tokens": 8192, "memory_fraction": 0.85, "prefix_caching": false, "disable_cuda_graph": false, "disable_overlap_schedule": true, "requested_max_running_requests": 64, "effective_max_running_requests": 64, "effective_state_pool_slots": 64, "engine_kv_tokens": 3343552, "spec_method": null, "spec_num_steps": null, "spec_num_tokens": null }, "points": [] }, { "name": "state-pool-128", "state": "failed", "raw_completed": 41, "raw_failed": 64, "cuda_illegal_memory": true, "quality": { "structured_before.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true } }, "medians": { "1": { "repeats": 1, "output_tok_s": 45.94293228252441, "ttft_p99_s": 14.285506313914665, "latency_p99_s": 16.416061545890404 }, "16": { "repeats": 1, "output_tok_s": 230.37919144087982, "ttft_p99_s": 13.079383591084042, "latency_p99_s": 18.95787175315054 } }, "reason": null, "config": { "hf_repo": "zai-org/GLM-5.3-Flash", "model_revision": "eb9eb208eb0d988989d07a6a12d0fdeb5f52574a", "image_digest": "lmsysorg/sglang@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", "engine_version": "v0.5.21", "tp": 4, "ep": 1, "gpus": 4, "cuda_devices": "0,1,2,3", "weight_dtype": "fp8", "kv_dtype": "bfloat16", "max_model_len": 32768, "max_num_batched_tokens": 8192, "memory_fraction": 0.85, "prefix_caching": false, "disable_cuda_graph": false, "disable_overlap_schedule": false, "requested_max_running_requests": 64, "effective_max_running_requests": 64, "effective_state_pool_slots": 128, "engine_kv_tokens": 3157760, "spec_method": null, "spec_num_steps": null, "spec_num_tokens": null }, "points": [ { "file": "r1-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 230.37919144087982, "ttft_s": { "mean": 11.536327121752038, "p50": 11.303974952519638, "p90": 12.264723458708612, "p99": 13.079383591084042 }, "latency_s": { "mean": 17.77264806803305, "p50": 17.772867571009556, "p90": 18.95620504229737, "p99": 18.95787175315054 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 45.94293228252441, "ttft_s": { "mean": 3.45167144361767, "p50": 0.19146403549530078, "p90": 11.751776408293516, "p99": 14.285506313914665 }, "latency_s": { "mean": 5.571619920494413, "p50": 2.3087469569873065, "p90": 13.875363507011206, "p99": 16.416061545890404 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c64_summary.json", "state": "failed", "concurrency": 64, "requested": 256, "issued": 65, "completed": 1, "failed": 64, "output_tokens": 81, "output_tok_s": 1.104470235684331, "ttft_s": { "mean": 1.046222127013607, "p50": 1.046222127013607, "p90": 1.046222127013607, "p99": 1.046222127013607 }, "latency_s": { "mean": 4.9321645210147835, "p50": 4.9321645210147835, "p90": 4.9321645210147835, "p99": 4.9321645210147835 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true } } ] }, { "name": "prefix-cache-on", "state": "passed", "raw_completed": 2424, "raw_failed": 0, "cuda_illegal_memory": false, "quality": { "structured_after-first-sweep.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true }, "structured_after.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true }, "structured_before.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true } }, "medians": { "1": { "repeats": 3, "output_tok_s": 112.7721615234847, "ttft_p99_s": 0.15869988559483317, "latency_p99_s": 2.2925816984815173 }, "16": { "repeats": 3, "output_tok_s": 661.9492589294154, "ttft_p99_s": 0.36712518836808156, "latency_p99_s": 6.231433296954783 }, "64": { "repeats": 3, "output_tok_s": 1406.4431098498862, "ttft_p99_s": 3.3802405343536517, "latency_p99_s": 13.152764865489704 }, "128": { "repeats": 3, "output_tok_s": 1240.415277422261, "ttft_p99_s": 22.4791590929075, "latency_p99_s": 33.96179403249204 } }, "reason": null, "config": { "hf_repo": "zai-org/GLM-5.3-Flash", "model_revision": "eb9eb208eb0d988989d07a6a12d0fdeb5f52574a", "image_digest": "lmsysorg/sglang@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", "engine_version": "v0.5.21", "tp": 4, "ep": 1, "gpus": 4, "cuda_devices": "0,1,2,3", "weight_dtype": "fp8", "kv_dtype": "bfloat16", "max_model_len": 32768, "max_num_batched_tokens": 8192, "memory_fraction": 0.85, "prefix_caching": true, "disable_cuda_graph": false, "disable_overlap_schedule": false, "requested_max_running_requests": 64, "effective_max_running_requests": 64, "effective_state_pool_slots": 575, "engine_kv_tokens": 1860480, "spec_method": null, "spec_num_steps": null, "spec_num_tokens": null }, "points": [ { "file": "r1-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129434, "output_tok_s": 889.2556382362172, "ttft_s": { "mean": 17.902615051418877, "p50": 12.724039923006785, "p90": 32.91898308222007, "p99": 35.28328892812103 }, "latency_s": { "mean": 34.53287414938052, "p50": 34.11946547801199, "p90": 53.94915376850404, "p99": 54.263225419922094 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 239.77806803516242, "ttft_s": { "mean": 11.198228920748988, "p50": 11.077324893994955, "p90": 11.621855959086679, "p99": 11.623322013140132 }, "latency_s": { "mean": 17.07730687543517, "p50": 17.076672781986417, "p90": 17.499832175610937, "p99": 17.501009427284007 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 31.274016157652184, "ttft_s": { "mean": 6.062830938124534, "p50": 5.9383444620034425, "p90": 11.945381194812944, "p99": 14.590837086483369 }, "latency_s": { "mean": 8.185282639995421, "p50": 8.059737960502389, "p90": 14.069022781809325, "p99": 16.721702152669312 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64734, "output_tok_s": 729.2034618962098, "ttft_s": { "mean": 3.839632379542195, "p50": 1.0031448229856323, "p90": 11.800510561995907, "p99": 11.974864140499268 }, "latency_s": { "mean": 21.921201689206328, "p50": 21.76161215301545, "p90": 32.64697553300357, "p99": 32.65606132200337 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129599, "output_tok_s": 1251.8624496441257, "ttft_s": { "mean": 13.095457514273846, "p50": 12.313535575507558, "p90": 21.627996011194774, "p99": 21.657545919224795 }, "latency_s": { "mean": 24.11957669959594, "p50": 23.507297032498172, "p90": 32.49556798368576, "p99": 32.520987741156425 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 662.12587413028, "ttft_s": { "mean": 0.3232818787510041, "p50": 0.31784115449409, "p90": 0.346356112594367, "p99": 0.34775608153548093 }, "latency_s": { "mean": 6.179577770218202, "p50": 6.180395362491254, "p90": 6.2146905788977165, "p99": 6.215481159040064 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 112.7721615234847, "ttft_s": { "mean": 0.14553320662889746, "p50": 0.14318407300743274, "p90": 0.15250045688881073, "p99": 0.15869988559483317 }, "latency_s": { "mean": 2.269609949868027, "p50": 2.266707848000806, "p90": 2.282100482986425, "p99": 2.2925816984815173 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64846, "output_tok_s": 1406.4431098498862, "ttft_s": { "mean": 0.7049455008087762, "p50": 0.658466076492914, "p90": 0.8704689709993545, "p99": 3.3777929882009627 }, "latency_s": { "mean": 11.231309496476342, "p50": 11.470050860996707, "p90": 11.564670274005039, "p99": 13.152764865489704 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129479, "output_tok_s": 1240.415277422261, "ttft_s": { "mean": 11.99741255822994, "p50": 12.167318688501837, "p90": 22.419171418680342, "p99": 22.4791590929075 }, "latency_s": { "mean": 24.234481008725936, "p50": 23.3892278179992, "p90": 33.94712836390245, "p99": 33.96179403249204 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 661.9492589294154, "ttft_s": { "mean": 0.3250069855021138, "p50": 0.32357691050856374, "p90": 0.36552545089507477, "p99": 0.36712518836808156 }, "latency_s": { "mean": 6.181522966594457, "p50": 6.18229974899441, "p90": 6.230435148303513, "p99": 6.231433296954783 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 112.78785258092671, "ttft_s": { "mean": 0.1447761861163599, "p50": 0.14282847849244718, "p90": 0.15124987699091436, "p99": 0.15738052668340968 }, "latency_s": { "mean": 2.269308627495775, "p50": 2.266360520006856, "p90": 2.277998927293811, "p99": 2.2913283797167243 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64685, "output_tok_s": 1411.032649334505, "ttft_s": { "mean": 0.6699291889523238, "p50": 0.5692388344905339, "p90": 0.8041502545092953, "p99": 3.3802405343536517 }, "latency_s": { "mean": 11.141236693539213, "p50": 11.381566248499439, "p90": 11.552835040492937, "p99": 13.119124967290524 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } } ] }, { "name": "quad-ep4", "state": "passed", "raw_completed": 2424, "raw_failed": 0, "cuda_illegal_memory": false, "quality": { "structured_after-first-sweep.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true }, "structured_after.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true }, "structured_before.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true } }, "medians": { "1": { "repeats": 3, "output_tok_s": 107.41192007845667, "ttft_p99_s": 0.16375609808164882, "latency_p99_s": 2.4030133180259146 }, "16": { "repeats": 3, "output_tok_s": 851.5517198406305, "ttft_p99_s": 0.5717285384223214, "latency_p99_s": 4.931252241091861 }, "64": { "repeats": 3, "output_tok_s": 1859.8673558548812, "ttft_p99_s": 2.9706487353119853, "latency_p99_s": 10.010227241709043 }, "128": { "repeats": 3, "output_tok_s": 1805.0300887899534, "ttft_p99_s": 11.796995312653307, "latency_p99_s": 20.558555910775613 } }, "reason": null, "config": { "hf_repo": "zai-org/GLM-5.3-Flash", "model_revision": "eb9eb208eb0d988989d07a6a12d0fdeb5f52574a", "image_digest": "lmsysorg/sglang@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", "engine_version": "v0.5.21", "tp": 4, "ep": 4, "gpus": 4, "cuda_devices": "0,1,2,3", "weight_dtype": "fp8", "kv_dtype": "bfloat16", "max_model_len": 32768, "max_num_batched_tokens": 8192, "memory_fraction": 0.85, "prefix_caching": false, "disable_cuda_graph": false, "disable_overlap_schedule": false, "requested_max_running_requests": 64, "effective_max_running_requests": 64, "effective_state_pool_slots": 64, "engine_kv_tokens": 3342848, "spec_method": null, "spec_num_steps": null, "spec_num_tokens": null }, "points": [ { "file": "r1-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129557, "output_tok_s": 1260.6024251732163, "ttft_s": { "mean": 12.53803722177139, "p50": 10.110097048993339, "p90": 20.016680595217622, "p99": 21.580754566847734 }, "latency_s": { "mean": 24.303681303160772, "p50": 23.10884406400146, "p90": 37.7173691351898, "p99": 37.738446691651774 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 100.11255612767451, "ttft_s": { "mean": 36.4503651282821, "p50": 36.23815697801183, "p90": 62.26043774179707, "p99": 62.280563969950016 }, "latency_s": { "mean": 40.90760100159696, "p50": 40.90764927800046, "p90": 66.66657820190304, "p99": 66.66711389339441 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 45.21810391126722, "ttft_s": { "mean": 3.4339063963780063, "p50": 0.16165891451237258, "p90": 11.67099390651274, "p99": 14.294833029670698 }, "latency_s": { "mean": 5.660982587378385, "p50": 2.3858853394922335, "p90": 13.89953873140912, "p99": 16.5292428918535 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64856, "output_tok_s": 557.7702795009943, "ttft_s": { "mean": 7.946943641520079, "p50": 10.882383638003375, "p90": 11.639878339512506, "p99": 13.106567623553559 }, "latency_s": { "mean": 28.831674741941356, "p50": 29.00142634099757, "p90": 38.77019071349059, "p99": 40.16378140760641 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129601, "output_tok_s": 1805.0300887899534, "ttft_s": { "mean": 8.546234029683603, "p50": 9.621542494991445, "p90": 10.165536415114184, "p99": 10.737305122296673 }, "latency_s": { "mean": 16.584480028310224, "p50": 17.724076520011295, "p90": 18.341811590312865, "p99": 19.476030042565252 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 851.6334607915486, "ttft_s": { "mean": 0.44005207165537286, "p50": 0.32981640301295556, "p90": 0.5702070230938261, "p99": 0.5716393425539718 }, "latency_s": { "mean": 4.8029363406894845, "p50": 4.802801112498855, "p90": 4.9211950323981, "p99": 4.922980476900702 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 107.4501651660193, "ttft_s": { "mean": 0.15378701636655023, "p50": 0.1524830969865434, "p90": 0.15897070879873354, "p99": 0.16375609808164882 }, "latency_s": { "mean": 2.382087886242516, "p50": 2.381398363999324, "p90": 2.390479864287772, "p99": 2.4030133180259146 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64701, "output_tok_s": 1865.941037465073, "ttft_s": { "mean": 1.0181207390577356, "p50": 1.1340590879990486, "p90": 1.4621227134921355, "p99": 2.5654582946881384 }, "latency_s": { "mean": 8.456026786726056, "p50": 8.48716824798612, "p90": 8.849389845505357, "p99": 9.863487366345337 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129434, "output_tok_s": 1815.1680985296514, "ttft_s": { "mean": 8.514694302830605, "p50": 9.615311854504398, "p90": 10.052984977795859, "p99": 11.796995312653307 }, "latency_s": { "mean": 16.473991589518562, "p50": 17.790496807501768, "p90": 18.269239388717686, "p99": 20.558555910775613 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 851.5517198406305, "ttft_s": { "mean": 0.44057971434631327, "p50": 0.3294966515095439, "p90": 0.5702705850097118, "p99": 0.5717285384223214 }, "latency_s": { "mean": 4.803534745407887, "p50": 4.8039437955012545, "p90": 4.929219816499971, "p99": 4.931252241091861 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 107.41192007845667, "ttft_s": { "mean": 0.15332633862271905, "p50": 0.1528857139928732, "p90": 0.15569065380259417, "p99": 0.15953891177487095 }, "latency_s": { "mean": 2.3829073529923335, "p50": 2.37906252600078, "p90": 2.3911076866846996, "p99": 2.398212788266246 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64773, "output_tok_s": 1859.8673558548812, "ttft_s": { "mean": 0.9965048687029139, "p50": 0.8939575044933008, "p90": 1.411614193508285, "p99": 2.9706487353119853 }, "latency_s": { "mean": 8.459404966308057, "p50": 8.544932325501577, "p90": 8.805561418499565, "p99": 10.010227241709043 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } } ] }, { "name": "quad-ep4-cache-state128", "state": "passed", "raw_completed": 2424, "raw_failed": 0, "cuda_illegal_memory": false, "quality": { "structured_after-first-sweep.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true }, "structured_after.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true }, "structured_before.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true } }, "medians": { "1": { "repeats": 3, "output_tok_s": 106.747515945367, "ttft_p99_s": 0.17561265323427505, "latency_p99_s": 2.4157723497145343 }, "16": { "repeats": 3, "output_tok_s": 841.9392963922523, "ttft_p99_s": 0.4920894549929653, "latency_p99_s": 4.863305985671759 }, "64": { "repeats": 3, "output_tok_s": 1003.3864363753423, "ttft_p99_s": 12.94812130939972, "latency_p99_s": 18.692349989354263 }, "128": { "repeats": 3, "output_tok_s": 961.6962736121735, "ttft_p99_s": 29.258219619827287, "latency_p99_s": 35.65107141215616 } }, "reason": null, "config": { "hf_repo": "zai-org/GLM-5.3-Flash", "model_revision": "eb9eb208eb0d988989d07a6a12d0fdeb5f52574a", "image_digest": "lmsysorg/sglang@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", "engine_version": "v0.5.21", "tp": 4, "ep": 4, "gpus": 4, "cuda_devices": "4,5,6,7", "weight_dtype": "fp8", "kv_dtype": "bfloat16", "max_model_len": 32768, "max_num_batched_tokens": 4096, "memory_fraction": 0.78, "prefix_caching": true, "disable_cuda_graph": false, "disable_overlap_schedule": false, "requested_max_running_requests": 64, "effective_max_running_requests": 25, "effective_state_pool_slots": 128, "engine_kv_tokens": 2341824, "spec_method": null, "spec_num_steps": null, "spec_num_tokens": null }, "points": [ { "file": "r1-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129455, "output_tok_s": 956.7079944031192, "ttft_s": { "mean": 23.71343947704696, "p50": 26.900126177002676, "p90": 28.310075086183495, "p99": 29.214488517113494 }, "latency_s": { "mean": 29.877172266831906, "p50": 33.32296501100063, "p90": 34.689227238087916, "p99": 35.571237427328306 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 195.48985917184942, "ttft_s": { "mean": 16.101610908503062, "p50": 11.966962461505318, "p90": 21.117307949386305, "p99": 21.117886125578078 }, "latency_s": { "mean": 20.946435738782384, "p50": 20.9482620540075, "p90": 25.48898380229366, "p99": 25.489353399465617 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 36.9149383742273, "ttft_s": { "mean": 4.703812390878738, "p50": 0.9274880919983843, "p90": 11.67991373160039, "p99": 14.281941931565523 }, "latency_s": { "mean": 6.934514052121813, "p50": 3.160289396502776, "p90": 13.912161105091217, "p99": 16.52051802990725 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64743, "output_tok_s": 410.53959304591945, "ttft_s": { "mean": 24.351881201836022, "p50": 22.999608693498885, "p90": 39.3341525404976, "p99": 53.481013788592826 }, "latency_s": { "mean": 37.25119530923473, "p50": 36.59697410349327, "p90": 58.83053999900585, "p99": 59.27752779225848 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129612, "output_tok_s": 966.6504266330996, "ttft_s": { "mean": 23.55089713869421, "p50": 26.628633070999058, "p90": 28.44119509549055, "p99": 29.506082176380442 }, "latency_s": { "mean": 29.63690955136576, "p50": 32.81510124950728, "p90": 34.82588586789789, "p99": 35.98200612100511 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 846.497292693759, "ttft_s": { "mean": 0.46050222390476847, "p50": 0.46253862501180265, "p90": 0.4882421837130096, "p99": 0.4896041996995336 }, "latency_s": { "mean": 4.832631183344347, "p50": 4.8326217955036554, "p90": 4.851132637500996, "p99": 4.851899682886142 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 106.97008138009704, "ttft_s": { "mean": 0.1601547602476785, "p50": 0.15826401599042583, "p90": 0.1703671079070773, "p99": 0.1729699421796249 }, "latency_s": { "mean": 2.392839877997176, "p50": 2.3880720485030906, "p90": 2.407221384299919, "p99": 2.4157723497145343 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64672, "output_tok_s": 1003.3864363753423, "ttft_s": { "mean": 8.810769659179869, "p50": 8.653540580518893, "p90": 12.661787386983633, "p99": 12.802381099750345 }, "latency_s": { "mean": 14.368599323312537, "p50": 14.350411446983344, "p90": 18.417136695003137, "p99": 18.692349989354263 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129441, "output_tok_s": 961.6962736121735, "ttft_s": { "mean": 23.58657237420192, "p50": 26.81598853202013, "p90": 28.36445585679612, "p99": 29.258219619827287 }, "latency_s": { "mean": 29.70889271700031, "p50": 33.190224086501985, "p90": 34.698747103920326, "p99": 35.65107141215616 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 841.9392963922523, "ttft_s": { "mean": 0.4752180895948186, "p50": 0.48332261049654335, "p90": 0.49090519708697683, "p99": 0.4920894549929653 }, "latency_s": { "mean": 4.858949542408482, "p50": 4.858937267505098, "p90": 4.862438027211465, "p99": 4.863305985671759 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 106.747515945367, "ttft_s": { "mean": 0.16427642625421868, "p50": 0.1624820724973688, "p90": 0.17130656919907777, "p99": 0.17561265323427505 }, "latency_s": { "mean": 2.397859836128191, "p50": 2.394386927000596, "p90": 2.407943995296955, "p99": 2.411527713118121 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64732, "output_tok_s": 1003.685905764353, "ttft_s": { "mean": 8.876780010050425, "p50": 9.130291273992043, "p90": 12.770702822992462, "p99": 12.94812130939972 }, "latency_s": { "mean": 14.409709841804215, "p50": 14.649078597489279, "p90": 18.519582961002016, "p99": 18.668182650792005 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } } ] }, { "name": "quad-ep4-mtp5", "state": "configuration_rejected", "raw_completed": 0, "raw_failed": 0, "cuda_illegal_memory": false, "quality": {}, "medians": {}, "reason": "Startup reduced active request limit to51 because256 state slots/5 slots per request. Stopped before measurement; replaced with384 slots for intended64 active requests.", "config": { "hf_repo": "zai-org/GLM-5.3-Flash", "model_revision": "eb9eb208eb0d988989d07a6a12d0fdeb5f52574a", "image_digest": "lmsysorg/sglang@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", "engine_version": "v0.5.21", "tp": 4, "ep": 4, "gpus": 4, "cuda_devices": "4,5,6,7", "weight_dtype": "fp8", "kv_dtype": "bfloat16", "max_model_len": 32768, "max_num_batched_tokens": 4096, "memory_fraction": 0.78, "prefix_caching": true, "disable_cuda_graph": false, "disable_overlap_schedule": false, "requested_max_running_requests": 64, "effective_max_running_requests": 51, "effective_state_pool_slots": 256, "engine_kv_tokens": 789440, "spec_method": "EAGLE", "spec_num_steps": 5, "spec_num_tokens": 6 }, "points": [] }, { "name": "quad-ep4-cache-state384", "state": "passed", "raw_completed": 2424, "raw_failed": 0, "cuda_illegal_memory": false, "quality": { "structured_after-first-sweep.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true }, "structured_after.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true }, "structured_before.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true } }, "medians": { "1": { "repeats": 3, "output_tok_s": 107.0263974111676, "ttft_p99_s": 0.1708983972138958, "latency_p99_s": 2.4144118319186965 }, "16": { "repeats": 3, "output_tok_s": 867.6706676816276, "ttft_p99_s": 0.3880503020714968, "latency_p99_s": 4.760133236583788 }, "64": { "repeats": 3, "output_tok_s": 1419.2578574234233, "ttft_p99_s": 7.233635237891512, "latency_p99_s": 19.24765919570491 }, "128": { "repeats": 3, "output_tok_s": 1813.6349010102242, "ttft_p99_s": 11.69733646986686, "latency_p99_s": 20.493336422392456 } }, "reason": null, "config": { "hf_repo": "zai-org/GLM-5.3-Flash", "model_revision": "eb9eb208eb0d988989d07a6a12d0fdeb5f52574a", "image_digest": "lmsysorg/sglang@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", "engine_version": "v0.5.21", "tp": 4, "ep": 4, "gpus": 4, "cuda_devices": "4,5,6,7", "weight_dtype": "fp8", "kv_dtype": "bfloat16", "max_model_len": 32768, "max_num_batched_tokens": 4096, "memory_fraction": 0.78, "prefix_caching": true, "disable_cuda_graph": false, "disable_overlap_schedule": false, "requested_max_running_requests": 64, "effective_max_running_requests": 64, "effective_state_pool_slots": 384, "engine_kv_tokens": 1598848, "spec_method": null, "spec_num_steps": null, "spec_num_tokens": null }, "points": [ { "file": "r1-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129473, "output_tok_s": 1400.4675646056667, "ttft_s": { "mean": 12.34447461057465, "p50": 10.028282332015806, "p90": 19.812020549879527, "p99": 20.064431254085793 }, "latency_s": { "mean": 21.700847532111823, "p50": 18.873653044982348, "p90": 28.168505218790962, "p99": 28.177365138957104 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 196.02882049692124, "ttft_s": { "mean": 16.086940944747766, "p50": 11.715863708988763, "p90": 21.288617826689734, "p99": 21.289745817192305 }, "latency_s": { "mean": 20.887112958813304, "p50": 20.886942543496843, "p90": 25.660026548893075, "p99": 25.66092306558916 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 31.256066522270757, "ttft_s": { "mean": 5.958445636755641, "p50": 5.947007195005426, "p90": 11.66408836460323, "p99": 14.220015514467375 }, "latency_s": { "mean": 8.190060809629358, "p50": 8.178574143501464, "p90": 13.898369899613316, "p99": 16.461388405073084 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64759, "output_tok_s": 670.5786261724154, "ttft_s": { "mean": 4.6208277129895805, "p50": 1.424288265989162, "p90": 11.723520936007844, "p99": 21.66363114757114 }, "latency_s": { "mean": 23.914846341340763, "p50": 19.174328552995576, "p90": 29.074958990502637, "p99": 39.20626108659635 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129542, "output_tok_s": 1815.5174506841518, "ttft_s": { "mean": 8.449183242332651, "p50": 9.525558060980984, "p90": 9.983712433400797, "p99": 11.69733646986686 }, "latency_s": { "mean": 16.456383327889625, "p50": 17.784303518987144, "p90": 18.13120666638424, "p99": 20.493336422392456 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 867.6706676816276, "ttft_s": { "mean": 0.352469017245312, "p50": 0.35181698951055296, "p90": 0.38769192730542273, "p99": 0.3880503020714968 }, "latency_s": { "mean": 4.714448424433613, "p50": 4.715015337482328, "p90": 4.759411276513129, "p99": 4.760133236583788 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 107.09244676871378, "ttft_s": { "mean": 0.1564327863707149, "p50": 0.15427912799350452, "p90": 0.16377665419131518, "p99": 0.1708983972138958 }, "latency_s": { "mean": 2.390104033118405, "p50": 2.38538954698015, "p90": 2.4028260953869904, "p99": 2.4144118319186965 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64842, "output_tok_s": 1419.2578574234233, "ttft_s": { "mean": 1.1681433341175307, "p50": 0.9980013555032201, "p90": 1.6093061050050892, "p99": 7.233635237891512 }, "latency_s": { "mean": 11.184142195440586, "p50": 8.82843138849421, "p90": 19.23714463399665, "p99": 19.24765919570491 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129455, "output_tok_s": 1813.6349010102242, "ttft_s": { "mean": 8.422065902757822, "p50": 9.489459064003313, "p90": 9.922320897615283, "p99": 11.616353029200688 }, "latency_s": { "mean": 16.450615299468723, "p50": 17.583508135998272, "p90": 18.176266406077776, "p99": 20.177052196552975 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 874.828027932002, "ttft_s": { "mean": 0.3205911238110275, "p50": 0.3204320259974338, "p90": 0.32422791720018723, "p99": 0.3249581069918349 }, "latency_s": { "mean": 4.676273671027047, "p50": 4.677102997491602, "p90": 4.690455510301399, "p99": 4.691132902046665 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 107.0263974111676, "ttft_s": { "mean": 0.15722132300652447, "p50": 0.15594268900167663, "p90": 0.1636932496126974, "p99": 0.1681338688803953 }, "latency_s": { "mean": 2.3915936243793112, "p50": 2.388493604506948, "p90": 2.403896652592812, "p99": 2.4135086835647233 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64843, "output_tok_s": 1831.2371182309644, "ttft_s": { "mean": 1.0082383599650484, "p50": 0.9662621110037435, "p90": 1.580227320009726, "p99": 3.171599101647731 }, "latency_s": { "mean": 8.611651210363675, "p50": 8.697822767004254, "p90": 9.018755870507448, "p99": 10.295388958847584 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } } ] }, { "name": "quad-ep4-mtp5-state384", "state": "passed", "raw_completed": 2424, "raw_failed": 0, "cuda_illegal_memory": false, "quality": { "structured_after-first-sweep.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true }, "structured_after.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true }, "structured_before.json": { "summary": { "json": "24/24", "tool": "8/8" }, "raw_records_pass": true } }, "medians": { "1": { "repeats": 3, "output_tok_s": 208.7558173164237, "ttft_p99_s": 0.17400925119698513, "latency_p99_s": 1.5823841545614414 }, "16": { "repeats": 3, "output_tok_s": 1072.7883447798731, "ttft_p99_s": 0.4688684874289902, "latency_p99_s": 5.478851231299051 }, "64": { "repeats": 3, "output_tok_s": 1886.1493629615538, "ttft_p99_s": 1.4678033792981295, "latency_p99_s": 12.298147680924739 }, "128": { "repeats": 3, "output_tok_s": 1788.980407600423, "ttft_p99_s": 10.659513297432277, "latency_p99_s": 21.726272275114315 } }, "reason": null, "config": { "hf_repo": "zai-org/GLM-5.3-Flash", "model_revision": "eb9eb208eb0d988989d07a6a12d0fdeb5f52574a", "image_digest": "lmsysorg/sglang@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", "engine_version": "v0.5.21", "tp": 4, "ep": 4, "gpus": 4, "cuda_devices": "4,5,6,7", "weight_dtype": "fp8", "kv_dtype": "bfloat16", "max_model_len": 32768, "max_num_batched_tokens": 4096, "memory_fraction": 0.78, "prefix_caching": true, "disable_cuda_graph": false, "disable_overlap_schedule": false, "requested_max_running_requests": 64, "effective_max_running_requests": 64, "effective_state_pool_slots": 384, "engine_kv_tokens": 241344, "spec_method": "EAGLE", "spec_num_steps": 5, "spec_num_tokens": 6 }, "points": [ { "file": "r1-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129449, "output_tok_s": 1788.980407600423, "ttft_s": { "mean": 8.245502630505541, "p50": 9.35590585700993, "p90": 10.154320300617837, "p99": 10.659513297432277 }, "latency_s": { "mean": 16.672811292382676, "p50": 17.857692324512755, "p90": 20.166028014398762, "p99": 21.55309171098692 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 193.34899098118052, "ttft_s": { "mean": 7.05318876162346, "p50": 10.933310898995842, "p90": 12.468401548999829, "p99": 12.55913724155369 }, "latency_s": { "mean": 20.546258937314633, "p50": 20.858476362016518, "p90": 37.20453969119408, "p99": 38.13200308669475 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 56.08018948727163, "ttft_s": { "mean": 3.482475886492466, "p50": 0.17814820898638573, "p90": 11.784385720090357, "p99": 14.433722488011407 }, "latency_s": { "mean": 4.564499684373004, "p50": 1.4839298269944265, "p90": 12.998766866489312, "p99": 15.436449992348788 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r1-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64779, "output_tok_s": 545.7311534379046, "ttft_s": { "mean": 1.565135894125774, "p50": 0.2618781464843778, "p90": 1.8208711695042439, "p99": 11.967997844931961 }, "latency_s": { "mean": 28.902642609730947, "p50": 20.989729542008718, "p90": 68.77164141151297, "p99": 71.00313389726216 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129545, "output_tok_s": 1812.9991950030508, "ttft_s": { "mean": 8.317098436491108, "p50": 9.549717617497663, "p90": 10.273876459314488, "p99": 10.609479958447627 }, "latency_s": { "mean": 16.728587903405582, "p50": 18.261010443005944, "p90": 20.571858275981505, "p99": 21.726272275114315 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 1125.223427761395, "ttft_s": { "mean": 0.3114544975942408, "p50": 0.3577597980038263, "p90": 0.3821968297008425, "p99": 0.38309218089678326 }, "latency_s": { "mean": 3.42390501837599, "p50": 3.3871518335072324, "p90": 4.378169138703379, "p99": 4.959234003903403 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 215.96244823260085, "ttft_s": { "mean": 0.1651920104995952, "p50": 0.16577396250795573, "p90": 0.17011197450046894, "p99": 0.17320370263507356 }, "latency_s": { "mean": 1.1850425607481156, "p50": 1.1731745324941585, "p90": 1.3318966712948168, "p99": 1.4723205660234089 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r2-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64748, "output_tok_s": 1886.1493629615538, "ttft_s": { "mean": 0.45453259929365686, "p50": 0.25077419899753295, "p90": 1.196247776490054, "p99": 1.383654780741199 }, "latency_s": { "mean": 8.177411416445011, "p50": 8.43034201048431, "p90": 10.920385110002826, "p99": 12.298147680924739 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c128_summary.json", "state": "passed", "concurrency": 128, "requested": 512, "issued": 512, "completed": 512, "failed": 0, "output_tokens": 129525, "output_tok_s": 1781.9556597609305, "ttft_s": { "mean": 8.416077384866696, "p50": 9.590674457504065, "p90": 10.230138231505407, "p99": 10.887093645761487 }, "latency_s": { "mean": 16.953964888145094, "p50": 18.19422331849637, "p90": 20.654836634595995, "p99": 21.978145165664028 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c16_summary.json", "state": "passed", "concurrency": 16, "requested": 32, "issued": 32, "completed": 32, "failed": 0, "output_tokens": 8192, "output_tok_s": 1072.7883447798731, "ttft_s": { "mean": 0.32959206062605517, "p50": 0.33666530648770276, "p90": 0.44550928859971484, "p99": 0.4688684874289902 }, "latency_s": { "mean": 3.4865789926889192, "p50": 3.295975904504303, "p90": 4.552401547500631, "p99": 5.478851231299051 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c1_summary.json", "state": "passed", "concurrency": 1, "requested": 8, "issued": 8, "completed": 8, "failed": 0, "output_tokens": 2048, "output_tok_s": 208.7558173164237, "ttft_s": { "mean": 0.16650415562617127, "p50": 0.16590189601993188, "p90": 0.17356520199682562, "p99": 0.17400925119698513 }, "latency_s": { "mean": 1.2259577072509273, "p50": 1.1743361554836156, "p90": 1.5146410244051367, "p99": 1.5823841545614414 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } }, { "file": "r3-c64_summary.json", "state": "passed", "concurrency": 64, "requested": 256, "issued": 256, "completed": 256, "failed": 0, "output_tokens": 64757, "output_tok_s": 1928.7288316906258, "ttft_s": { "mean": 0.4592012473358409, "p50": 0.25204493549244944, "p90": 1.0024924500030465, "p99": 1.4678033792981295 }, "latency_s": { "mean": 8.010479643573944, "p50": 8.287515199495829, "p90": 10.624594943001284, "p99": 12.206432534151826 }, "audit": { "raw_count": true, "completed": true, "failed": true, "tokens": true, "all_requested": true } } ] } ] }