{
  "original_rows": 683,
  "replicas": [
    {
      "run_id": "qwen36_35b__r_par_r1x8__256k__vllm__1011-070943",
      "model": "qwen36_35b",
      "hf_repo": "Qwen/Qwen3.6-35B-A3B-FP8",
      "model_revision": "95a723d08a9490559dae23d0cff1d9466213d989",
      "engine": "vllm",
      "engine_version": "0.31.0",
      "image_digest": "vllm/vllm-openai@sha256:c1c9f6fd5c109ba7f0546a59f5b2f15fb87f64c77782e90a27b648b42a8e67c3",
      "layout": "r_par_r1x8",
      "gpus": "8",
      "weight_dtype": "fp8",
      "kv_dtype": "fp8",
      "spec_method": "",
      "workload": "random",
      "isl": "1024",
      "osl": "1024",
      "concurrency": "256",
      "num_prompts": "1024",
      "completed": "1024",
      "failed": "0",
      "out_tok_s": "18390.559181682693",
      "ttft_p50_ms": "98.80615000292892",
      "ttft_p99_ms": "1414.7140253953694",
      "tpot_p50_ms": "12.233390813662655",
      "tpot_p99_ms": "13.456139430813224",
      "publishable": "True",
      "invalid": "False"
    },
    {
      "run_id": "qwen36_35b__r_par_tep4dp2__256k__vllm__1011-065213",
      "model": "qwen36_35b",
      "hf_repo": "Qwen/Qwen3.6-35B-A3B-FP8",
      "model_revision": "95a723d08a9490559dae23d0cff1d9466213d989",
      "engine": "vllm",
      "engine_version": "0.31.0",
      "image_digest": "vllm/vllm-openai@sha256:c1c9f6fd5c109ba7f0546a59f5b2f15fb87f64c77782e90a27b648b42a8e67c3",
      "layout": "r_par_tep4dp2",
      "gpus": "8",
      "weight_dtype": "fp8",
      "kv_dtype": "fp8",
      "spec_method": "",
      "workload": "random",
      "isl": "1024",
      "osl": "1024",
      "concurrency": "256",
      "num_prompts": "1024",
      "completed": "1024",
      "failed": "0",
      "out_tok_s": "6645.974256062758",
      "ttft_p50_ms": "250.4019115003757",
      "ttft_p99_ms": "3777.0022100438655",
      "tpot_p50_ms": "36.46606046591844",
      "tpot_p99_ms": "45.25901280489544",
      "publishable": "True",
      "invalid": "False"
    },
    {
      "run_id": "qwen38_27b__r_par_r2x4__256k__vllm__1010-211609",
      "model": "qwen38_27b",
      "hf_repo": "Qwen/Qwen3.8-27B-FP8",
      "model_revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a",
      "engine": "vllm",
      "engine_version": "0.31.0",
      "image_digest": "vllm/vllm-openai@sha256:c1c9f6fd5c109ba7f0546a59f5b2f15fb87f64c77782e90a27b648b42a8e67c3",
      "layout": "r_par_r2x4",
      "gpus": "8",
      "weight_dtype": "fp8",
      "kv_dtype": "fp8",
      "spec_method": "",
      "workload": "random",
      "isl": "1024",
      "osl": "1024",
      "concurrency": "256",
      "num_prompts": "1024",
      "completed": "1024",
      "failed": "0",
      "out_tok_s": "12327.701616562988",
      "ttft_p50_ms": "106.62261149991537",
      "ttft_p99_ms": "2867.1602272185555",
      "tpot_p50_ms": "17.83297610091173",
      "tpot_p99_ms": "20.44600988691907",
      "publishable": "True",
      "invalid": "False"
    },
    {
      "run_id": "qwen38_27b__x_tp4dp2__256k__vllm__1011-071732",
      "model": "qwen38_27b",
      "hf_repo": "Qwen/Qwen3.8-27B-FP8",
      "model_revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a",
      "engine": "vllm",
      "engine_version": "0.31.0",
      "image_digest": "vllm/vllm-openai@sha256:c1c9f6fd5c109ba7f0546a59f5b2f15fb87f64c77782e90a27b648b42a8e67c3",
      "layout": "x_tp4dp2",
      "gpus": "8",
      "weight_dtype": "fp8",
      "kv_dtype": "fp8",
      "spec_method": "",
      "workload": "random",
      "isl": "1024",
      "osl": "1024",
      "concurrency": "256",
      "num_prompts": "1024",
      "completed": "1024",
      "failed": "0",
      "out_tok_s": "11438.662061755009",
      "ttft_p50_ms": "117.45466949651018",
      "ttft_p99_ms": "3118.540089433809",
      "tpot_p50_ms": "20.36480907333544",
      "tpot_p99_ms": "23.49396596490645",
      "publishable": "True",
      "invalid": "False"
    },
    {
      "run_id": "qwen38_27b_int4__r_par_r2x4__256k__vllm__1010-215751",
      "model": "qwen38_27b_int4",
      "hf_repo": "RedHatAI/Qwen3.8-27B-INT4",
      "model_revision": "91bd022d5b49442a868bc35008f6c21e1860edfa",
      "engine": "vllm",
      "engine_version": "0.31.0",
      "image_digest": "vllm/vllm-openai@sha256:c1c9f6fd5c109ba7f0546a59f5b2f15fb87f64c77782e90a27b648b42a8e67c3",
      "layout": "r_par_r2x4",
      "gpus": "8",
      "weight_dtype": "int4-w4a16-g128",
      "kv_dtype": "fp8",
      "spec_method": "",
      "workload": "random",
      "isl": "1024",
      "osl": "1024",
      "concurrency": "256",
      "num_prompts": "1024",
      "completed": "1024",
      "failed": "0",
      "out_tok_s": "10890.50669466602",
      "ttft_p50_ms": "129.3811450013891",
      "ttft_p99_ms": "3980.7707089581527",
      "tpot_p50_ms": "20.475261454863627",
      "tpot_p99_ms": "24.764486336825417",
      "publishable": "True",
      "invalid": "False"
    }
  ],
  "long_context": [
    {
      "run_id": "glm53_flash__tp4_a__1024k__vllm__1010-000210",
      "model": "glm53_flash",
      "hf_repo": "zai-org/GLM-5.3-Flash",
      "model_revision": "",
      "engine": "vllm",
      "engine_version": "0.31.0",
      "image_digest": "",
      "layout": "tp4_a",
      "gpus": "4",
      "weight_dtype": "fp8",
      "kv_dtype": "bf16",
      "spec_method": "",
      "workload": "longdoc",
      "isl": "1000000",
      "osl": "512",
      "concurrency": "1",
      "num_prompts": "2",
      "completed": "2",
      "failed": "0",
      "out_tok_s": "8.22763107139344",
      "ttft_p50_ms": "58638.548105499925",
      "ttft_p99_ms": "58845.86617302975",
      "tpot_p50_ms": "7.026094304304446",
      "tpot_p99_ms": "7.030087214578448",
      "publishable": "True",
      "invalid": "False"
    },
    {
      "run_id": "qwen38_flash_next__tep4_b__1024k__vllm__1010-041653",
      "model": "qwen38_flash_next",
      "hf_repo": "Qwen/Qwen3.8-Flash-Next-FP8",
      "model_revision": "",
      "engine": "vllm",
      "engine_version": "0.1.dev20073+g8e685d198",
      "image_digest": "",
      "layout": "tep4_b",
      "gpus": "4",
      "weight_dtype": "fp8",
      "kv_dtype": "auto",
      "spec_method": "",
      "workload": "longdoc",
      "isl": "1000000",
      "osl": "512",
      "concurrency": "1",
      "num_prompts": "2",
      "completed": "2",
      "failed": "0",
      "out_tok_s": "5.766409952785981",
      "ttft_p50_ms": "84750.92425099865",
      "ttft_p99_ms": "85479.06413241806",
      "tpot_p50_ms": "7.903742092956898",
      "tpot_p99_ms": "7.929620907888546",
      "publishable": "True",
      "invalid": "False"
    },
    {
      "run_id": "dsv4_flash_fp8__tp4_b__1024k__sglang__1010-004006",
      "model": "dsv4_flash_fp8",
      "hf_repo": "sgl-project/DeepSeek-V4-Flash-FP8",
      "model_revision": "",
      "engine": "sglang",
      "engine_version": "v0.5.21",
      "image_digest": "",
      "layout": "tp4_b",
      "gpus": "4",
      "weight_dtype": "fp8-block128-ue8m0",
      "kv_dtype": "fp8_e4m3",
      "spec_method": "",
      "workload": "longdoc",
      "isl": "1000000",
      "osl": "512",
      "concurrency": "1",
      "num_prompts": "2",
      "completed": "2",
      "failed": "0",
      "out_tok_s": "3.9460284034015247",
      "ttft_p50_ms": "125765.7875495006",
      "ttft_p99_ms": "126251.54621386972",
      "tpot_p50_ms": "7.765761825831847",
      "tpot_p99_ms": "7.767587117064131",
      "publishable": "True",
      "invalid": "False"
    },
    {
      "run_id": "qwen38_27b__dp4_a__1024k__vllm__1011-023705",
      "model": "qwen38_27b",
      "hf_repo": "Qwen/Qwen3.8-27B-FP8",
      "model_revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a",
      "engine": "vllm",
      "engine_version": "0.31.0",
      "image_digest": "vllm/vllm-openai@sha256:c1c9f6fd5c109ba7f0546a59f5b2f15fb87f64c77782e90a27b648b42a8e67c3",
      "layout": "dp4_a",
      "gpus": "4",
      "weight_dtype": "fp8",
      "kv_dtype": "fp8",
      "spec_method": "",
      "workload": "longdoc",
      "isl": "1000000",
      "osl": "512",
      "concurrency": "1",
      "num_prompts": "2",
      "completed": "2",
      "failed": "0",
      "out_tok_s": "0.9025053338258372",
      "ttft_p50_ms": "550539.3019729963",
      "ttft_p99_ms": "551151.4653665161",
      "tpot_p50_ms": "32.81791792367357",
      "tpot_p99_ms": "32.826671696007395",
      "publishable": "True",
      "invalid": "False"
    }
  ],
  "glm": [
    {
      "isl": 1024.0,
      "osl": 1024.0,
      "concurrency": 1.0,
      "aggregate_out_tok_s": 225.5359872446006,
      "historical_pd_out_tok_s": 128.766126492117,
      "throughput_ratio": 1.7515164382801234,
      "aggregate_ttft_p50_ms": 139.051105506951,
      "historical_pd_ttft_p50_ms": 7443.510118500853,
      "aggregate_tpot_p50_ms": 3.5139987944390527,
      "historical_pd_tpot_p50_ms": 3.6069404323213057,
      "aggregate_ttft_p99_ms": 416.3299401414406,
      "aggregate_tpot_p99_ms": 11.319028911643947,
      "completed": 16.0,
      "prefix_hit_rate": 0.0
    },
    {
      "isl": 1024.0,
      "osl": 1024.0,
      "concurrency": 64.0,
      "aggregate_out_tok_s": 2982.340430708227,
      "historical_pd_out_tok_s": 2374.0055548636715,
      "throughput_ratio": 1.2562482950379994,
      "aggregate_ttft_p50_ms": 196.16452699847287,
      "historical_pd_ttft_p50_ms": 9383.410590002311,
      "aggregate_tpot_p50_ms": 14.124842266934188,
      "historical_pd_tpot_p50_ms": 13.053135834319592,
      "aggregate_ttft_p99_ms": 4839.71465945142,
      "aggregate_tpot_p99_ms": 39.64175889252358,
      "completed": 256.0,
      "prefix_hit_rate": 0.0
    },
    {
      "isl": 1024.0,
      "osl": 1024.0,
      "concurrency": 256.0,
      "aggregate_out_tok_s": 7978.655026073107,
      "historical_pd_out_tok_s": 3809.391639058134,
      "throughput_ratio": 2.094469611437961,
      "aggregate_ttft_p50_ms": 277.93744950031396,
      "historical_pd_ttft_p50_ms": 32364.57270900428,
      "aggregate_tpot_p50_ms": 24.087571268557184,
      "historical_pd_tpot_p50_ms": 16.895028457475426,
      "aggregate_ttft_p99_ms": 4660.029466179985,
      "aggregate_tpot_p99_ms": 70.23654656198663,
      "completed": 1024.0,
      "prefix_hit_rate": 0.0
    },
    {
      "isl": 8192.0,
      "osl": 1024.0,
      "concurrency": 64.0,
      "aggregate_out_tok_s": 2351.3540315609,
      "historical_pd_out_tok_s": 1659.2867144760223,
      "throughput_ratio": 1.417087240587846,
      "aggregate_ttft_p50_ms": 619.3940325028962,
      "historical_pd_ttft_p50_ms": 15594.541901496996,
      "aggregate_tpot_p50_ms": 21.7483595471836,
      "historical_pd_tpot_p50_ms": 14.304678803561238,
      "aggregate_ttft_p99_ms": 9400.905234293892,
      "aggregate_tpot_p99_ms": 59.10662866981839,
      "completed": 256.0,
      "prefix_hit_rate": 0.0
    }
  ],
  "notes": {
    "glm": "Historical configuration comparison; saved PD prefill eager, new aggregated CUDA graphs. Rescue correctness limited to ~16K. No energy telemetry. Original publication gate not applied.",
    "topology": "Post-IOMMU-passthrough NCCL bus bandwidth, GB/s. In-quad all-reduce 337; all-8 default 64.3; all-8 SYS 83.1. All-to-all default 16.5 vs SYS 1.25."
  }
}