From b6c868c0f9ba4cdd0f817e9d95c3251600ab5c8e Mon Sep 17 00:00:00 2001 From: kudomcho Date: Fri, 10 Jul 2026 23:55:37 +0000 Subject: [PATCH] perf(ep): add MI350X IntraNode/IntraNodeLL tuning configs for EP2/4/8 Add tuned dispatch+combine configs for MI350X (gfx950, 256 CUs) and fix detect_gpu_model() fallback for devices with empty name string. Co-Authored-By: Claude Opus 4 (1M context) --- docs/ep2_mi350x_tuning_results.md | 171 +++++++++ docs/ep4_mi350x_tuning_results.md | 171 +++++++++ docs/ep8_mi350x_IntraNodeLL_tuning_results.md | 70 ++++ docs/ep8_mi350x_tuning_results.md | 177 +++++++++ docs/mi350x_profiler_analysis.md | 83 +++++ python/mori/ops/tuning_config.py | 17 +- ...gfx950_mi350x_IntraNodeLL_ep8_combine.json | 46 +++ ...fx950_mi350x_IntraNodeLL_ep8_dispatch.json | 40 ++ .../gfx950_mi350x_IntraNode_ep2_combine.json | 346 ++++++++++++++++++ .../gfx950_mi350x_IntraNode_ep2_dispatch.json | 150 ++++++++ .../gfx950_mi350x_IntraNode_ep4_combine.json | 346 ++++++++++++++++++ .../gfx950_mi350x_IntraNode_ep4_dispatch.json | 150 ++++++++ .../gfx950_mi350x_IntraNode_ep8_combine.json | 346 ++++++++++++++++++ .../gfx950_mi350x_IntraNode_ep8_dispatch.json | 160 ++++++++ tests/python/ops/bench_dispatch_combine.py | 26 +- 15 files changed, 2295 insertions(+), 4 deletions(-) create mode 100644 docs/ep2_mi350x_tuning_results.md create mode 100644 docs/ep4_mi350x_tuning_results.md create mode 100644 docs/ep8_mi350x_IntraNodeLL_tuning_results.md create mode 100644 docs/ep8_mi350x_tuning_results.md create mode 100644 docs/mi350x_profiler_analysis.md create mode 100644 python/mori/ops/tuning_configs/gfx950_mi350x_IntraNodeLL_ep8_combine.json create mode 100644 python/mori/ops/tuning_configs/gfx950_mi350x_IntraNodeLL_ep8_dispatch.json create mode 100644 python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep2_combine.json create mode 100644 python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep2_dispatch.json create mode 100644 python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep4_combine.json create mode 100644 python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep4_dispatch.json create mode 100644 python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep8_combine.json create mode 100644 python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep8_dispatch.json diff --git a/docs/ep2_mi350x_tuning_results.md b/docs/ep2_mi350x_tuning_results.md new file mode 100644 index 000000000..73e264620 --- /dev/null +++ b/docs/ep2_mi350x_tuning_results.md @@ -0,0 +1,171 @@ +# EP2 IntraNode Tuning Results — MI350X (gfx950, 256 CUs) + +Date: 2026-07-10 +Hardware: 8x MI350X (gfx950, 256 CUs, 288 GB HBM3e), AMD EPYC 9575F +Sweep: full scope (35 block_num x 9 warp_per_block = 315 configs/shape) +Configs: `python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep2_{dispatch,combine}.json` + +## 1. Recommended Configs + +### Dispatch — best config per shape + +| Tokens | Dtype | Hidden | block_num | warp_per_block | BW (GB/s) | Latency | +|--------|-------|--------|-----------|----------------|-----------|---------| +| 64 | fp4 | 3584 | 64 | 8 | 16.0 | 28.6 us | +| 128 | fp4 | 3584 | 184 | 6 | 28.0 | 32.7 us | +| 256 | fp4 | 3584 | 208 | 10 | 43.2 | 42.3 us | +| 512 | fp4 | 3584 | 208 | 10 | 58.8 | 62.4 us | +| 1024 | fp4 | 3584 | 128 | 16 | 70.7 | 103.3 us | +| 2048 | fp4 | 3584 | 256 | 16 | 81.1 | 180.5 us | +| 4096 | fp4 | 3584 | 255 | 16 | 87.9 | 332.5 us | +| 64 | fp8_e4m3 | 7168 | 64 | 8 | 32.7 | 28.0 us | +| 128 | fp8_e4m3 | 7168 | 128 | 8 | 49.6 | 36.9 us | +| 256 | fp8_e4m3 | 7168 | 128 | 8 | 68.0 | 53.9 us | +| 512 | fp8_e4m3 | 7168 | 176 | 8 | 82.4 | 89.0 us | +| 1024 | fp8_e4m3 | 7168 | 208 | 8 | 92.9 | 157.5 us | +| 2048 | fp8_e4m3 | 7168 | 128 | 16 | 99.1 | 295.4 us | +| 4096 | fp8_e4m3 | 7168 | 128 | 16 | 103.2 | 567.1 us | + +### Combine — best config per shape (winner of P2P vs non-P2P) + +| Tokens | Quant | Path | block_num | warp_per_block | BW (GB/s) | Latency | +|--------|-------|------|-----------|----------------|-----------|---------| +| 64 | fp8_direct_cast | non-P2P | 224 | 4 | 63.4 | 28.8 us | +| 128 | fp8_direct_cast | non-P2P | 224 | 8 | 108.7 | 33.7 us | +| 256 | fp8_direct_cast | non-P2P | 224 | 8 | 144.1 | 50.8 us | +| 512 | fp8_direct_cast | non-P2P | 224 | 16 | 175.7 | 83.4 us | +| 1024 | fp8_direct_cast | non-P2P | 256 | 8 | 188.8 | 155.0 us | +| 2048 | fp8_direct_cast | non-P2P | 256 | 16 | 202.2 | 289.4 us | +| 4096 | fp8_direct_cast | non-P2P | 256 | 16 | 206.8 | 566.1 us | +| 64 | none | P2P | 64 | 8 | 58.7 | 31.2 us | +| 128 | none | P2P | 56 | 16 | 78.2 | 46.9 us | +| 256 | none | P2P | 128 | 16 | 94.7 | 77.4 us | +| 512 | none | P2P | 64 | 16 | 104.9 | 139.2 us | +| 1024 | none | P2P | 80 | 14 | 110.8 | 264.0 us | +| 2048 | none | P2P | 32 | 16 | 113.2 | 516.9 us | +| 4096 | none | P2P | 160 | 16 | 115.0 | 1019.0 us | + +### P2P vs non-P2P decision rule + +- **With fp8_direct_cast quant:** use non-P2P at all token counts (+8% at 64 tokens, up to +79% at 4096). Stronger effect than EP4/EP8 — fewer peers means even less XGMI contention for the staging path. +- **Without quant:** use P2P at all token counts (+3-9%). + +## 2. Before/After — MI350X Tuned vs Shipped Baseline + +Baseline: MI355X bn/wpb configs measured on MI350X hardware (from sweep data). + +### Dispatch + +| Tokens | Dtype | Baseline bn/wpb | Baseline BW | Tuned bn/wpb | Tuned BW | Delta | +|--------|-------|-----------------|-------------|--------------|----------|-------| +| 64 | fp4 | 64/8 | 15.9 | 64/8 | 16.0 | +0.1% | +| 128 | fp4 | 256/4 | 28.3 | 184/6 | 28.0 | -0.8% | +| 256 | fp4 | 256/8 | 6.7 | 208/10 | 43.2 | +543% | +| 512 | fp4 | 256/8 | 58.5 | 208/10 | 58.8 | +0.5% | +| 1024 | fp4 | 128/16 | 70.1 | 128/16 | 70.7 | +0.9% | +| 2048 | fp4 | 256/16 | 80.2 | 256/16 | 81.1 | +1.1% | +| 4096 | fp4 | 256/16 | 85.6 | 255/16 | 87.9 | +2.8% | +| 64 | fp8_e4m3 | 64/8 | 32.2 | 64/8 | 32.7 | +1.6% | +| 128 | fp8_e4m3 | 128/8 | 49.3 | 128/8 | 49.6 | +0.5% | +| 256 | fp8_e4m3 | 128/8 | 67.1 | 128/8 | 68.0 | +1.2% | +| 512 | fp8_e4m3 | 256/8 | 82.4 | 176/8 | 82.4 | +0.0% | +| 1024 | fp8_e4m3 | 256/8 | 93.5 | 208/8 | 92.9 | -0.7% | +| 2048 | fp8_e4m3 | 256/8 | 97.3 | 128/16 | 99.1 | +1.8% | +| 4096 | fp8_e4m3 | 256/8 | 100.4 | 128/16 | 103.2 | +2.8% | + +Dispatch: mostly within noise (+/-3%). Major outlier at fp4/256 tokens: MI355X's bn=256/wpb=8 hits a performance cliff on MI350X (6.7 GB/s), tuned config avoids it (+543%). + +### Combine + +| Tokens | Quant | Path | Baseline bn/wpb | Baseline BW | Tuned bn/wpb | Tuned BW | Delta | +|--------|-------|------|-----------------|-------------|--------------|----------|-------| +| 64 | fp8_direct_cast | non-P2P | 128/8 | 61.4 | 224/4 | 63.4 | +3.3% | +| 128 | fp8_direct_cast | non-P2P | 256/8 | 107.5 | 224/8 | 108.7 | +1.1% | +| 256 | fp8_direct_cast | non-P2P | 256/8 | 143.9 | 224/8 | 144.1 | +0.1% | +| 512 | fp8_direct_cast | non-P2P | 256/16 | 175.2 | 224/16 | 175.7 | +0.3% | +| 1024 | fp8_direct_cast | non-P2P | 128/16 | 186.0 | 256/8 | 188.8 | +1.5% | +| 2048 | fp8_direct_cast | non-P2P | 256/16 | 202.0 | 256/16 | 202.2 | +0.1% | +| 4096 | fp8_direct_cast | non-P2P | 256/16 | 167.2 | 256/16 | 206.8 | +23.7% | +| 64 | none | non-P2P | 128/8 | 54.1 | 224/8 | 57.0 | +5.3% | +| 128 | none | non-P2P | 256/8 | 74.5 | 224/8 | 75.4 | +1.1% | +| 256 | none | non-P2P | 256/8 | 90.6 | 224/8 | 90.1 | -0.5% | +| 512 | none | non-P2P | 256/16 | 99.1 | 224/16 | 98.8 | -0.4% | +| 1024 | none | non-P2P | 256/16 | 103.3 | 256/16 | 103.4 | +0.1% | +| 2048 | none | non-P2P | 128/16 | 103.8 | 128/16 | 103.9 | +0.2% | +| 4096 | none | non-P2P | 128/16 | 104.9 | 256/16 | 106.5 | +1.5% | +| 64 | fp8_direct_cast | P2P | 32/8 | 56.8 | 64/8 | 58.9 | +3.6% | +| 128 | fp8_direct_cast | P2P | 64/8 | 77.2 | 112/8 | 78.6 | +1.8% | +| 256 | fp8_direct_cast | P2P | 64/16 | 94.4 | 64/16 | 94.4 | +0.0% | +| 512 | fp8_direct_cast | P2P | 64/16 | 104.7 | 64/16 | 104.8 | +0.1% | +| 1024 | fp8_direct_cast | P2P | 128/16 | 110.9 | 32/16 | 110.3 | -0.6% | +| 2048 | fp8_direct_cast | P2P | 32/16 | 112.7 | 152/16 | 114.7 | +1.8% | +| 4096 | fp8_direct_cast | P2P | 32/16 | 114.0 | 56/15 | 115.4 | +1.3% | +| 64 | none | P2P | 32/8 | 56.5 | 64/8 | 58.7 | +3.9% | +| 128 | none | P2P | 64/8 | 78.2 | 56/16 | 78.2 | +0.0% | +| 256 | none | P2P | 64/16 | 94.2 | 128/16 | 94.7 | +0.5% | +| 512 | none | P2P | 64/16 | 104.9 | 64/16 | 104.9 | +0.0% | +| 1024 | none | P2P | 128/16 | 110.8 | 80/14 | 110.8 | +0.0% | +| 2048 | none | P2P | 32/16 | 113.1 | 32/16 | 113.2 | +0.1% | +| 4096 | none | P2P | 32/16 | 113.9 | 160/16 | 115.0 | +0.9% | + +Combine: mostly +0-5%. Two outliers where MI355X configs hit performance cliffs on MI350X: fp8dc/nP2P at 4096 tokens (+23.7%), and none/nP2P at 64 tokens (+5.3%). + +## 3. P2P vs non-P2P Detail + +### With fp8_direct_cast quant (hidden=7168) + +| Tokens | P2P BW | P2P lat | P2P bn/wpb | non-P2P BW | non-P2P lat | non-P2P bn/wpb | Winner | Uplift | +|--------|--------|---------|-----------|-----------|------------|---------------|--------|--------| +| 64 | 58.9 | 31.1 us | 64/8 | 63.4 | 28.8 us | 224/4 | non-P2P | +8% | +| 128 | 78.6 | 46.6 us | 112/8 | 108.7 | 33.7 us | 224/8 | non-P2P | +38% | +| 256 | 94.4 | 77.4 us | 64/16 | 144.1 | 50.8 us | 224/8 | non-P2P | +53% | +| 512 | 104.8 | 139.5 us | 64/16 | 175.7 | 83.4 us | 224/16 | non-P2P | +68% | +| 1024 | 110.3 | 265.1 us | 32/16 | 188.8 | 155.0 us | 256/8 | non-P2P | +71% | +| 2048 | 114.7 | 509.8 us | 152/16 | 202.2 | 289.4 us | 256/16 | non-P2P | +76% | +| 4096 | 115.4 | 1014.5 us | 56/15 | 206.8 | 566.1 us | 256/16 | non-P2P | +79% | + +### Without quant (hidden=7168) + +| Tokens | P2P BW | P2P lat | P2P bn/wpb | non-P2P BW | non-P2P lat | non-P2P bn/wpb | Winner | Uplift | +|--------|--------|---------|-----------|-----------|------------|---------------|--------|--------| +| 64 | 58.7 | 31.2 us | 64/8 | 57.0 | 32.3 us | 224/8 | P2P | +3% | +| 128 | 78.2 | 46.9 us | 56/16 | 75.4 | 48.6 us | 224/8 | P2P | +4% | +| 256 | 94.7 | 77.4 us | 128/16 | 90.1 | 81.2 us | 224/8 | P2P | +5% | +| 512 | 104.9 | 139.2 us | 64/16 | 98.8 | 148.2 us | 224/16 | P2P | +6% | +| 1024 | 110.8 | 264.0 us | 80/14 | 103.4 | 282.9 us | 256/16 | P2P | +7% | +| 2048 | 113.2 | 516.9 us | 32/16 | 103.9 | 563.4 us | 128/16 | P2P | +9% | +| 4096 | 115.0 | 1019.0 us | 160/16 | 106.5 | 1098.7 us | 256/16 | P2P | +8% | + +## 4. Dispatch Detail + +### fp4 (hidden=3584) + +| Tokens | BW (GB/s) | Latency | block_num | warp_per_block | +|--------|-----------|---------|-----------|----------------| +| 64 | 16.0 | 28.6 us | 64 | 8 | +| 128 | 28.0 | 32.7 us | 184 | 6 | +| 256 | 43.2 | 42.3 us | 208 | 10 | +| 512 | 58.8 | 62.4 us | 208 | 10 | +| 1024 | 70.7 | 103.3 us | 128 | 16 | +| 2048 | 81.1 | 180.5 us | 256 | 16 | +| 4096 | 87.9 | 332.5 us | 255 | 16 | + +### fp8_e4m3 (hidden=7168) + +| Tokens | BW (GB/s) | Latency | block_num | warp_per_block | +|--------|-----------|---------|-----------|----------------| +| 64 | 32.7 | 28.0 us | 64 | 8 | +| 128 | 49.6 | 36.9 us | 128 | 8 | +| 256 | 68.0 | 53.9 us | 128 | 8 | +| 512 | 82.4 | 89.0 us | 176 | 8 | +| 1024 | 92.9 | 157.5 us | 208 | 8 | +| 2048 | 99.1 | 295.4 us | 128 | 16 | +| 4096 | 103.2 | 567.1 us | 128 | 16 | + +## 5. Remaining Work + +- [x] EP8 IntraNode (all 4 groups) +- [x] EP4 IntraNode (all 4 groups) +- [x] EP2 IntraNode (all 4 groups) +- [x] IntraNodeLL tuning +- [x] Profiler analysis (static occupancy — see `mi350x_profiler_analysis.md`) diff --git a/docs/ep4_mi350x_tuning_results.md b/docs/ep4_mi350x_tuning_results.md new file mode 100644 index 000000000..c1884c6b3 --- /dev/null +++ b/docs/ep4_mi350x_tuning_results.md @@ -0,0 +1,171 @@ +# EP4 IntraNode Tuning Results — MI350X (gfx950, 256 CUs) + +Date: 2026-07-09 +Hardware: 8x MI350X (gfx950, 256 CUs, 288 GB HBM3e), AMD EPYC 9575F +Sweep: full scope (35 block_num x 9 warp_per_block = 315 configs/shape) +Configs: `python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep4_{dispatch,combine}.json` + +## 1. Recommended Configs + +### Dispatch — best config per shape + +| Tokens | Dtype | Hidden | block_num | warp_per_block | BW (GB/s) | Latency | +|--------|-------|--------|-----------|----------------|-----------|---------| +| 64 | fp4 | 3584 | 72 | 8 | 26.3 | 31.5 us | +| 128 | fp4 | 3584 | 128 | 8 | 45.9 | 36.6 us | +| 256 | fp4 | 3584 | 256 | 8 | 70.8 | 47.1 us | +| 512 | fp4 | 3584 | 256 | 8 | 96.3 | 68.8 us | +| 1024 | fp4 | 3584 | 256 | 8 | 119.7 | 111.0 us | +| 2048 | fp4 | 3584 | 184 | 15 | 140.1 | 189.6 us | +| 4096 | fp4 | 3584 | 200 | 15 | 157.7 | 336.6 us | +| 64 | fp8_e4m3 | 7168 | 208 | 4 | 55.2 | 30.4 us | +| 128 | fp8_e4m3 | 7168 | 176 | 6 | 85.9 | 39.0 us | +| 256 | fp8_e4m3 | 7168 | 256 | 4 | 117.6 | 56.7 us | +| 512 | fp8_e4m3 | 7168 | 128 | 8 | 142.1 | 93.2 us | +| 1024 | fp8_e4m3 | 7168 | 208 | 8 | 164.1 | 161.8 us | +| 2048 | fp8_e4m3 | 7168 | 228 | 8 | 182.0 | 291.7 us | +| 4096 | fp8_e4m3 | 7168 | 208 | 16 | 192.1 | 552.9 us | + +### Combine — best config per shape (winner of P2P vs non-P2P) + +| Tokens | Quant | Path | block_num | warp_per_block | BW (GB/s) | Latency | +|--------|-------|------|-----------|----------------|-----------|---------| +| 64 | fp8_direct_cast | non-P2P | 224 | 8 | 108.0 | 30.5 us | +| 128 | fp8_direct_cast | non-P2P | 224 | 8 | 186.2 | 36.0 us | +| 256 | fp8_direct_cast | non-P2P | 256 | 16 | 252.7 | 52.4 us | +| 512 | fp8_direct_cast | non-P2P | 224 | 16 | 309.3 | 85.8 us | +| 1024 | fp8_direct_cast | non-P2P | 256 | 16 | 337.3 | 157.0 us | +| 2048 | fp8_direct_cast | non-P2P | 256 | 16 | 356.4 | 297.0 us | +| 4096 | fp8_direct_cast | non-P2P | 256 | 16 | 369.1 | 575.5 us | +| 64 | none | P2P | 112 | 16 | 104.2 | 32.1 us | +| 128 | none | P2P | 112 | 16 | 142.6 | 46.6 us | +| 256 | none | P2P | 128 | 16 | 173.6 | 76.4 us | +| 512 | none | P2P | 128 | 16 | 195.7 | 135.8 us | +| 1024 | none | P2P | 128 | 16 | 208.0 | 254.9 us | +| 2048 | none | P2P | 256 | 16 | 215.6 | 492.5 us | +| 4096 | none | P2P | 256 | 16 | 215.0 | 988.9 us | + +### P2P vs non-P2P decision rule + +- **With fp8_direct_cast quant:** use non-P2P at all token counts (even 64 tokens — unlike EP8). +- **Without quant:** use P2P at all token counts. +- **Rule:** quant flips the winner. With quant, halved FP8 read traffic makes the staging buffer copy worthwhile. Without quant, the staging copy is pure overhead. + +## 2. Before/After — MI350X Tuned vs Shipped Baseline + +Baseline: MI355X bn/wpb configs measured on MI350X hardware (from sweep data). + +### Dispatch + +| Tokens | Dtype | Baseline bn/wpb | Baseline BW | Tuned bn/wpb | Tuned BW | Delta | +|--------|-------|-----------------|-------------|--------------|----------|-------| +| 64 | fp4 | 128/4 | 26.5 | 72/8 | 26.3 | -0.8% | +| 128 | fp4 | 256/4 | 45.9 | 128/8 | 45.9 | +0.0% | +| 256 | fp4 | 256/8 | 69.5 | 256/8 | 70.8 | +1.8% | +| 512 | fp4 | 256/8 | 94.7 | 256/8 | 96.3 | +1.6% | +| 1024 | fp4 | 256/8 | 118.3 | 256/8 | 119.7 | +1.1% | +| 2048 | fp4 | 256/8 | 137.8 | 184/15 | 140.1 | +1.7% | +| 4096 | fp4 | 256/16 | 153.4 | 200/15 | 157.7 | +2.8% | +| 64 | fp8_e4m3 | 128/4 | 54.2 | 208/4 | 55.2 | +1.8% | +| 128 | fp8_e4m3 | 256/4 | 85.7 | 176/6 | 85.9 | +0.2% | +| 256 | fp8_e4m3 | 256/4 | 115.5 | 256/4 | 117.6 | +1.8% | +| 512 | fp8_e4m3 | 256/8 | 141.0 | 128/8 | 142.1 | +0.8% | +| 1024 | fp8_e4m3 | 256/8 | 162.6 | 208/8 | 164.1 | +0.9% | +| 2048 | fp8_e4m3 | 256/8 | 178.8 | 228/8 | 182.0 | +1.8% | +| 4096 | fp8_e4m3 | 256/8 | 189.8 | 208/16 | 192.1 | +1.2% | + +Dispatch: +0-3% gains across the board. + +### Combine + +| Tokens | Quant | Path | Baseline bn/wpb | Baseline BW | Tuned bn/wpb | Tuned BW | Delta | +|--------|-------|------|-----------------|-------------|--------------|----------|-------| +| 64 | fp8_direct_cast | non-P2P | 256/8 | 106.5 | 224/8 | 108.0 | +1.5% | +| 128 | fp8_direct_cast | non-P2P | 256/8 | 124.6 | 224/8 | 186.2 | +49.4% | +| 256 | fp8_direct_cast | non-P2P | 256/16 | 249.7 | 256/16 | 252.7 | +1.2% | +| 512 | fp8_direct_cast | non-P2P | 256/16 | 305.9 | 224/16 | 309.3 | +1.1% | +| 1024 | fp8_direct_cast | non-P2P | 256/16 | 336.7 | 256/16 | 337.3 | +0.2% | +| 2048 | fp8_direct_cast | non-P2P | 256/16 | 356.1 | 256/16 | 356.4 | +0.1% | +| 4096 | fp8_direct_cast | non-P2P | 256/16 | 369.1 | 256/16 | 369.1 | +0.0% | +| 64 | none | non-P2P | 256/8 | 93.2 | 224/8 | 98.7 | +5.8% | +| 128 | none | non-P2P | 256/8 | 129.7 | 224/16 | 132.8 | +2.4% | +| 256 | none | non-P2P | 256/16 | 162.8 | 224/16 | 163.8 | +0.6% | +| 512 | none | non-P2P | 256/16 | 180.2 | 224/16 | 180.7 | +0.3% | +| 1024 | none | non-P2P | 256/16 | 189.1 | 256/16 | 190.4 | +0.7% | +| 2048 | none | non-P2P | 256/16 | 193.1 | 256/16 | 194.3 | +0.6% | +| 4096 | none | non-P2P | 256/16 | 192.1 | 256/16 | 192.9 | +0.4% | +| 64 | fp8_direct_cast | P2P | 64/16 | 99.0 | 112/16 | 104.4 | +5.4% | +| 128 | fp8_direct_cast | P2P | 128/16 | 141.4 | 112/16 | 142.3 | +0.7% | +| 256 | fp8_direct_cast | P2P | 128/16 | 172.9 | 112/16 | 172.7 | -0.1% | +| 512 | fp8_direct_cast | P2P | 128/16 | 196.7 | 128/16 | 196.8 | +0.0% | +| 1024 | fp8_direct_cast | P2P | 128/16 | 204.6 | 256/16 | 208.1 | +1.7% | +| 2048 | fp8_direct_cast | P2P | 256/16 | 211.2 | 256/16 | 215.1 | +1.9% | +| 4096 | fp8_direct_cast | P2P | 256/16 | 213.6 | 256/16 | 216.0 | +1.1% | +| 64 | none | P2P | 256/16 | 99.2 | 112/16 | 104.2 | +5.0% | +| 128 | none | P2P | 128/16 | 140.8 | 112/16 | 142.6 | +1.3% | +| 256 | none | P2P | 128/16 | 173.0 | 128/16 | 173.6 | +0.3% | +| 512 | none | P2P | 128/16 | 195.6 | 128/16 | 195.7 | +0.0% | +| 1024 | none | P2P | 128/16 | 206.6 | 128/16 | 208.0 | +0.7% | +| 2048 | none | P2P | 128/16 | 213.3 | 256/16 | 215.6 | +1.1% | +| 4096 | none | P2P | 256/16 | 214.2 | 256/16 | 215.0 | +0.4% | + +Combine: mostly +0-6% gains. Notable outlier: fp8dc/nP2P at 128 tokens shows **+49%** — bn=256/wpb=8 hits a performance cliff on MI350X (124.6 GB/s), while bn=224/wpb=8 avoids it (186.2 GB/s). + +## 3. P2P vs non-P2P Detail + +### With fp8_direct_cast quant (hidden=7168) + +| Tokens | P2P BW | P2P lat | P2P bn/wpb | non-P2P BW | non-P2P lat | non-P2P bn/wpb | Winner | Uplift | +|--------|--------|---------|-----------|-----------|------------|---------------|--------|--------| +| 64 | 104.4 | 32.1 us | 112/16 | 108.0 | 30.5 us | 224/8 | non-P2P | +3% | +| 128 | 142.3 | 46.7 us | 112/16 | 186.2 | 36.0 us | 224/8 | non-P2P | +31% | +| 256 | 172.7 | 77.1 us | 112/16 | 252.7 | 52.4 us | 256/16 | non-P2P | +46% | +| 512 | 196.8 | 135.0 us | 128/16 | 309.3 | 85.8 us | 224/16 | non-P2P | +57% | +| 1024 | 208.1 | 254.5 us | 256/16 | 337.3 | 157.0 us | 256/16 | non-P2P | +62% | +| 2048 | 215.1 | 492.1 us | 256/16 | 356.4 | 297.0 us | 256/16 | non-P2P | +66% | +| 4096 | 216.0 | 983.9 us | 256/16 | 369.1 | 575.5 us | 256/16 | non-P2P | +71% | + +### Without quant (hidden=7168) + +| Tokens | P2P BW | P2P lat | P2P bn/wpb | non-P2P BW | non-P2P lat | non-P2P bn/wpb | Winner | Uplift | +|--------|--------|---------|-----------|-----------|------------|---------------|--------|--------| +| 64 | 104.2 | 32.1 us | 112/16 | 98.7 | 33.4 us | 224/8 | P2P | +6% | +| 128 | 142.6 | 46.6 us | 112/16 | 132.8 | 49.5 us | 224/16 | P2P | +7% | +| 256 | 173.6 | 76.4 us | 128/16 | 163.8 | 80.9 us | 224/16 | P2P | +6% | +| 512 | 195.7 | 135.8 us | 128/16 | 180.7 | 146.9 us | 224/16 | P2P | +8% | +| 1024 | 208.0 | 254.9 us | 128/16 | 190.4 | 279.7 us | 256/16 | P2P | +9% | +| 2048 | 215.6 | 492.5 us | 256/16 | 194.3 | 546.6 us | 256/16 | P2P | +11% | +| 4096 | 215.0 | 988.9 us | 256/16 | 192.9 | 1101.6 us | 256/16 | P2P | +11% | + +## 4. Dispatch Detail + +### fp4 (hidden=3584) + +| Tokens | BW (GB/s) | Latency | block_num | warp_per_block | +|--------|-----------|---------|-----------|----------------| +| 64 | 26.3 | 31.5 us | 72 | 8 | +| 128 | 45.9 | 36.6 us | 128 | 8 | +| 256 | 70.8 | 47.1 us | 256 | 8 | +| 512 | 96.3 | 68.8 us | 256 | 8 | +| 1024 | 119.7 | 111.0 us | 256 | 8 | +| 2048 | 140.1 | 189.6 us | 184 | 15 | +| 4096 | 157.7 | 336.6 us | 200 | 15 | + +### fp8_e4m3 (hidden=7168) + +| Tokens | BW (GB/s) | Latency | block_num | warp_per_block | +|--------|-----------|---------|-----------|----------------| +| 64 | 55.2 | 30.4 us | 208 | 4 | +| 128 | 85.9 | 39.0 us | 176 | 6 | +| 256 | 117.6 | 56.7 us | 256 | 4 | +| 512 | 142.1 | 93.2 us | 128 | 8 | +| 1024 | 164.1 | 161.8 us | 208 | 8 | +| 2048 | 182.0 | 291.7 us | 228 | 8 | +| 4096 | 192.1 | 552.9 us | 208 | 16 | + +## 5. Remaining Work + +- [x] EP4 IntraNode (all 4 groups) +- [x] EP2 IntraNode +- [x] IntraNodeLL +- [x] Profiler analysis (static occupancy — see `mi350x_profiler_analysis.md`) diff --git a/docs/ep8_mi350x_IntraNodeLL_tuning_results.md b/docs/ep8_mi350x_IntraNodeLL_tuning_results.md new file mode 100644 index 000000000..24ce67b62 --- /dev/null +++ b/docs/ep8_mi350x_IntraNodeLL_tuning_results.md @@ -0,0 +1,70 @@ +# EP8 IntraNodeLL Tuning Results — MI350X (gfx950, 256 CUs) + +Date: 2026-07-10 +Hardware: 8x MI350X (gfx950, 256 CUs, 288 GB HBM3e), AMD EPYC 9575F +Sweep: full scope (35 block_num x 9 warp_per_block = 315 configs/shape) +Configs: `python/mori/ops/tuning_configs/gfx950_mi350x_IntraNodeLL_ep8_{dispatch,combine}.json` +Token counts: 1, 32, 64 (latency-optimal, small batch regime) + +## 1. Recommended Configs + +### Dispatch — best config per shape + +| Tokens | block_num | warp_per_block | BW (GB/s) | Latency | +|--------|-----------|----------------|-----------|---------| +| 1 | 144 | 16 | 3.95 | 22.3 us | +| 32 | 32 | 10 | 95.0 | 25.7 us | +| 64 | 72 | 10 | 148.8 | 32.7 us | + +### Combine — best config per shape + +| Tokens | block_num | warp_per_block | BW (GB/s) | Latency | +|--------|-----------|----------------|-----------|---------| +| 1 | 32 | 10 | 3.42 | 19.4 us | +| 32 | 56 | 8 | 104.8 | 23.2 us | +| 64 | 112 | 8 | 169.9 | 28.7 us | + +## 2. Before/After — MI350X Tuned vs Shipped Baseline + +Baseline: MI355X bn/wpb configs measured on MI350X hardware. +Note: MI355X only shipped dispatch configs for IntraNodeLL; combine configs are new. + +### Dispatch + +| Tokens | Baseline bn/wpb | Baseline BW | Tuned bn/wpb | Tuned BW | Delta | +|--------|-----------------|-------------|--------------|----------|-------| +| 1 | 1/9 | 2.8 | 144/16 | 3.95 | +41% | +| 32 | 32/9 | 79.0 | 32/10 | 95.0 | +20% | +| 64 | 64/9 | 127.9 | 72/10 | 148.8 | +16% | + +Significant gains across all token counts. MI355X's wpb=9 is suboptimal on MI350X — wpb=10 or 16 performs better. At 1 token, bn=1 severely underutilizes the GPU; bn=144 allows more warps to overlap memory latency. + +### Combine (new — no MI355X baseline existed) + +| Tokens | block_num | warp_per_block | BW (GB/s) | Latency | +|--------|-----------|----------------|-----------|---------| +| 1 | 32 | 10 | 3.42 | 19.4 us | +| 32 | 56 | 8 | 104.8 | 23.2 us | +| 64 | 112 | 8 | 169.9 | 28.7 us | + +## 3. IntraNodeLL vs IntraNode — EP8 Latency Comparison + +| Tokens | LL Dispatch lat | IntraNode Dispatch lat | LL Combine lat | IntraNode Combine lat | +|--------|-----------------|------------------------|----------------|----------------------| +| 1 | 22.3 us | — | 19.4 us | — | +| 32 | 25.7 us | — | 23.2 us | — | +| 64 | 32.7 us | 32.1 us (fp8_e4m3) | 28.7 us | 28.6 us (P2P) | + +At 64 tokens, IntraNodeLL latency is comparable to IntraNode. The LL kernel's advantage is at very small token counts (1-32) where its cooperative 1-block-per-token design avoids the IntraNode kernel's fixed block overhead. + +## 4. Code Changes + +1. **`tests/python/ops/bench_dispatch_combine.py`** — Added `--kernel-type` CLI arg (`IntraNode` or `IntraNodeLL`) to support tuning both kernel types. Threads through to config creation and JSON save. + +## 5. Remaining Work + +- [x] EP8 IntraNode (all 4 groups) +- [x] EP4 IntraNode (all 4 groups) +- [x] EP2 IntraNode (all 4 groups) +- [x] EP8 IntraNodeLL +- [x] Profiler analysis (static occupancy — see `mi350x_profiler_analysis.md`) diff --git a/docs/ep8_mi350x_tuning_results.md b/docs/ep8_mi350x_tuning_results.md new file mode 100644 index 000000000..722679683 --- /dev/null +++ b/docs/ep8_mi350x_tuning_results.md @@ -0,0 +1,177 @@ +# EP8 IntraNode Tuning Results — MI350X (gfx950, 256 CUs) + +Date: 2026-07-06 +Hardware: 8x MI350X (gfx950, 256 CUs, 288 GB HBM3e), AMD EPYC 9575F +Sweep: full scope (35 block_num x 9 warp_per_block = 315 configs/shape) +Configs: `python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep8_{dispatch,combine}.json` + +## 1. Recommended Configs + +### Dispatch — best config per shape + +| Tokens | Dtype | Hidden | block_num | warp_per_block | BW (GB/s) | Latency | +|--------|-------|--------|-----------|----------------|-----------|---------| +| 64 | fp8_e4m3 | 7168 | 128 | 4 | 76.0 | 32.1 us | +| 128 | fp8_e4m3 | 7168 | 208 | 5 | 121.5 | 39.9 us | +| 256 | fp8_e4m3 | 7168 | 256 | 4 | 166.6 | 58.1 us | +| 512 | fp8_e4m3 | 7168 | 228 | 6 | 221.3 | 87.9 us | +| 1024 | fp8_e4m3 | 7168 | 256 | 8 | 272.0 | 142.6 us | +| 2048 | fp8_e4m3 | 7168 | 208 | 8 | 314.4 | 247.6 us | +| 4096 | fp8_e4m3 | 7168 | 256 | 8 | 342.4 | 454.2 us | +| 64 | fp4 | 3584 | 128 | 4 | 35.7 | 34.1 us | +| 128 | fp4 | 3584 | 112 | 15 | 61.4 | 39.8 us | +| 256 | fp4 | 3584 | 256 | 8 | 93.8 | 51.9 us | +| 512 | fp4 | 3584 | 216 | 10 | 128.8 | 75.6 us | +| 1024 | fp4 | 3584 | 184 | 15 | 165.9 | 117.5 us | +| 2048 | fp4 | 3584 | 228 | 12 | 201.7 | 192.8 us | +| 4096 | fp4 | 3584 | 184 | 15 | 237.4 | 328.0 us | + +### Combine — best config per shape (winner of P2P vs non-P2P) + +| Tokens | Quant | Path | block_num | warp_per_block | BW (GB/s) | Latency | +|--------|-------|------|-----------|----------------|-----------|---------| +| 64 | fp8_direct_cast | P2P | 56 | 8 | 168.2 | 28.6 us | +| 128 | fp8_direct_cast | non-P2P | 224 | 8 | 285.9 | 34.1 us | +| 256 | fp8_direct_cast | non-P2P | 224 | 16 | 421.3 | 46.3 us | +| 512 | fp8_direct_cast | non-P2P | 224 | 16 | 525.9 | 73.5 us | +| 1024 | fp8_direct_cast | non-P2P | 256 | 8 | 544.5 | 143.1 us | +| 2048 | fp8_direct_cast | non-P2P | 256 | 16 | 639.0 | 242.6 us | +| 4096 | fp8_direct_cast | non-P2P | 256 | 16 | 642.2 | 483.8 us | +| 64 | none | P2P | 64 | 8 | 170.3 | 28.5 us | +| 128 | none | P2P | 64 | 8 | 244.8 | 39.7 us | +| 256 | none | P2P | 64 | 8 | 313.5 | 62.2 us | +| 512 | none | P2P | 64 | 8 | 368.1 | 105.5 us | +| 1024 | none | P2P | 56 | 10 | 402.4 | 193.3 us | +| 2048 | none | P2P | 72 | 10 | 424.3 | 366.4 us | +| 4096 | none | P2P | 56 | 15 | 435.5 | 713.4 us | + +### P2P vs non-P2P decision rule + +- **With fp8_direct_cast quant:** use P2P only at 64 tokens; use non-P2P at >= 128 tokens (17-51% faster). +- **Without quant:** use P2P at all token counts. + +## 2. Before/After — MI350X Tuned vs Shipped Baseline + +Baseline: MI355X bn/wpb configs measured on MI350X hardware (from sweep data). + +### Dispatch + +| Tokens | Dtype | Baseline bn/wpb | Baseline BW | Tuned bn/wpb | Tuned BW | Delta | +|--------|-------|-----------------|-------------|--------------|----------|-------| +| 64 | fp4 | 64/8 | 34.7 | 128/4 | 35.7 | +2.8% | +| 128 | fp4 | 256/4 | 60.6 | 112/15 | 61.4 | +1.3% | +| 256 | fp4 | 128/16 | 91.1 | 256/8 | 93.8 | +3.0% | +| 512 | fp4 | 256/8 | 126.5 | 216/10 | 128.8 | +1.8% | +| 1024 | fp4 | 256/8 | 163.5 | 184/15 | 165.9 | +1.4% | +| 2048 | fp4 | 256/8 | 201.2 | 228/12 | 201.7 | +0.3% | +| 4096 | fp4 | 256/16 | 222.5 | 184/15 | 237.4 | +6.7% | +| 64 | fp8_e4m3 | 128/4 | 75.3 | 128/4 | 76.0 | +0.8% | +| 128 | fp8_e4m3 | 256/4 | 118.1 | 208/5 | 121.5 | +2.9% | +| 256 | fp8_e4m3 | 256/4 | 165.6 | 256/4 | 166.6 | +0.6% | +| 512 | fp8_e4m3 | 256/8 | 219.2 | 228/6 | 221.3 | +1.0% | +| 1024 | fp8_e4m3 | 256/8 | 266.3 | 256/8 | 272.0 | +2.1% | +| 2048 | fp8_e4m3 | 256/8 | 313.6 | 208/8 | 314.4 | +0.3% | +| 4096 | fp8_e4m3 | 256/8 | 342.4 | 256/8 | 342.4 | +0.0% | + +Dispatch: +0-7% gains. Largest at fp4/4096 tokens (+6.7%). + +### Combine + +| Tokens | Quant | Path | Baseline bn/wpb | Baseline BW | Tuned bn/wpb | Tuned BW | Delta | +|--------|-------|------|-----------------|-------------|--------------|----------|-------| +| 64 | fp8_direct_cast | non-P2P | 256/4 | 147.8 | 112/8 | 149.4 | +1.1% | +| 128 | fp8_direct_cast | non-P2P | 256/8 | 284.6 | 224/8 | 285.9 | +0.5% | +| 256 | fp8_direct_cast | non-P2P | 256/16 | 409.3 | 224/16 | 421.3 | +3.0% | +| 512 | fp8_direct_cast | non-P2P | 256/16 | 523.6 | 224/16 | 525.9 | +0.4% | +| 1024 | fp8_direct_cast | non-P2P | 256/8 | 543.0 | 256/8 | 544.5 | +0.3% | +| 2048 | fp8_direct_cast | non-P2P | 256/16 | 635.2 | 256/16 | 639.0 | +0.6% | +| 4096 | fp8_direct_cast | non-P2P | 256/16 | 642.2 | 256/16 | 642.2 | +0.0% | +| 64 | none | non-P2P | 128/8 | 144.0 | 224/8 | 156.5 | +8.7% | +| 128 | none | non-P2P | 256/8 | 216.6 | 224/8 | 222.2 | +2.6% | +| 256 | none | non-P2P | 256/8 | 282.2 | 224/16 | 288.7 | +2.3% | +| 512 | none | non-P2P | 256/16 | 321.3 | 224/16 | 325.7 | +1.4% | +| 1024 | none | non-P2P | 256/8 | 344.1 | 256/8 | 344.8 | +0.2% | +| 2048 | none | non-P2P | 256/16 | 356.9 | 256/16 | 360.5 | +1.0% | +| 4096 | none | non-P2P | 256/16 | 364.2 | 256/16 | 366.6 | +0.7% | +| 64 | fp8_direct_cast | P2P | 64/8 | 165.1 | 56/8 | 168.2 | +1.9% | +| 128 | fp8_direct_cast | P2P | 64/8 | 243.5 | 64/8 | 244.3 | +0.3% | +| 256 | fp8_direct_cast | P2P | 64/8 | 313.1 | 64/8 | 313.9 | +0.2% | +| 512 | fp8_direct_cast | P2P | 64/8 | 367.1 | 64/8 | 367.9 | +0.2% | +| 1024 | fp8_direct_cast | P2P | 64/8 | 397.6 | 64/16 | 401.8 | +1.1% | +| 2048 | fp8_direct_cast | P2P | 64/8 | 420.0 | 88/8 | 423.5 | +0.9% | +| 4096 | fp8_direct_cast | P2P | 64/8 | 430.9 | 56/15 | 434.1 | +0.7% | +| 64 | none | P2P | 64/8 | 166.0 | 64/8 | 170.2 | +2.6% | +| 128 | none | P2P | 64/8 | 242.8 | 64/8 | 244.8 | +0.8% | +| 256 | none | P2P | 64/8 | 312.0 | 64/8 | 313.5 | +0.5% | +| 512 | none | P2P | 64/8 | 365.8 | 64/8 | 368.1 | +0.6% | +| 1024 | none | P2P | 64/8 | 398.4 | 56/10 | 402.4 | +1.0% | +| 2048 | none | P2P | 64/8 | 419.6 | 72/10 | 424.3 | +1.1% | +| 4096 | none | P2P | 128/4 | 433.8 | 56/15 | 435.5 | +0.4% | + +Combine: +0-9% gains. Largest at none/nP2P/64 tokens (+8.7%) where bn=128→224 better fills MI350X's 256 CUs. + +## 3. P2P vs non-P2P Detail + +### With fp8_direct_cast quant (hidden=7168) + +| Tokens | P2P BW | P2P lat | P2P bn/wpb | non-P2P BW | non-P2P lat | non-P2P bn/wpb | Winner | Uplift | +|--------|--------|---------|-----------|-----------|------------|---------------|--------|--------| +| 64 | 168.2 | 28.6 us | 56/8 | 149.4 | 32.7 us | 112/8 | P2P | +13% | +| 128 | 244.4 | 39.8 us | 64/8 | 285.9 | 34.1 us | 224/8 | non-P2P | +17% | +| 256 | 313.9 | 62.2 us | 64/8 | 421.3 | 46.3 us | 224/16 | non-P2P | +34% | +| 512 | 367.9 | 105.7 us | 64/8 | 525.9 | 73.5 us | 224/16 | non-P2P | +43% | +| 1024 | 401.8 | 194.1 us | 64/16 | 544.5 | 143.1 us | 256/8 | non-P2P | +36% | +| 2048 | 423.5 | 367.3 us | 88/8 | 639.0 | 242.6 us | 256/16 | non-P2P | +51% | +| 4096 | 434.2 | 717.2 us | 56/15 | 642.2 | 483.8 us | 256/16 | non-P2P | +48% | + +### Without quant (hidden=7168) + +| Tokens | P2P BW | P2P lat | P2P bn/wpb | non-P2P BW | non-P2P lat | non-P2P bn/wpb | Winner | Uplift | +|--------|--------|---------|-----------|-----------|------------|---------------|--------|--------| +| 64 | 170.3 | 28.5 us | 64/8 | 156.5 | 30.8 us | 224/8 | P2P | +9% | +| 128 | 244.8 | 39.7 us | 64/8 | 222.2 | 43.9 us | 224/8 | P2P | +10% | +| 256 | 313.5 | 62.2 us | 64/8 | 288.7 | 67.6 us | 224/16 | P2P | +9% | +| 512 | 368.1 | 105.5 us | 64/8 | 325.7 | 119.8 us | 224/16 | P2P | +13% | +| 1024 | 402.4 | 193.3 us | 56/10 | 344.8 | 225.0 us | 256/8 | P2P | +17% | +| 2048 | 424.3 | 366.4 us | 72/10 | 360.5 | 431.4 us | 256/16 | P2P | +18% | +| 4096 | 435.5 | 713.4 us | 56/15 | 366.6 | 848.5 us | 256/16 | P2P | +19% | + +## 4. Dispatch Detail + +### fp8_e4m3 (hidden=7168) + +| Tokens | BW (GB/s) | Latency | block_num | warp_per_block | +|--------|-----------|---------|-----------|----------------| +| 64 | 76.0 | 32.1 us | 128 | 4 | +| 128 | 121.5 | 39.9 us | 208 | 5 | +| 256 | 166.6 | 58.1 us | 256 | 4 | +| 512 | 221.3 | 87.9 us | 228 | 6 | +| 1024 | 272.0 | 142.6 us | 256 | 8 | +| 2048 | 314.4 | 247.6 us | 208 | 8 | +| 4096 | 342.4 | 454.2 us | 256 | 8 | + +### fp4 (hidden=3584) + +| Tokens | BW (GB/s) | Latency | block_num | warp_per_block | +|--------|-----------|---------|-----------|----------------| +| 64 | 35.7 | 34.1 us | 128 | 4 | +| 128 | 61.4 | 39.8 us | 112 | 15 | +| 256 | 93.8 | 51.9 us | 256 | 8 | +| 512 | 128.8 | 75.6 us | 216 | 10 | +| 1024 | 165.9 | 117.5 us | 184 | 15 | +| 2048 | 201.7 | 192.8 us | 228 | 12 | +| 4096 | 237.4 | 328.0 us | 184 | 15 | + +## 5. Code Changes + +1. **`python/mori/ops/tuning_config.py`** — Added `_ARCH_CU_TO_MODEL` fallback table so `detect_gpu_model()` returns `"mi350x"` when `torch.cuda.get_device_properties(0).name` is empty (as on MI350X). Maps `(gfx950, 256 CUs) -> "mi350x"`. + +## 6. Remaining Work + +- [x] EP8 IntraNode (all 4 groups) +- [x] EP4 IntraNode (all 4 groups) +- [x] EP2 IntraNode +- [x] IntraNodeLL +- [x] Profiler analysis (static occupancy — see `mi350x_profiler_analysis.md`; trace profiling needs `ENABLE_PROFILER=ON` rebuild) + + diff --git a/docs/mi350x_profiler_analysis.md b/docs/mi350x_profiler_analysis.md new file mode 100644 index 000000000..d1783cd9b --- /dev/null +++ b/docs/mi350x_profiler_analysis.md @@ -0,0 +1,83 @@ +# MI350X IntraNode Profiler Analysis + +Date: 2026-07-10 +Hardware: 8x MI350X (gfx950, 256 CUs, max 32 waves/CU = 8192 wave slots) + +Note: Runtime trace profiling (`--cmd profile`) requires `ENABLE_PROFILER=ON` build, which is not available in this environment. This analysis is based on static occupancy calculations from the tuned configs. + +## 1. Dispatch Occupancy + +Theoretical occupancy = (block_num × warp_per_block) / 8192 wave slots. + +### EP8 + +| Tokens | Dtype | bn/wpb | Total warps | Warps/CU | Occupancy | BW (GB/s) | +|--------|-------|--------|-------------|----------|-----------|-----------| +| 64 | fp8_e4m3 | 128/4 | 512 | 2.0 | 6% | 76.0 | +| 128 | fp8_e4m3 | 208/5 | 1040 | 4.1 | 13% | 121.5 | +| 256 | fp8_e4m3 | 256/4 | 1024 | 4.0 | 12% | 166.6 | +| 512 | fp8_e4m3 | 228/6 | 1368 | 5.3 | 17% | 221.3 | +| 1024 | fp8_e4m3 | 256/8 | 2048 | 8.0 | 25% | 272.0 | +| 2048 | fp8_e4m3 | 208/8 | 1664 | 6.5 | 20% | 314.4 | +| 4096 | fp8_e4m3 | 256/8 | 2048 | 8.0 | 25% | 342.4 | +| 64 | fp4 | 128/4 | 512 | 2.0 | 6% | 35.7 | +| 512 | fp4 | 216/10 | 2160 | 8.4 | 26% | 128.8 | +| 4096 | fp4 | 184/15 | 2760 | 10.8 | 34% | 237.4 | + +**Observation:** Dispatch occupancy is deliberately low (6-34%). This is XGMI bandwidth-bound, not compute-bound — more warps don't help because the bottleneck is cross-GPU data transfer, not ALU throughput. The sweep correctly converges on configs that have just enough warps to keep XGMI links busy without wasting register/LDS resources. + +### EP2 — Higher occupancy needed + +| Tokens | Dtype | bn/wpb | Warps/CU | Occupancy | BW (GB/s) | +|--------|-------|--------|----------|-----------|-----------| +| 2048 | fp4 | 256/16 | 16.0 | 50% | 81.1 | +| 4096 | fp4 | 255/16 | 15.9 | 50% | 87.9 | +| 2048 | fp8_e4m3 | 128/16 | 8.0 | 25% | 99.1 | +| 4096 | fp8_e4m3 | 128/16 | 8.0 | 25% | 103.2 | + +EP2 dispatch prefers higher occupancy (25-50%) than EP8 (6-25%). With only 1 peer, each GPU must process all its own data — more compute parallelism is needed. + +## 2. Combine Shared Memory Pressure + +Estimated shared memory per block ≈ worldSize × warp_per_block × 64 × sizeof(element). + +### EP8 Combine + +| Tokens | Quant | Path | bn/wpb | Est shmem/block | BW (GB/s) | +|--------|-------|------|--------|-----------------|-----------| +| 64 | fp8_direct_cast | non-P2P | 112/8 | 16 KB | 149.4 | +| 512 | fp8_direct_cast | non-P2P | 224/16 | 32 KB | 525.9 | +| 4096 | fp8_direct_cast | non-P2P | 256/16 | 32 KB | 642.2 | +| 64 | none | P2P | 64/8 | 16 KB | 170.2 | +| 4096 | none | P2P | 56/15 | 30 KB | 435.5 | + +**Key pattern:** Non-P2P combine at large token counts uses 32 KB shared memory (wpb=16). MI350X has 64 KB LDS per CU, so 32 KB allows 2 blocks per CU — sufficient for the bn=224-256 configs. No shared memory pressure issues observed. + +P2P combine uses smaller bn (56-88) with 16-30 KB shmem — well within limits. + +### Shared memory scales with EP size + +| EP | Max shmem/block | Observation | +|----|-----------------|-------------| +| EP8 | 32 KB | 2 blocks/CU possible | +| EP4 | 16 KB | 4 blocks/CU possible | +| EP2 | 8 KB | 8 blocks/CU possible | + +Lower EP sizes use less shared memory per block, allowing more concurrent blocks per CU. This is consistent with the tuned configs: EP2 combine can use more blocks without hitting LDS limits. + +## 3. Config Pattern Summary + +| Pattern | Explanation | +|---------|-------------| +| Dispatch occupancy 6-34% | XGMI bandwidth-bound — more warps don't help | +| EP2 dispatch occupancy higher (25-50%) | Single peer requires more compute parallelism | +| Combine non-P2P prefers bn=224 | Slightly below CU count (256) for cleaner scheduling | +| Combine P2P prefers bn=56-88 | ~1 block per GPU peer, minimizes contention | +| wpb=8-16 for combine | Balances LDS usage vs warp-level parallelism | +| wpb=4-10 for dispatch | Lower warp count sufficient; XGMI-limited | + +## 4. Recommendations + +1. **No LDS pressure issues** — all configs fit comfortably within MI350X's 64 KB/CU LDS +2. **Barrier stalls** cannot be analyzed without trace profiling (`ENABLE_PROFILER=ON` build). If investigating latency outliers, rebuild with profiling enabled and use `analyze_ep_kernel_trace.py` +3. The `bn=256` performance cliff on MI350X (seen in EP2 fp4/256 tokens and EP4 fp8dc/nP2P/128 tokens) warrants investigation — may be related to CU scheduling when block count exactly matches CU count diff --git a/python/mori/ops/tuning_config.py b/python/mori/ops/tuning_config.py index 9e684083a..bf5e44a1d 100644 --- a/python/mori/ops/tuning_config.py +++ b/python/mori/ops/tuning_config.py @@ -127,6 +127,13 @@ def dtype_to_config_str(dtype: torch.dtype) -> str: _gpu_model_cache: str | None = None _gpu_model_detected: bool = False +_ARCH_CU_TO_MODEL: dict[tuple[str, int], str] = { + ("gfx942", 304): "mi300x", + ("gfx942", 228): "mi308x", + ("gfx950", 304): "mi355x", + ("gfx950", 256): "mi350x", +} + def detect_gpu_model() -> str | None: """Detect GPU model from device name, e.g. 'mi300x', 'mi308x'.""" @@ -135,7 +142,8 @@ def detect_gpu_model() -> str | None: return _gpu_model_cache _gpu_model_detected = True try: - name = torch.cuda.get_device_properties(0).name.lower() + props = torch.cuda.get_device_properties(0) + name = props.name.lower() except Exception: return None import re @@ -143,6 +151,13 @@ def detect_gpu_model() -> str | None: m = re.search(r"\bmi\d+\w*", name) if m: _gpu_model_cache = m.group(0) + else: + try: + arch = props.gcnArchName.split(":")[0] + cus = props.multi_processor_count + _gpu_model_cache = _ARCH_CU_TO_MODEL.get((arch, cus)) + except Exception: + pass return _gpu_model_cache diff --git a/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNodeLL_ep8_combine.json b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNodeLL_ep8_combine.json new file mode 100644 index 000000000..99afe09b6 --- /dev/null +++ b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNodeLL_ep8_combine.json @@ -0,0 +1,46 @@ +{ + "version": "1.0", + "gpu_arch": "gfx950", + "gpu_model": "mi350x", + "kernel_type": "IntraNodeLL", + "ep_size": 8, + "phase": "combine", + "rules": [ + { + "dtype": "bf16", + "num_tokens": 1, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 32, + "rdma_block_num": 0, + "warp_per_block": 10, + "bandwidth_gbps": 3.42, + "latency_us": 19.35 + }, + { + "dtype": "bf16", + "num_tokens": 32, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 56, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 104.8, + "latency_us": 23.21 + }, + { + "dtype": "bf16", + "num_tokens": 64, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 112, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 169.92, + "latency_us": 28.72 + } + ] +} diff --git a/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNodeLL_ep8_dispatch.json b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNodeLL_ep8_dispatch.json new file mode 100644 index 000000000..c85560a88 --- /dev/null +++ b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNodeLL_ep8_dispatch.json @@ -0,0 +1,40 @@ +{ + "version": "1.0", + "gpu_arch": "gfx950", + "gpu_model": "mi350x", + "kernel_type": "IntraNodeLL", + "ep_size": 8, + "phase": "dispatch", + "rules": [ + { + "dtype": "bf16", + "num_tokens": 1, + "hidden_dim": 7168, + "block_num": 144, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 3.95, + "latency_us": 22.25 + }, + { + "dtype": "bf16", + "num_tokens": 32, + "hidden_dim": 7168, + "block_num": 32, + "rdma_block_num": 0, + "warp_per_block": 10, + "bandwidth_gbps": 94.96, + "latency_us": 25.72 + }, + { + "dtype": "bf16", + "num_tokens": 64, + "hidden_dim": 7168, + "block_num": 72, + "rdma_block_num": 0, + "warp_per_block": 10, + "bandwidth_gbps": 148.82, + "latency_us": 32.7 + } + ] +} diff --git a/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep2_combine.json b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep2_combine.json new file mode 100644 index 000000000..58574dc5e --- /dev/null +++ b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep2_combine.json @@ -0,0 +1,346 @@ +{ + "version": "1.0", + "gpu_arch": "gfx950", + "gpu_model": "mi350x", + "kernel_type": "IntraNode", + "ep_size": 2, + "phase": "combine", + "rules": [ + { + "dtype": "bf16", + "num_tokens": 64, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 4, + "bandwidth_gbps": 63.42, + "latency_us": 28.8 + }, + { + "dtype": "bf16", + "num_tokens": 128, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 108.72, + "latency_us": 33.65 + }, + { + "dtype": "bf16", + "num_tokens": 256, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 144.05, + "latency_us": 50.8 + }, + { + "dtype": "bf16", + "num_tokens": 512, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 175.66, + "latency_us": 83.35 + }, + { + "dtype": "bf16", + "num_tokens": 1024, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 188.84, + "latency_us": 155.0 + }, + { + "dtype": "bf16", + "num_tokens": 2048, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 202.18, + "latency_us": 289.4 + }, + { + "dtype": "bf16", + "num_tokens": 4096, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 206.83, + "latency_us": 566.05 + }, + { + "dtype": "bf16", + "num_tokens": 64, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 56.97, + "latency_us": 32.25 + }, + { + "dtype": "bf16", + "num_tokens": 128, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 75.4, + "latency_us": 48.55 + }, + { + "dtype": "bf16", + "num_tokens": 256, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 90.12, + "latency_us": 81.2 + }, + { + "dtype": "bf16", + "num_tokens": 512, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 98.75, + "latency_us": 148.2 + }, + { + "dtype": "bf16", + "num_tokens": 1024, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 103.42, + "latency_us": 282.9 + }, + { + "dtype": "bf16", + "num_tokens": 2048, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 103.91, + "latency_us": 563.35 + }, + { + "dtype": "bf16", + "num_tokens": 4096, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 106.49, + "latency_us": 1098.65 + }, + { + "dtype": "bf16", + "num_tokens": 64, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 64, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 58.88, + "latency_us": 31.05 + }, + { + "dtype": "bf16", + "num_tokens": 128, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 112, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 78.58, + "latency_us": 46.55 + }, + { + "dtype": "bf16", + "num_tokens": 256, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 64, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 94.42, + "latency_us": 77.4 + }, + { + "dtype": "bf16", + "num_tokens": 512, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 64, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 104.84, + "latency_us": 139.45 + }, + { + "dtype": "bf16", + "num_tokens": 1024, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 32, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 110.28, + "latency_us": 265.05 + }, + { + "dtype": "bf16", + "num_tokens": 2048, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 152, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 114.7, + "latency_us": 509.75 + }, + { + "dtype": "bf16", + "num_tokens": 4096, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 56, + "rdma_block_num": 0, + "warp_per_block": 15, + "bandwidth_gbps": 115.42, + "latency_us": 1014.45 + }, + { + "dtype": "bf16", + "num_tokens": 64, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 64, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 58.73, + "latency_us": 31.15 + }, + { + "dtype": "bf16", + "num_tokens": 128, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 56, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 78.22, + "latency_us": 46.85 + }, + { + "dtype": "bf16", + "num_tokens": 256, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 94.72, + "latency_us": 77.4 + }, + { + "dtype": "bf16", + "num_tokens": 512, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 64, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 104.93, + "latency_us": 139.2 + }, + { + "dtype": "bf16", + "num_tokens": 1024, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 80, + "rdma_block_num": 0, + "warp_per_block": 14, + "bandwidth_gbps": 110.83, + "latency_us": 263.95 + }, + { + "dtype": "bf16", + "num_tokens": 2048, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 32, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 113.18, + "latency_us": 516.85 + }, + { + "dtype": "bf16", + "num_tokens": 4096, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 160, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 114.97, + "latency_us": 1018.95 + } + ] +} diff --git a/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep2_dispatch.json b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep2_dispatch.json new file mode 100644 index 000000000..4218e1f92 --- /dev/null +++ b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep2_dispatch.json @@ -0,0 +1,150 @@ +{ + "version": "1.0", + "gpu_arch": "gfx950", + "gpu_model": "mi350x", + "kernel_type": "IntraNode", + "ep_size": 2, + "phase": "dispatch", + "rules": [ + { + "dtype": "fp4", + "num_tokens": 64, + "hidden_dim": 3584, + "block_num": 64, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 15.97, + "latency_us": 28.6 + }, + { + "dtype": "fp4", + "num_tokens": 128, + "hidden_dim": 3584, + "block_num": 184, + "rdma_block_num": 0, + "warp_per_block": 6, + "bandwidth_gbps": 28.03, + "latency_us": 32.65 + }, + { + "dtype": "fp4", + "num_tokens": 256, + "hidden_dim": 3584, + "block_num": 208, + "rdma_block_num": 0, + "warp_per_block": 10, + "bandwidth_gbps": 43.25, + "latency_us": 42.3 + }, + { + "dtype": "fp4", + "num_tokens": 512, + "hidden_dim": 3584, + "block_num": 208, + "rdma_block_num": 0, + "warp_per_block": 10, + "bandwidth_gbps": 58.78, + "latency_us": 62.35 + }, + { + "dtype": "fp4", + "num_tokens": 1024, + "hidden_dim": 3584, + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 70.71, + "latency_us": 103.25 + }, + { + "dtype": "fp4", + "num_tokens": 2048, + "hidden_dim": 3584, + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 81.06, + "latency_us": 180.45 + }, + { + "dtype": "fp4", + "num_tokens": 4096, + "hidden_dim": 3584, + "block_num": 255, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 87.94, + "latency_us": 332.5 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 64, + "hidden_dim": 7168, + "block_num": 64, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 32.73, + "latency_us": 27.95 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 128, + "hidden_dim": 7168, + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 49.56, + "latency_us": 36.85 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 256, + "hidden_dim": 7168, + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 67.97, + "latency_us": 53.9 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 512, + "hidden_dim": 7168, + "block_num": 176, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 82.42, + "latency_us": 88.95 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 1024, + "hidden_dim": 7168, + "block_num": 208, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 92.88, + "latency_us": 157.5 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 2048, + "hidden_dim": 7168, + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 99.07, + "latency_us": 295.4 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 4096, + "hidden_dim": 7168, + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 103.19, + "latency_us": 567.1 + } + ] +} diff --git a/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep4_combine.json b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep4_combine.json new file mode 100644 index 000000000..aa4b2ab20 --- /dev/null +++ b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep4_combine.json @@ -0,0 +1,346 @@ +{ + "version": "1.0", + "gpu_arch": "gfx950", + "gpu_model": "mi350x", + "kernel_type": "IntraNode", + "ep_size": 4, + "phase": "combine", + "rules": [ + { + "dtype": "bf16", + "num_tokens": 64, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 108.02, + "latency_us": 30.48 + }, + { + "dtype": "bf16", + "num_tokens": 128, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 186.22, + "latency_us": 35.95 + }, + { + "dtype": "bf16", + "num_tokens": 256, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 252.69, + "latency_us": 52.42 + }, + { + "dtype": "bf16", + "num_tokens": 512, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 309.29, + "latency_us": 85.83 + }, + { + "dtype": "bf16", + "num_tokens": 1024, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 337.34, + "latency_us": 156.95 + }, + { + "dtype": "bf16", + "num_tokens": 2048, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 356.35, + "latency_us": 297.0 + }, + { + "dtype": "bf16", + "num_tokens": 4096, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 369.1, + "latency_us": 575.52 + }, + { + "dtype": "bf16", + "num_tokens": 64, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 98.67, + "latency_us": 33.38 + }, + { + "dtype": "bf16", + "num_tokens": 128, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 132.82, + "latency_us": 49.45 + }, + { + "dtype": "bf16", + "num_tokens": 256, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 163.77, + "latency_us": 80.92 + }, + { + "dtype": "bf16", + "num_tokens": 512, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 180.69, + "latency_us": 146.9 + }, + { + "dtype": "bf16", + "num_tokens": 1024, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 190.39, + "latency_us": 279.65 + }, + { + "dtype": "bf16", + "num_tokens": 2048, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 194.26, + "latency_us": 546.62 + }, + { + "dtype": "bf16", + "num_tokens": 4096, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 192.92, + "latency_us": 1101.6 + }, + { + "dtype": "bf16", + "num_tokens": 64, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 112, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 104.36, + "latency_us": 32.05 + }, + { + "dtype": "bf16", + "num_tokens": 128, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 112, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 142.34, + "latency_us": 46.65 + }, + { + "dtype": "bf16", + "num_tokens": 256, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 112, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 172.68, + "latency_us": 77.12 + }, + { + "dtype": "bf16", + "num_tokens": 512, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 196.78, + "latency_us": 134.98 + }, + { + "dtype": "bf16", + "num_tokens": 1024, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 208.09, + "latency_us": 254.45 + }, + { + "dtype": "bf16", + "num_tokens": 2048, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 215.11, + "latency_us": 492.1 + }, + { + "dtype": "bf16", + "num_tokens": 4096, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 215.97, + "latency_us": 983.9 + }, + { + "dtype": "bf16", + "num_tokens": 64, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 112, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 104.21, + "latency_us": 32.1 + }, + { + "dtype": "bf16", + "num_tokens": 128, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 112, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 142.6, + "latency_us": 46.58 + }, + { + "dtype": "bf16", + "num_tokens": 256, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 173.57, + "latency_us": 76.42 + }, + { + "dtype": "bf16", + "num_tokens": 512, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 195.66, + "latency_us": 135.82 + }, + { + "dtype": "bf16", + "num_tokens": 1024, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 208.0, + "latency_us": 254.92 + }, + { + "dtype": "bf16", + "num_tokens": 2048, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 215.62, + "latency_us": 492.5 + }, + { + "dtype": "bf16", + "num_tokens": 4096, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 214.96, + "latency_us": 988.87 + } + ] +} diff --git a/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep4_dispatch.json b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep4_dispatch.json new file mode 100644 index 000000000..ca9cb03c2 --- /dev/null +++ b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep4_dispatch.json @@ -0,0 +1,150 @@ +{ + "version": "1.0", + "gpu_arch": "gfx950", + "gpu_model": "mi350x", + "kernel_type": "IntraNode", + "ep_size": 4, + "phase": "dispatch", + "rules": [ + { + "dtype": "fp4", + "num_tokens": 64, + "hidden_dim": 3584, + "block_num": 72, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 26.33, + "latency_us": 31.52 + }, + { + "dtype": "fp4", + "num_tokens": 128, + "hidden_dim": 3584, + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 45.86, + "latency_us": 36.55 + }, + { + "dtype": "fp4", + "num_tokens": 256, + "hidden_dim": 3584, + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 70.77, + "latency_us": 47.08 + }, + { + "dtype": "fp4", + "num_tokens": 512, + "hidden_dim": 3584, + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 96.26, + "latency_us": 68.75 + }, + { + "dtype": "fp4", + "num_tokens": 1024, + "hidden_dim": 3584, + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 119.66, + "latency_us": 111.1 + }, + { + "dtype": "fp4", + "num_tokens": 2048, + "hidden_dim": 3584, + "block_num": 184, + "rdma_block_num": 0, + "warp_per_block": 15, + "bandwidth_gbps": 140.09, + "latency_us": 189.6 + }, + { + "dtype": "fp4", + "num_tokens": 4096, + "hidden_dim": 3584, + "block_num": 200, + "rdma_block_num": 0, + "warp_per_block": 15, + "bandwidth_gbps": 157.7, + "latency_us": 336.6 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 64, + "hidden_dim": 7168, + "block_num": 208, + "rdma_block_num": 0, + "warp_per_block": 4, + "bandwidth_gbps": 55.18, + "latency_us": 30.38 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 128, + "hidden_dim": 7168, + "block_num": 176, + "rdma_block_num": 0, + "warp_per_block": 6, + "bandwidth_gbps": 85.89, + "latency_us": 38.97 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 256, + "hidden_dim": 7168, + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 4, + "bandwidth_gbps": 117.58, + "latency_us": 56.65 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 512, + "hidden_dim": 7168, + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 142.13, + "latency_us": 93.15 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 1024, + "hidden_dim": 7168, + "block_num": 208, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 164.07, + "latency_us": 161.8 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 2048, + "hidden_dim": 7168, + "block_num": 228, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 181.98, + "latency_us": 291.65 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 4096, + "hidden_dim": 7168, + "block_num": 208, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 192.07, + "latency_us": 552.9 + } + ] +} diff --git a/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep8_combine.json b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep8_combine.json new file mode 100644 index 000000000..426d468b9 --- /dev/null +++ b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep8_combine.json @@ -0,0 +1,346 @@ +{ + "version": "1.0", + "gpu_arch": "gfx950", + "gpu_model": "mi350x", + "kernel_type": "IntraNode", + "ep_size": 8, + "phase": "combine", + "rules": [ + { + "dtype": "bf16", + "num_tokens": 64, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 112, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 149.38, + "latency_us": 32.65 + }, + { + "dtype": "bf16", + "num_tokens": 128, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 285.89, + "latency_us": 34.12 + }, + { + "dtype": "bf16", + "num_tokens": 256, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 421.34, + "latency_us": 46.29 + }, + { + "dtype": "bf16", + "num_tokens": 512, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 525.91, + "latency_us": 73.51 + }, + { + "dtype": "bf16", + "num_tokens": 1024, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 544.54, + "latency_us": 143.05 + }, + { + "dtype": "bf16", + "num_tokens": 2048, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 638.98, + "latency_us": 242.56 + }, + { + "dtype": "bf16", + "num_tokens": 4096, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "fp8_direct_cast", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 642.2, + "latency_us": 483.78 + }, + { + "dtype": "bf16", + "num_tokens": 64, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 156.52, + "latency_us": 30.83 + }, + { + "dtype": "bf16", + "num_tokens": 128, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 222.2, + "latency_us": 43.9 + }, + { + "dtype": "bf16", + "num_tokens": 256, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 288.69, + "latency_us": 67.56 + }, + { + "dtype": "bf16", + "num_tokens": 512, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 224, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 325.69, + "latency_us": 119.75 + }, + { + "dtype": "bf16", + "num_tokens": 1024, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 344.82, + "latency_us": 225.04 + }, + { + "dtype": "bf16", + "num_tokens": 2048, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 360.49, + "latency_us": 431.36 + }, + { + "dtype": "bf16", + "num_tokens": 4096, + "hidden_dim": 7168, + "zero_copy": false, + "quant_type": "none", + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 366.6, + "latency_us": 848.52 + }, + { + "dtype": "bf16", + "num_tokens": 64, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 56, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 168.24, + "latency_us": 28.63 + }, + { + "dtype": "bf16", + "num_tokens": 128, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 64, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 244.35, + "latency_us": 39.77 + }, + { + "dtype": "bf16", + "num_tokens": 256, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 64, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 313.85, + "latency_us": 62.16 + }, + { + "dtype": "bf16", + "num_tokens": 512, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 64, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 367.93, + "latency_us": 105.74 + }, + { + "dtype": "bf16", + "num_tokens": 1024, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 64, + "rdma_block_num": 0, + "warp_per_block": 16, + "bandwidth_gbps": 401.8, + "latency_us": 194.14 + }, + { + "dtype": "bf16", + "num_tokens": 2048, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 88, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 423.54, + "latency_us": 367.31 + }, + { + "dtype": "bf16", + "num_tokens": 4096, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "fp8_direct_cast", + "block_num": 56, + "rdma_block_num": 0, + "warp_per_block": 15, + "bandwidth_gbps": 434.15, + "latency_us": 717.24 + }, + { + "dtype": "bf16", + "num_tokens": 64, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 64, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 170.25, + "latency_us": 28.46 + }, + { + "dtype": "bf16", + "num_tokens": 128, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 64, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 244.84, + "latency_us": 39.7 + }, + { + "dtype": "bf16", + "num_tokens": 256, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 64, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 313.48, + "latency_us": 62.22 + }, + { + "dtype": "bf16", + "num_tokens": 512, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 64, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 368.13, + "latency_us": 105.47 + }, + { + "dtype": "bf16", + "num_tokens": 1024, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 56, + "rdma_block_num": 0, + "warp_per_block": 10, + "bandwidth_gbps": 402.41, + "latency_us": 193.34 + }, + { + "dtype": "bf16", + "num_tokens": 2048, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 72, + "rdma_block_num": 0, + "warp_per_block": 10, + "bandwidth_gbps": 424.33, + "latency_us": 366.36 + }, + { + "dtype": "bf16", + "num_tokens": 4096, + "hidden_dim": 7168, + "zero_copy": true, + "quant_type": "none", + "block_num": 56, + "rdma_block_num": 0, + "warp_per_block": 15, + "bandwidth_gbps": 435.5, + "latency_us": 713.4 + } + ] +} diff --git a/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep8_dispatch.json b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep8_dispatch.json new file mode 100644 index 000000000..0e39fcfc7 --- /dev/null +++ b/python/mori/ops/tuning_configs/gfx950_mi350x_IntraNode_ep8_dispatch.json @@ -0,0 +1,160 @@ +{ + "version": "1.0", + "gpu_arch": "gfx950", + "gpu_model": "mi350x", + "kernel_type": "IntraNode", + "ep_size": 8, + "phase": "dispatch", + "rules": [ + { + "dtype": "bf16", + "num_tokens": 64, + "hidden_dim": 7168, + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 4, + "bandwidth_gbps": 127.8, + "latency_us": 38.0 + }, + { + "dtype": "fp4", + "num_tokens": 64, + "hidden_dim": 3584, + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 4, + "bandwidth_gbps": 35.72, + "latency_us": 34.14 + }, + { + "dtype": "fp4", + "num_tokens": 128, + "hidden_dim": 3584, + "block_num": 112, + "rdma_block_num": 0, + "warp_per_block": 15, + "bandwidth_gbps": 61.36, + "latency_us": 39.8 + }, + { + "dtype": "fp4", + "num_tokens": 256, + "hidden_dim": 3584, + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 93.81, + "latency_us": 51.9 + }, + { + "dtype": "fp4", + "num_tokens": 512, + "hidden_dim": 3584, + "block_num": 216, + "rdma_block_num": 0, + "warp_per_block": 10, + "bandwidth_gbps": 128.78, + "latency_us": 75.55 + }, + { + "dtype": "fp4", + "num_tokens": 1024, + "hidden_dim": 3584, + "block_num": 184, + "rdma_block_num": 0, + "warp_per_block": 15, + "bandwidth_gbps": 165.88, + "latency_us": 117.48 + }, + { + "dtype": "fp4", + "num_tokens": 2048, + "hidden_dim": 3584, + "block_num": 228, + "rdma_block_num": 0, + "warp_per_block": 12, + "bandwidth_gbps": 201.73, + "latency_us": 192.84 + }, + { + "dtype": "fp4", + "num_tokens": 4096, + "hidden_dim": 3584, + "block_num": 184, + "rdma_block_num": 0, + "warp_per_block": 15, + "bandwidth_gbps": 237.41, + "latency_us": 327.98 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 64, + "hidden_dim": 7168, + "block_num": 128, + "rdma_block_num": 0, + "warp_per_block": 4, + "bandwidth_gbps": 75.95, + "latency_us": 32.14 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 128, + "hidden_dim": 7168, + "block_num": 208, + "rdma_block_num": 0, + "warp_per_block": 5, + "bandwidth_gbps": 121.52, + "latency_us": 39.89 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 256, + "hidden_dim": 7168, + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 4, + "bandwidth_gbps": 166.64, + "latency_us": 58.14 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 512, + "hidden_dim": 7168, + "block_num": 228, + "rdma_block_num": 0, + "warp_per_block": 6, + "bandwidth_gbps": 221.32, + "latency_us": 87.86 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 1024, + "hidden_dim": 7168, + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 272.03, + "latency_us": 142.62 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 2048, + "hidden_dim": 7168, + "block_num": 208, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 314.39, + "latency_us": 247.56 + }, + { + "dtype": "fp8_e4m3", + "num_tokens": 4096, + "hidden_dim": 7168, + "block_num": 256, + "rdma_block_num": 0, + "warp_per_block": 8, + "bandwidth_gbps": 342.44, + "latency_us": 454.18 + } + ] +} diff --git a/tests/python/ops/bench_dispatch_combine.py b/tests/python/ops/bench_dispatch_combine.py index 0734ba07c..8be3d5e19 100644 --- a/tests/python/ops/bench_dispatch_combine.py +++ b/tests/python/ops/bench_dispatch_combine.py @@ -731,6 +731,7 @@ def _save_intranode_tuning_result( zero_copy=True, best_disp_lat=None, best_comb_lat=None, + kernel_type_str="IntraNode", ): from pathlib import Path from mori.ops.tuning_config import ( @@ -751,7 +752,7 @@ def _save_intranode_tuning_result( metadata = { "gpu_arch": gpu_arch, "gpu_model": gpu_model, - "kernel_type": "IntraNode", + "kernel_type": kernel_type_str, "ep_size": world_size, } @@ -791,7 +792,7 @@ def _save_intranode_tuning_result( repo_tuning_dir / build_config_filename( gpu_arch, - "IntraNode", + kernel_type_str, world_size, gpu_model, "dispatch", @@ -801,7 +802,7 @@ def _save_intranode_tuning_result( repo_tuning_dir / build_config_filename( gpu_arch, - "IntraNode", + kernel_type_str, world_size, gpu_model, "combine", @@ -898,6 +899,7 @@ def _bench_dispatch_combine( input_shift=0.0, force_scale_active=False, report_scale_stats=False, + kernel_type_str="IntraNode", ): if combine_data_type is None: combine_data_type = data_type @@ -909,6 +911,12 @@ def _bench_dispatch_combine( else: combine_hidden_dim = hidden_dim + _kernel_type_map = { + "IntraNode": mori.ops.EpDispatchCombineKernelType.IntraNode, + "IntraNodeLL": mori.ops.EpDispatchCombineKernelType.IntraNodeLL, + } + kernel_type_enum = _kernel_type_map[kernel_type_str] + if quant_type == "fp8_direct_cast" and combine_data_type is not torch.bfloat16: raise ValueError( "fp8_direct_cast quant requires combine dtype to be bfloat16, " @@ -940,6 +948,7 @@ def _bench_dispatch_combine( use_external_inp_buf=not zero_copy, # zero-copy mode requires use_external_inp_buf=False gpu_per_node=world_size, quant_type=quant_type, + kernel_type=kernel_type_enum, ) with TorchDistContext(rank=rank, world_size=world_size, master_port=port): mori.shmem.shmem_torch_process_group_init("default") @@ -1205,6 +1214,7 @@ def _bench_dispatch_combine( zero_copy=bool(zero_copy), best_disp_lat=best_disp_lat, best_comb_lat=best_comb_lat, + kernel_type_str=kernel_type_str, ) else: @@ -1236,6 +1246,7 @@ def bench_dispatch_combine( input_shift=0.0, force_scale_active=False, report_scale_stats=False, + kernel_type_str="IntraNode", ): if combine_data_type is None: combine_data_type = dtype @@ -1268,6 +1279,7 @@ def bench_dispatch_combine( input_shift, force_scale_active, report_scale_stats, + kernel_type_str, ), nprocs=world_size, join=True, @@ -1413,6 +1425,13 @@ def bench_dispatch_combine( "MORI_FP8_COMBINE_SCALE_DIM for its internal scale layout." ), ) + parser.add_argument( + "--kernel-type", + type=str, + default="IntraNode", + choices=["IntraNode", "IntraNodeLL"], + help="Kernel type to tune (default: IntraNode)", + ) parser.add_argument( "--input-dist", type=str, @@ -1537,4 +1556,5 @@ def bench_dispatch_combine( input_shift=args.input_shift, force_scale_active=bool(args.force_scale_active), report_scale_stats=bool(args.report_scale_stats), + kernel_type_str=args.kernel_type, )